@torrent-tv/proxy 2.75.0 → 2.76.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1509 -1461
- package/CLAUDE.md +11 -1
- package/biome.json +182 -1
- package/docs/container-architecture.md +2 -1
- package/docs/encode-run-state.md +1 -1
- package/knip.json +14 -0
- package/package.json +1 -1
- package/routes/api/sources/warm/post.js +1 -1
- package/routes/api/transcode-sessions/post.js +185 -185
- package/routes/api/transcode-sessions/progress/get.js +5 -1
- package/routes/transcode/audio-file/get.js +11 -1
- package/routes/transcode/audio-warm/get.js +11 -1
- package/routes/transcode/session-file/get.js +1 -1
- package/routes/transcode/variant-file/get.js +10 -1
- package/scripts/render-run-graph.js +2 -2
- package/server.js +25 -0
- package/services/audio-inventory.js +206 -201
- package/services/container/AviContainer.js +1 -1
- package/services/container/Container.js +33 -1
- package/services/container/MatroskaContainer.js +1 -1
- package/services/container/Mp4Container.js +1 -1
- package/services/container/SubtitleFileContainer.js +0 -1
- package/services/controllers/SubtitleController.js +128 -128
- package/services/demand/index.js +7 -10
- package/services/download/registry.js +0 -14
- package/services/encode/CoverageMap.js +281 -0
- package/services/encode/EncodePlan.js +255 -0
- package/services/encode/EncodeRun.js +587 -0
- package/services/encode/Encoder.js +84 -0
- package/services/encode/NvencEncoder.js +45 -0
- package/services/encode/QsvEncoder.js +47 -0
- package/services/encode/SegmentDemand.js +0 -0
- package/services/encode/SegmentStore.js +529 -0
- package/services/encode/SoftwareEncoder.js +111 -0
- package/services/encode/V4l2m2mEncoder.js +53 -0
- package/services/encode/VaapiEncoder.js +53 -0
- package/services/encode/args.js +200 -0
- package/services/{encode-exit.js → encode/encode-exit.js} +17 -0
- package/services/encode/index.js +9 -0
- package/services/encode/run-command.js +647 -0
- package/services/hls-session-manager.js +11100 -10711
- package/services/hwaccel.js +1688 -1992
- package/services/orchestrators/EncodeOrchestrator.js +359 -0
- package/services/output/LiveOutputs.js +213 -0
- package/services/output/Output.js +94 -0
- package/services/output/OutputSpec.js +195 -0
- package/services/output/Timeline.js +220 -0
- package/services/output/index.js +1 -0
- package/services/output/ladder.js +26 -0
- package/services/playback-planner.js +806 -775
- package/services/produced-index.js +222 -300
- package/services/source/SourceFile.js +346 -0
- package/services/{sidecar-files.js → torrent/files.js} +107 -11
- package/services/torrent/naming.js +619 -0
- package/services/torrent-worker/client.js +10 -0
- package/services/torrent-worker/container-tracks.js +71 -43
- package/services/torrent-worker/pool-adapter.js +18 -0
- package/services/torrent-worker/protocol.js +7 -0
- package/services/torrent-worker/subtitle-cues.js +549 -549
- package/services/torrent-worker/worker.js +18 -0
- package/services/tracks/TextSubtitleTrack.js +287 -287
- package/services/tracks/index.js +15 -14
- package/services/viewer/Viewer.js +145 -0
- package/services/viewer/Viewers.js +124 -0
- package/test/auto-quality-step.test.js +508 -506
- package/test/behind-head-repair.test.js +17 -7
- package/test/coverage-map.test.js +153 -0
- package/test/cut-times-timeline.test.js +6 -5
- package/test/cuts-follow-published-grid.test.js +4 -4
- package/test/decode-cost.test.js +31 -12
- package/test/encode-exit.test.js +1 -1
- package/test/encode-orchestrator.test.js +196 -0
- package/test/encode-plan.test.js +245 -0
- package/test/encode-run-state.test.js +2 -2
- package/test/encode-run.test.js +168 -0
- package/test/encoder-kinds.test.js +122 -0
- package/test/held-request-width.test.js +9 -3
- package/test/helpers/encode-run.js +128 -0
- package/test/keyframe-index-accuracy.test.js +19 -12
- package/test/keyframes-belong-to-the-file.test.js +132 -0
- package/test/orchestrator-wired.test.js +164 -0
- package/test/output-shape.test.js +68 -0
- package/test/output-spec.test.js +157 -0
- package/test/produced-copy-choice.test.js +58 -92
- package/test/produced-index.test.js +142 -188
- package/test/quality-variants.test.js +1079 -1075
- package/test/run-graph-drift.test.js +1 -1
- package/test/run-intervals.test.js +329 -0
- package/test/run-position-follows-published-grid.test.js +4 -4
- package/test/seek-landing.test.js +8 -8
- package/test/seek-target-not-superseded.test.js +21 -9
- package/test/segment-demand.test.js +82 -0
- package/test/segment-serve-wiring.test.js +47 -52
- package/test/segment-store.test.js +187 -0
- package/test/segments-are-shared.test.js +175 -0
- package/test/sidecar-naming.test.js +142 -0
- package/test/source-file.test.js +133 -0
- package/test/stale-request-after-seek.test.js +18 -12
- package/test/subtitle-language.test.js +252 -252
- package/test/timeline.test.js +95 -0
- package/test/{sidecar-files.test.js → torrent-files.test.js} +44 -1
- package/test/torrent-naming.test.js +255 -0
- package/test/tracks-begin-together.test.js +44 -32
- package/test/two-viewers-one-picture.test.js +347 -0
- package/test/viewer-outputs.test.js +273 -0
- package/test/viewer.test.js +91 -0
- package/utils/perf.js +1 -63
- package/services/container/index.js +0 -6
- package/services/controllers/index.js +0 -2
- package/services/download/index.js +0 -8
- package/services/orchestrators/index.js +0 -2
- /package/services/{encode-run-state.js → encode/encode-run-state.js} +0 -0
- /package/services/{language-detect.js → tracks/language-detect.js} +0 -0
|
@@ -1,252 +1,252 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file What the language detector is FED.
|
|
3
|
-
*
|
|
4
|
-
* Field 2026-09-01: a Russian subtitle file was offered to the viewer as
|
|
5
|
-
* `English (Stan WarHammer & Nesitach)`. The detector was not at fault — it was
|
|
6
|
-
* handed the raw `.ass` file, which is half Latin markup, while the markup-free
|
|
7
|
-
* WebVTT it was about to serve sat in the variable beside it. Measured on that
|
|
8
|
-
* file: 5040 Latin characters against 5983 Cyrillic, `franc(the file) = eng`,
|
|
9
|
-
* `franc(the dialogue) = rus`.
|
|
10
|
-
* `research/subtitle-language-ass-markup-2026-09-01.md`.
|
|
11
|
-
*
|
|
12
|
-
* These checks pin the input at each of the three places a language is read.
|
|
13
|
-
*
|
|
14
|
-
* **On the size of the fixtures.** franc's answer among the Cyrillic languages
|
|
15
|
-
* is not stable on a small sample: measured 2026-09-01 by growing one Russian
|
|
16
|
-
* text line by line, it answered `bul` at 129 characters, `srp` at 158, 241 and
|
|
17
|
-
* `bul` again at 292, then `rus` at every length from 337 to 881. So the
|
|
18
|
-
* dialogue here is ~880 Cyrillic characters — past that boundary by a factor of
|
|
19
|
-
* about 2.6, and still an order of magnitude below a real episode's 5983. The
|
|
20
|
-
* fixtures are NOT sized to make these checks pass; they are sized to resemble
|
|
21
|
-
* the thing. That instability is a separate defect and is recorded as one.
|
|
22
|
-
*/
|
|
23
|
-
|
|
24
|
-
import test from "node:test";
|
|
25
|
-
import assert from "node:assert/strict";
|
|
26
|
-
|
|
27
|
-
import { detectLanguage } from "../services/language-detect.js";
|
|
28
|
-
import { SubtitleFileContainer } from "../services/container/SubtitleFileContainer.js";
|
|
29
|
-
import { TextSubtitleTrack } from "../services/tracks/TextSubtitleTrack.js";
|
|
30
|
-
import { MatroskaContainer } from "../services/container/MatroskaContainer.js";
|
|
31
|
-
|
|
32
|
-
/** Varied Russian dialogue, the length a few minutes of an episode carries. */
|
|
33
|
-
const RUSSIAN_DIALOGUE = [
|
|
34
|
-
"Ты видишь их?",
|
|
35
|
-
"Да.",
|
|
36
|
-
"Но не столько вижу, сколько чувствую.",
|
|
37
|
-
"Скорее всего, они будут здесь с минуты на минуту.",
|
|
38
|
-
"Почему господа октябристы решили напасть именно сейчас?",
|
|
39
|
-
"Им никогда не преодолеть эти стены.",
|
|
40
|
-
"Мы держали эту крепость три года и продержим ещё столько же.",
|
|
41
|
-
"Готовьте лучников на восточной стороне.",
|
|
42
|
-
"Если они подойдут ближе, мы откроем огонь без предупреждения.",
|
|
43
|
-
"Я не собираюсь умирать здесь, в этой богом забытой дыре.",
|
|
44
|
-
"Тогда возьми меч и вставай рядом со мной.",
|
|
45
|
-
"Сколько у нас осталось воды и хлеба?",
|
|
46
|
-
"На неделю, если не считать раненых.",
|
|
47
|
-
"Раненых считать придётся, они тоже люди.",
|
|
48
|
-
"Отправь гонца к южным воротам и передай приказ отступать.",
|
|
49
|
-
"Он не успеет, дорога перерезана ещё вчера вечером.",
|
|
50
|
-
"Значит пойду сам, а ты останешься за меня командовать.",
|
|
51
|
-
"Это безумие, и ты прекрасно об этом знаешь.",
|
|
52
|
-
"Безумие — это сидеть и ждать, пока нас перебьют по одному.",
|
|
53
|
-
"Хорошо. Но возьми с собой хотя бы двоих.",
|
|
54
|
-
"Двоих я возьму. Больше не могу себе позволить.",
|
|
55
|
-
"Береги себя. Мы будем держать стену до последнего.",
|
|
56
|
-
"Я знаю. Именно поэтому я и ухожу спокойно.",
|
|
57
|
-
"Смотри, дым над лесом. Они уже жгут деревни.",
|
|
58
|
-
"Тогда времени у нас меньше, чем мы думали."
|
|
59
|
-
];
|
|
60
|
-
|
|
61
|
-
/** `hh:mm:ss.cc` for the ASS `Start`/`End` fields. */
|
|
62
|
-
function assTime(seconds) {
|
|
63
|
-
const mm = String(Math.floor(seconds / 60)).padStart(2, "0");
|
|
64
|
-
const ss = String(seconds % 60).padStart(2, "0");
|
|
65
|
-
return `0:${mm}:${ss}.00`;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* An ASS file in the shape a fansub release ships: an Aegisub header, a styles
|
|
70
|
-
* section with English font and style names, and Russian dialogue carrying
|
|
71
|
-
* override groups. Modelled on the field file.
|
|
72
|
-
*/
|
|
73
|
-
const RUSSIAN_ASS = `[Script Info]
|
|
74
|
-
Title: Default Aegisub file
|
|
75
|
-
ScriptType: v4.00+
|
|
76
|
-
PlayResX: 1280
|
|
77
|
-
PlayResY: 720
|
|
78
|
-
Original Translation: Nesitach
|
|
79
|
-
Original Editing: Stan WarHammer
|
|
80
|
-
WrapStyle: 0
|
|
81
|
-
ScaledBorderAndShadow: no
|
|
82
|
-
Video Aspect Ratio: c1.77778
|
|
83
|
-
YCbCr Matrix: TV.601
|
|
84
|
-
Aegisub Video Aspect Ratio: c1.777778
|
|
85
|
-
|
|
86
|
-
[Aegisub Project Garbage]
|
|
87
|
-
Last Style Storage: Default
|
|
88
|
-
Audio File: [HorribleSubs] Drifters - 03 [720p].mkv
|
|
89
|
-
Video File: [HorribleSubs] Drifters - 03 [720p].mkv
|
|
90
|
-
Video AR Mode: 4
|
|
91
|
-
Video AR Value: 1.777778
|
|
92
|
-
Video Zoom Percent: 0.600000
|
|
93
|
-
Scroll Position: 256
|
|
94
|
-
Active Line: 257
|
|
95
|
-
Video Position: 33159
|
|
96
|
-
|
|
97
|
-
[V4+ Styles]
|
|
98
|
-
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
|
|
99
|
-
Style: Default,Trebuchet MS,54,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,2.5,1.5,2,20,20,25,1
|
|
100
|
-
Style: Signs,Times New Roman,48,&H00FFFF00,&H000000FF,&H00202020,&H00000000,-1,0,0,0,100,100,0,0,1,2,1,8,20,20,20,1
|
|
101
|
-
Style: Italics,Trebuchet MS Italic,54,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,-1,0,0,100,100,0,0,1,2.5,1.5,2,20,20,25,1
|
|
102
|
-
|
|
103
|
-
[Events]
|
|
104
|
-
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
|
|
105
|
-
${RUSSIAN_DIALOGUE.map((line, index) => {
|
|
106
|
-
const style = index % 7 === 6 ? "Signs" : "Default";
|
|
107
|
-
const tag = index % 5 === 4 ? "{\\pos(640,620)}" : "";
|
|
108
|
-
return `Dialogue: 0,${assTime(index * 4)},${assTime(index * 4 + 3)},${style},,0,0,0,,${tag}${line}`;
|
|
109
|
-
}).join("\n")}
|
|
110
|
-
`;
|
|
111
|
-
|
|
112
|
-
test("a Russian .ass is reported as Russian, and its markup does not reach the detector", () => {
|
|
113
|
-
const vtt = SubtitleFileContainer.toVtt(RUSSIAN_ASS, ".ass");
|
|
114
|
-
assert.ok(vtt.startsWith("WEBVTT"), "the conversion produced a WebVTT document");
|
|
115
|
-
|
|
116
|
-
const spoken = TextSubtitleTrack.cueTextOfVtt(vtt);
|
|
117
|
-
// Every one of these is a piece of the file the viewer never reads, and each
|
|
118
|
-
// was competing with the dialogue for the detector's answer.
|
|
119
|
-
for (const markup of [
|
|
120
|
-
"Aegisub", "Script Info", "V4+ Styles", "Format:", "Dialogue:",
|
|
121
|
-
"Trebuchet MS", "Times New Roman", "Default", "Signs", "pos(",
|
|
122
|
-
"-->", "0:00:12"
|
|
123
|
-
]) {
|
|
124
|
-
assert.ok(!spoken.includes(markup), `cue text still carries "${markup}"`);
|
|
125
|
-
}
|
|
126
|
-
assert.ok(spoken.includes("Ты видишь их?"), "cue text keeps the dialogue");
|
|
127
|
-
|
|
128
|
-
assert.deepEqual(TextSubtitleTrack.detectLanguageFromVtt(vtt), { code: "ru", name: "Russian" });
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
test("the fixture is the hard case: the file itself is nearly half Latin", () => {
|
|
132
|
-
// Not an assertion about franc — an assertion that this fixture reproduces the
|
|
133
|
-
// field file's proportions. A fixture whose markup were negligible would pass
|
|
134
|
-
// the check above with or without the fix.
|
|
135
|
-
const cyrillic = (RUSSIAN_ASS.match(/[Ѐ-ӿ]/gu) ?? []).length;
|
|
136
|
-
const latin = (RUSSIAN_ASS.match(/[A-Za-z]/g) ?? []).length;
|
|
137
|
-
assert.ok(cyrillic > 800, `too little dialogue: ${cyrillic} Cyrillic characters`);
|
|
138
|
-
assert.ok(latin > cyrillic * 0.5, `markup too small to reproduce the fault: ${latin} vs ${cyrillic}`);
|
|
139
|
-
});
|
|
140
|
-
|
|
141
|
-
test("cue identifiers, NOTE, STYLE and REGION blocks are not text", () => {
|
|
142
|
-
const vtt = [
|
|
143
|
-
"WEBVTT - This file has cues.",
|
|
144
|
-
"Kind: captions",
|
|
145
|
-
"Language: en",
|
|
146
|
-
"",
|
|
147
|
-
"NOTE",
|
|
148
|
-
"Translated by an English speaking volunteer, all rights reserved.",
|
|
149
|
-
"",
|
|
150
|
-
"STYLE",
|
|
151
|
-
"::cue { background-image: linear-gradient(to bottom, dimgray, lightgray); }",
|
|
152
|
-
"",
|
|
153
|
-
"REGION",
|
|
154
|
-
"id:speaker width:40% lines:3 regionanchor:0%,100%",
|
|
155
|
-
"",
|
|
156
|
-
"opening-line",
|
|
157
|
-
"00:00:12.060 --> 00:00:13.270 align:start position:0%",
|
|
158
|
-
"<v Тоёхиса>Ты видишь их?</v>",
|
|
159
|
-
"",
|
|
160
|
-
"2",
|
|
161
|
-
"00:00:15.400 --> 00:00:16.650",
|
|
162
|
-
"<i>Но не столько вижу,</i>",
|
|
163
|
-
"сколько чувствую & ощущаю.",
|
|
164
|
-
""
|
|
165
|
-
].join("\n");
|
|
166
|
-
|
|
167
|
-
const spoken = TextSubtitleTrack.cueTextOfVtt(vtt);
|
|
168
|
-
assert.equal(
|
|
169
|
-
spoken,
|
|
170
|
-
"Ты видишь их?\nНо не столько вижу,\nсколько чувствую ощущаю."
|
|
171
|
-
);
|
|
172
|
-
});
|
|
173
|
-
|
|
174
|
-
test("a document with no cues yields no language rather than a guess", () => {
|
|
175
|
-
assert.equal(TextSubtitleTrack.cueTextOfVtt("WEBVTT\n"), "");
|
|
176
|
-
assert.equal(TextSubtitleTrack.detectLanguageFromVtt("WEBVTT\n"), null);
|
|
177
|
-
assert.equal(TextSubtitleTrack.detectLanguageFromVtt(null), null);
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
test("an embedded ASS cue is unframed by the container, then read", () => {
|
|
181
|
-
// What a Matroska block actually holds, per
|
|
182
|
-
// `matroska.org/technical/subtitles.html`: "Events are stored in the Block in
|
|
183
|
-
// this order: ReadOrder, Layer, Style, Name, MarginL, MarginR, MarginV,
|
|
184
|
-
// Effect, Text", with Start and End taken out into the block's own timing.
|
|
185
|
-
// Eight fields, and the earlier fixture here wrongly carried ten — it kept
|
|
186
|
-
// the two timestamps a FILE has, which is why it passed while the field
|
|
187
|
-
// subtitles of a real `.mkv` were shown to the viewer whole.
|
|
188
|
-
const blocks = RUSSIAN_DIALOGUE.map((line, index) => Buffer.from(
|
|
189
|
-
`${index + 1},0,Default,,0000,0000,0000,,` +
|
|
190
|
-
`${index % 5 === 4 ? "{\\pos(640,620)}" : ""}${line}`,
|
|
191
|
-
"utf8"
|
|
192
|
-
));
|
|
193
|
-
|
|
194
|
-
const cues = blocks.map((payload, index) => ({
|
|
195
|
-
startSeconds: index * 4,
|
|
196
|
-
endSeconds: index * 4 + 3,
|
|
197
|
-
text: MatroskaContainer.cueTextOf(payload, "S_TEXT/ASS")
|
|
198
|
-
}));
|
|
199
|
-
|
|
200
|
-
const spoken = TextSubtitleTrack.finalizeCues(cues, "S_TEXT/ASS").map((cue) => cue.text).join("\n");
|
|
201
|
-
assert.ok(!spoken.includes("Default"), "the style field survived into the text");
|
|
202
|
-
assert.ok(!spoken.includes("0000"), "a margin field survived into the text");
|
|
203
|
-
assert.ok(!spoken.includes("pos("), "an override group survived into the text");
|
|
204
|
-
assert.deepEqual(detectLanguage(spoken), { code: "ru", name: "Russian" });
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
test("too little text is answered with nothing, not with a guess", () => {
|
|
208
|
-
// Measured 2026-09-02 (`research/franc-boundary-2026-09-02.md`): franc's
|
|
209
|
-
// answer for Russian walks between Bulgarian, Serbian and Russian until about
|
|
210
|
-
// 650 characters. Below that the honest answer is none — the browser then
|
|
211
|
-
// shows "Unknown", and the label moves when enough of the file has arrived.
|
|
212
|
-
const short = RUSSIAN_DIALOGUE.slice(0, 6).join("\n");
|
|
213
|
-
assert.ok(short.length < 300, `the fixture must be short: ${short.length}`);
|
|
214
|
-
assert.equal(detectLanguage(short), null);
|
|
215
|
-
|
|
216
|
-
const long = RUSSIAN_DIALOGUE.join("\n");
|
|
217
|
-
assert.ok(long.length > 650, `the fixture must be long enough: ${long.length}`);
|
|
218
|
-
assert.deepEqual(detectLanguage(long), { code: "ru", name: "Russian" });
|
|
219
|
-
});
|
|
220
|
-
|
|
221
|
-
test("an answer that does not survive losing half the text is not an answer", () => {
|
|
222
|
-
// A file carrying two languages — a bilingual release, or a track that turns
|
|
223
|
-
// into signs-only English partway. franc answers something for the whole; the
|
|
224
|
-
// halves disagree with it, and that disagreement is the text saying the
|
|
225
|
-
// answer rests on where it happened to be cut.
|
|
226
|
-
const english = [
|
|
227
|
-
"Do you see them out there beyond the wall?",
|
|
228
|
-
"I do, though not as clearly as I would like to.",
|
|
229
|
-
"They have been gathering since the morning came.",
|
|
230
|
-
"We should send word to the southern gate at once.",
|
|
231
|
-
"The road was cut yesterday and no rider will pass.",
|
|
232
|
-
"Then we hold what we have and wait for the dawn."
|
|
233
|
-
].join("\n");
|
|
234
|
-
const mixed = `${RUSSIAN_DIALOGUE.slice(0, 13).join("\n")}\n${english}`;
|
|
235
|
-
assert.equal(detectLanguage(mixed), null);
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
test("a WebVTT body is decoded whole, so no character is cut in half", () => {
|
|
239
|
-
// The ffmpeg extraction path used to detect on `String(body.subarray(0, 4096))`.
|
|
240
|
-
// Two faults: a byte cut lands mid-character on any non-Latin track, and most
|
|
241
|
-
// of those bytes are timestamps rather than words. Cyrillic is two bytes per
|
|
242
|
-
// character in UTF-8, so 4096 bytes of this document is a few hundred
|
|
243
|
-
// characters of dialogue — inside the unstable region measured above.
|
|
244
|
-
const cues = RUSSIAN_DIALOGUE.map((line, index) => {
|
|
245
|
-
const start = `00:00:${String(index * 2).padStart(2, "0")}.000`;
|
|
246
|
-
const end = `00:00:${String(index * 2 + 1).padStart(2, "0")}.000`;
|
|
247
|
-
return `${start} --> ${end}\n${line}`;
|
|
248
|
-
});
|
|
249
|
-
const body = Buffer.from(`WEBVTT\n\n${cues.join("\n\n")}\n`, "utf8");
|
|
250
|
-
|
|
251
|
-
assert.deepEqual(TextSubtitleTrack.detectLanguageFromVtt(body.toString("utf8")), { code: "ru", name: "Russian" });
|
|
252
|
-
});
|
|
1
|
+
/**
|
|
2
|
+
* @file What the language detector is FED.
|
|
3
|
+
*
|
|
4
|
+
* Field 2026-09-01: a Russian subtitle file was offered to the viewer as
|
|
5
|
+
* `English (Stan WarHammer & Nesitach)`. The detector was not at fault — it was
|
|
6
|
+
* handed the raw `.ass` file, which is half Latin markup, while the markup-free
|
|
7
|
+
* WebVTT it was about to serve sat in the variable beside it. Measured on that
|
|
8
|
+
* file: 5040 Latin characters against 5983 Cyrillic, `franc(the file) = eng`,
|
|
9
|
+
* `franc(the dialogue) = rus`.
|
|
10
|
+
* `research/subtitle-language-ass-markup-2026-09-01.md`.
|
|
11
|
+
*
|
|
12
|
+
* These checks pin the input at each of the three places a language is read.
|
|
13
|
+
*
|
|
14
|
+
* **On the size of the fixtures.** franc's answer among the Cyrillic languages
|
|
15
|
+
* is not stable on a small sample: measured 2026-09-01 by growing one Russian
|
|
16
|
+
* text line by line, it answered `bul` at 129 characters, `srp` at 158, 241 and
|
|
17
|
+
* `bul` again at 292, then `rus` at every length from 337 to 881. So the
|
|
18
|
+
* dialogue here is ~880 Cyrillic characters — past that boundary by a factor of
|
|
19
|
+
* about 2.6, and still an order of magnitude below a real episode's 5983. The
|
|
20
|
+
* fixtures are NOT sized to make these checks pass; they are sized to resemble
|
|
21
|
+
* the thing. That instability is a separate defect and is recorded as one.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import test from "node:test";
|
|
25
|
+
import assert from "node:assert/strict";
|
|
26
|
+
|
|
27
|
+
import { detectLanguage } from "../services/tracks/language-detect.js";
|
|
28
|
+
import { SubtitleFileContainer } from "../services/container/SubtitleFileContainer.js";
|
|
29
|
+
import { TextSubtitleTrack } from "../services/tracks/TextSubtitleTrack.js";
|
|
30
|
+
import { MatroskaContainer } from "../services/container/MatroskaContainer.js";
|
|
31
|
+
|
|
32
|
+
/** Varied Russian dialogue, the length a few minutes of an episode carries. */
|
|
33
|
+
const RUSSIAN_DIALOGUE = [
|
|
34
|
+
"Ты видишь их?",
|
|
35
|
+
"Да.",
|
|
36
|
+
"Но не столько вижу, сколько чувствую.",
|
|
37
|
+
"Скорее всего, они будут здесь с минуты на минуту.",
|
|
38
|
+
"Почему господа октябристы решили напасть именно сейчас?",
|
|
39
|
+
"Им никогда не преодолеть эти стены.",
|
|
40
|
+
"Мы держали эту крепость три года и продержим ещё столько же.",
|
|
41
|
+
"Готовьте лучников на восточной стороне.",
|
|
42
|
+
"Если они подойдут ближе, мы откроем огонь без предупреждения.",
|
|
43
|
+
"Я не собираюсь умирать здесь, в этой богом забытой дыре.",
|
|
44
|
+
"Тогда возьми меч и вставай рядом со мной.",
|
|
45
|
+
"Сколько у нас осталось воды и хлеба?",
|
|
46
|
+
"На неделю, если не считать раненых.",
|
|
47
|
+
"Раненых считать придётся, они тоже люди.",
|
|
48
|
+
"Отправь гонца к южным воротам и передай приказ отступать.",
|
|
49
|
+
"Он не успеет, дорога перерезана ещё вчера вечером.",
|
|
50
|
+
"Значит пойду сам, а ты останешься за меня командовать.",
|
|
51
|
+
"Это безумие, и ты прекрасно об этом знаешь.",
|
|
52
|
+
"Безумие — это сидеть и ждать, пока нас перебьют по одному.",
|
|
53
|
+
"Хорошо. Но возьми с собой хотя бы двоих.",
|
|
54
|
+
"Двоих я возьму. Больше не могу себе позволить.",
|
|
55
|
+
"Береги себя. Мы будем держать стену до последнего.",
|
|
56
|
+
"Я знаю. Именно поэтому я и ухожу спокойно.",
|
|
57
|
+
"Смотри, дым над лесом. Они уже жгут деревни.",
|
|
58
|
+
"Тогда времени у нас меньше, чем мы думали."
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
/** `hh:mm:ss.cc` for the ASS `Start`/`End` fields. */
|
|
62
|
+
function assTime(seconds) {
|
|
63
|
+
const mm = String(Math.floor(seconds / 60)).padStart(2, "0");
|
|
64
|
+
const ss = String(seconds % 60).padStart(2, "0");
|
|
65
|
+
return `0:${mm}:${ss}.00`;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* An ASS file in the shape a fansub release ships: an Aegisub header, a styles
|
|
70
|
+
* section with English font and style names, and Russian dialogue carrying
|
|
71
|
+
* override groups. Modelled on the field file.
|
|
72
|
+
*/
|
|
73
|
+
const RUSSIAN_ASS = `[Script Info]
|
|
74
|
+
Title: Default Aegisub file
|
|
75
|
+
ScriptType: v4.00+
|
|
76
|
+
PlayResX: 1280
|
|
77
|
+
PlayResY: 720
|
|
78
|
+
Original Translation: Nesitach
|
|
79
|
+
Original Editing: Stan WarHammer
|
|
80
|
+
WrapStyle: 0
|
|
81
|
+
ScaledBorderAndShadow: no
|
|
82
|
+
Video Aspect Ratio: c1.77778
|
|
83
|
+
YCbCr Matrix: TV.601
|
|
84
|
+
Aegisub Video Aspect Ratio: c1.777778
|
|
85
|
+
|
|
86
|
+
[Aegisub Project Garbage]
|
|
87
|
+
Last Style Storage: Default
|
|
88
|
+
Audio File: [HorribleSubs] Drifters - 03 [720p].mkv
|
|
89
|
+
Video File: [HorribleSubs] Drifters - 03 [720p].mkv
|
|
90
|
+
Video AR Mode: 4
|
|
91
|
+
Video AR Value: 1.777778
|
|
92
|
+
Video Zoom Percent: 0.600000
|
|
93
|
+
Scroll Position: 256
|
|
94
|
+
Active Line: 257
|
|
95
|
+
Video Position: 33159
|
|
96
|
+
|
|
97
|
+
[V4+ Styles]
|
|
98
|
+
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
|
|
99
|
+
Style: Default,Trebuchet MS,54,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,2.5,1.5,2,20,20,25,1
|
|
100
|
+
Style: Signs,Times New Roman,48,&H00FFFF00,&H000000FF,&H00202020,&H00000000,-1,0,0,0,100,100,0,0,1,2,1,8,20,20,20,1
|
|
101
|
+
Style: Italics,Trebuchet MS Italic,54,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,-1,0,0,100,100,0,0,1,2.5,1.5,2,20,20,25,1
|
|
102
|
+
|
|
103
|
+
[Events]
|
|
104
|
+
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
|
|
105
|
+
${RUSSIAN_DIALOGUE.map((line, index) => {
|
|
106
|
+
const style = index % 7 === 6 ? "Signs" : "Default";
|
|
107
|
+
const tag = index % 5 === 4 ? "{\\pos(640,620)}" : "";
|
|
108
|
+
return `Dialogue: 0,${assTime(index * 4)},${assTime(index * 4 + 3)},${style},,0,0,0,,${tag}${line}`;
|
|
109
|
+
}).join("\n")}
|
|
110
|
+
`;
|
|
111
|
+
|
|
112
|
+
test("a Russian .ass is reported as Russian, and its markup does not reach the detector", () => {
|
|
113
|
+
const vtt = SubtitleFileContainer.toVtt(RUSSIAN_ASS, ".ass");
|
|
114
|
+
assert.ok(vtt.startsWith("WEBVTT"), "the conversion produced a WebVTT document");
|
|
115
|
+
|
|
116
|
+
const spoken = TextSubtitleTrack.cueTextOfVtt(vtt);
|
|
117
|
+
// Every one of these is a piece of the file the viewer never reads, and each
|
|
118
|
+
// was competing with the dialogue for the detector's answer.
|
|
119
|
+
for (const markup of [
|
|
120
|
+
"Aegisub", "Script Info", "V4+ Styles", "Format:", "Dialogue:",
|
|
121
|
+
"Trebuchet MS", "Times New Roman", "Default", "Signs", "pos(",
|
|
122
|
+
"-->", "0:00:12"
|
|
123
|
+
]) {
|
|
124
|
+
assert.ok(!spoken.includes(markup), `cue text still carries "${markup}"`);
|
|
125
|
+
}
|
|
126
|
+
assert.ok(spoken.includes("Ты видишь их?"), "cue text keeps the dialogue");
|
|
127
|
+
|
|
128
|
+
assert.deepEqual(TextSubtitleTrack.detectLanguageFromVtt(vtt), { code: "ru", name: "Russian" });
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("the fixture is the hard case: the file itself is nearly half Latin", () => {
|
|
132
|
+
// Not an assertion about franc — an assertion that this fixture reproduces the
|
|
133
|
+
// field file's proportions. A fixture whose markup were negligible would pass
|
|
134
|
+
// the check above with or without the fix.
|
|
135
|
+
const cyrillic = (RUSSIAN_ASS.match(/[Ѐ-ӿ]/gu) ?? []).length;
|
|
136
|
+
const latin = (RUSSIAN_ASS.match(/[A-Za-z]/g) ?? []).length;
|
|
137
|
+
assert.ok(cyrillic > 800, `too little dialogue: ${cyrillic} Cyrillic characters`);
|
|
138
|
+
assert.ok(latin > cyrillic * 0.5, `markup too small to reproduce the fault: ${latin} vs ${cyrillic}`);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test("cue identifiers, NOTE, STYLE and REGION blocks are not text", () => {
|
|
142
|
+
const vtt = [
|
|
143
|
+
"WEBVTT - This file has cues.",
|
|
144
|
+
"Kind: captions",
|
|
145
|
+
"Language: en",
|
|
146
|
+
"",
|
|
147
|
+
"NOTE",
|
|
148
|
+
"Translated by an English speaking volunteer, all rights reserved.",
|
|
149
|
+
"",
|
|
150
|
+
"STYLE",
|
|
151
|
+
"::cue { background-image: linear-gradient(to bottom, dimgray, lightgray); }",
|
|
152
|
+
"",
|
|
153
|
+
"REGION",
|
|
154
|
+
"id:speaker width:40% lines:3 regionanchor:0%,100%",
|
|
155
|
+
"",
|
|
156
|
+
"opening-line",
|
|
157
|
+
"00:00:12.060 --> 00:00:13.270 align:start position:0%",
|
|
158
|
+
"<v Тоёхиса>Ты видишь их?</v>",
|
|
159
|
+
"",
|
|
160
|
+
"2",
|
|
161
|
+
"00:00:15.400 --> 00:00:16.650",
|
|
162
|
+
"<i>Но не столько вижу,</i>",
|
|
163
|
+
"сколько чувствую & ощущаю.",
|
|
164
|
+
""
|
|
165
|
+
].join("\n");
|
|
166
|
+
|
|
167
|
+
const spoken = TextSubtitleTrack.cueTextOfVtt(vtt);
|
|
168
|
+
assert.equal(
|
|
169
|
+
spoken,
|
|
170
|
+
"Ты видишь их?\nНо не столько вижу,\nсколько чувствую ощущаю."
|
|
171
|
+
);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test("a document with no cues yields no language rather than a guess", () => {
|
|
175
|
+
assert.equal(TextSubtitleTrack.cueTextOfVtt("WEBVTT\n"), "");
|
|
176
|
+
assert.equal(TextSubtitleTrack.detectLanguageFromVtt("WEBVTT\n"), null);
|
|
177
|
+
assert.equal(TextSubtitleTrack.detectLanguageFromVtt(null), null);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("an embedded ASS cue is unframed by the container, then read", () => {
|
|
181
|
+
// What a Matroska block actually holds, per
|
|
182
|
+
// `matroska.org/technical/subtitles.html`: "Events are stored in the Block in
|
|
183
|
+
// this order: ReadOrder, Layer, Style, Name, MarginL, MarginR, MarginV,
|
|
184
|
+
// Effect, Text", with Start and End taken out into the block's own timing.
|
|
185
|
+
// Eight fields, and the earlier fixture here wrongly carried ten — it kept
|
|
186
|
+
// the two timestamps a FILE has, which is why it passed while the field
|
|
187
|
+
// subtitles of a real `.mkv` were shown to the viewer whole.
|
|
188
|
+
const blocks = RUSSIAN_DIALOGUE.map((line, index) => Buffer.from(
|
|
189
|
+
`${index + 1},0,Default,,0000,0000,0000,,` +
|
|
190
|
+
`${index % 5 === 4 ? "{\\pos(640,620)}" : ""}${line}`,
|
|
191
|
+
"utf8"
|
|
192
|
+
));
|
|
193
|
+
|
|
194
|
+
const cues = blocks.map((payload, index) => ({
|
|
195
|
+
startSeconds: index * 4,
|
|
196
|
+
endSeconds: index * 4 + 3,
|
|
197
|
+
text: MatroskaContainer.cueTextOf(payload, "S_TEXT/ASS")
|
|
198
|
+
}));
|
|
199
|
+
|
|
200
|
+
const spoken = TextSubtitleTrack.finalizeCues(cues, "S_TEXT/ASS").map((cue) => cue.text).join("\n");
|
|
201
|
+
assert.ok(!spoken.includes("Default"), "the style field survived into the text");
|
|
202
|
+
assert.ok(!spoken.includes("0000"), "a margin field survived into the text");
|
|
203
|
+
assert.ok(!spoken.includes("pos("), "an override group survived into the text");
|
|
204
|
+
assert.deepEqual(detectLanguage(spoken), { code: "ru", name: "Russian" });
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
test("too little text is answered with nothing, not with a guess", () => {
|
|
208
|
+
// Measured 2026-09-02 (`research/franc-boundary-2026-09-02.md`): franc's
|
|
209
|
+
// answer for Russian walks between Bulgarian, Serbian and Russian until about
|
|
210
|
+
// 650 characters. Below that the honest answer is none — the browser then
|
|
211
|
+
// shows "Unknown", and the label moves when enough of the file has arrived.
|
|
212
|
+
const short = RUSSIAN_DIALOGUE.slice(0, 6).join("\n");
|
|
213
|
+
assert.ok(short.length < 300, `the fixture must be short: ${short.length}`);
|
|
214
|
+
assert.equal(detectLanguage(short), null);
|
|
215
|
+
|
|
216
|
+
const long = RUSSIAN_DIALOGUE.join("\n");
|
|
217
|
+
assert.ok(long.length > 650, `the fixture must be long enough: ${long.length}`);
|
|
218
|
+
assert.deepEqual(detectLanguage(long), { code: "ru", name: "Russian" });
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("an answer that does not survive losing half the text is not an answer", () => {
|
|
222
|
+
// A file carrying two languages — a bilingual release, or a track that turns
|
|
223
|
+
// into signs-only English partway. franc answers something for the whole; the
|
|
224
|
+
// halves disagree with it, and that disagreement is the text saying the
|
|
225
|
+
// answer rests on where it happened to be cut.
|
|
226
|
+
const english = [
|
|
227
|
+
"Do you see them out there beyond the wall?",
|
|
228
|
+
"I do, though not as clearly as I would like to.",
|
|
229
|
+
"They have been gathering since the morning came.",
|
|
230
|
+
"We should send word to the southern gate at once.",
|
|
231
|
+
"The road was cut yesterday and no rider will pass.",
|
|
232
|
+
"Then we hold what we have and wait for the dawn."
|
|
233
|
+
].join("\n");
|
|
234
|
+
const mixed = `${RUSSIAN_DIALOGUE.slice(0, 13).join("\n")}\n${english}`;
|
|
235
|
+
assert.equal(detectLanguage(mixed), null);
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
test("a WebVTT body is decoded whole, so no character is cut in half", () => {
|
|
239
|
+
// The ffmpeg extraction path used to detect on `String(body.subarray(0, 4096))`.
|
|
240
|
+
// Two faults: a byte cut lands mid-character on any non-Latin track, and most
|
|
241
|
+
// of those bytes are timestamps rather than words. Cyrillic is two bytes per
|
|
242
|
+
// character in UTF-8, so 4096 bytes of this document is a few hundred
|
|
243
|
+
// characters of dialogue — inside the unstable region measured above.
|
|
244
|
+
const cues = RUSSIAN_DIALOGUE.map((line, index) => {
|
|
245
|
+
const start = `00:00:${String(index * 2).padStart(2, "0")}.000`;
|
|
246
|
+
const end = `00:00:${String(index * 2 + 1).padStart(2, "0")}.000`;
|
|
247
|
+
return `${start} --> ${end}\n${line}`;
|
|
248
|
+
});
|
|
249
|
+
const body = Buffer.from(`WEBVTT\n\n${cues.join("\n\n")}\n`, "utf8");
|
|
250
|
+
|
|
251
|
+
assert.deepEqual(TextSubtitleTrack.detectLanguageFromVtt(body.toString("utf8")), { code: "ru", name: "Russian" });
|
|
252
|
+
});
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Where a file is cut is a fact about the FILE, held once.
|
|
3
|
+
*
|
|
4
|
+
* Every quality step of one film has to be cut at exactly the same times, and
|
|
5
|
+
* every session serving it has to publish the same playlist. That agreement was
|
|
6
|
+
* arranged by COPYING — a table handed to each new session at creation — and it
|
|
7
|
+
* drifted twice in the field: 0.6-2.9 s between two sessions of one film on
|
|
8
|
+
* 2026-08-17, and segments arriving a uniform 2.002 s before the times the
|
|
9
|
+
* playlist named for them on 2026-08-20, four times what a player bridges.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import test from "node:test";
|
|
13
|
+
import assert from "node:assert/strict";
|
|
14
|
+
import { Timeline, Timelines } from "../services/output/Timeline.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* @returns {Timeline}
|
|
18
|
+
*/
|
|
19
|
+
function fourSecondGrid() {
|
|
20
|
+
// Only what is per grid. The container's own keyframe table, how exact it is,
|
|
21
|
+
// which container answered and how long the file runs are facts of the FILE
|
|
22
|
+
// and live on the source file — stating them here stated nothing.
|
|
23
|
+
return new Timeline({
|
|
24
|
+
boundaries: [0, 4, 8, 12, 16],
|
|
25
|
+
cutGrid: "keyframe"
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
test("the table the player was given does not move when the live one is corrected", () => {
|
|
30
|
+
const timeline = fourSecondGrid();
|
|
31
|
+
|
|
32
|
+
// A produced segment says where the file's cut really is. That corrects what
|
|
33
|
+
// a run will cut at; it must never correct what the player was told, because
|
|
34
|
+
// the player places a fragment by the playlist it holds and that text was
|
|
35
|
+
// written once.
|
|
36
|
+
timeline.boundaries[2] = 8.5;
|
|
37
|
+
|
|
38
|
+
assert.equal(timeline.liveStartOf(2), 8.5);
|
|
39
|
+
assert.equal(timeline.publishedStartOf(2), 8);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("one file and grid answer with one table, whoever asks", () => {
|
|
43
|
+
const timelines = new Timelines();
|
|
44
|
+
const key = Timelines.keyFor("torrent:abc", 0, "keyframe");
|
|
45
|
+
|
|
46
|
+
const first = timelines.get(key, fourSecondGrid);
|
|
47
|
+
const second = timelines.get(key, () => {
|
|
48
|
+
throw new Error("a second table would be the drift this exists to remove");
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
assert.equal(second, first);
|
|
52
|
+
// A correction found by one session is seen by every other, because there is
|
|
53
|
+
// nothing to keep in step.
|
|
54
|
+
first.boundaries[1] = 4.5;
|
|
55
|
+
assert.equal(second.liveStartOf(1), 4.5);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("the same file cut two ways is two tables", () => {
|
|
59
|
+
const timelines = new Timelines();
|
|
60
|
+
const onKeyframes = timelines.get(Timelines.keyFor("torrent:abc", 0, "keyframe"), fourSecondGrid);
|
|
61
|
+
const onTheEvenGrid = timelines.get(Timelines.keyFor("torrent:abc", 0, "uniform"), fourSecondGrid);
|
|
62
|
+
|
|
63
|
+
assert.notEqual(onTheEvenGrid, onKeyframes);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("which segment holds a moment", () => {
|
|
67
|
+
const timeline = fourSecondGrid();
|
|
68
|
+
|
|
69
|
+
assert.equal(timeline.indexForTime(0), 0);
|
|
70
|
+
assert.equal(timeline.indexForTime(3.9), 0);
|
|
71
|
+
assert.equal(timeline.indexForTime(4), 1);
|
|
72
|
+
assert.equal(timeline.indexForTime(15.9), 3);
|
|
73
|
+
assert.equal(timeline.indexForTime(99), 3, "past the end is the last segment, not an error");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test("a fresh table starts with an empty tally of how well its index matched", () => {
|
|
77
|
+
const timeline = fourSecondGrid();
|
|
78
|
+
|
|
79
|
+
// The tally is a fact about the FILE and its index: asked per session it
|
|
80
|
+
// would be answered a different number of times for one film depending on
|
|
81
|
+
// how many people happened to watch it.
|
|
82
|
+
assert.equal(timeline.indexCheck.checked, 0);
|
|
83
|
+
assert.equal(timeline.indexCheck.disagreed, 0);
|
|
84
|
+
assert.equal(timeline.indexCheck.firstDisagreementIndex, -1);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("a timeline nobody holds is dropped", () => {
|
|
88
|
+
const timelines = new Timelines();
|
|
89
|
+
const kept = timelines.get(Timelines.keyFor("torrent:abc", 0, "keyframe"), fourSecondGrid);
|
|
90
|
+
timelines.get(Timelines.keyFor("torrent:xyz", 0, "keyframe"), fourSecondGrid);
|
|
91
|
+
|
|
92
|
+
assert.equal(timelines.size, 2);
|
|
93
|
+
assert.equal(timelines.forgetUnused(new Set([kept])), 1);
|
|
94
|
+
assert.equal(timelines.size, 1);
|
|
95
|
+
});
|