@torrent-tv/proxy 2.80.18 → 2.81.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/docs/encode-architecture.md +51 -2
  3. package/package.json +1 -1
  4. package/research/double-spawn-2026-09-10.md +171 -0
  5. package/services/disk/DiskSpace.js +146 -0
  6. package/services/disk/wire.js +60 -0
  7. package/services/encode/EncodeRun.js +37 -9
  8. package/services/encode/SegmentStore.js +284 -232
  9. package/services/encode/run-command.js +16 -1
  10. package/services/hls-session-manager.js +31 -128
  11. package/services/orchestrators/EncodeOrchestrator.js +52 -38
  12. package/services/piece-store/allowance.js +107 -0
  13. package/services/piece-store/piece-disk-store.js +365 -0
  14. package/services/piece-store/shared-piece-store.js +1549 -1535
  15. package/services/segment-formats/fmp4.js +54 -0
  16. package/services/segment-formats/mpegts.js +54 -0
  17. package/services/torrent-worker/client.js +32 -0
  18. package/services/torrent-worker/pool-adapter.js +15 -0
  19. package/services/torrent-worker/protocol.js +9 -0
  20. package/services/torrent-worker/worker.js +8 -1
  21. package/services/viewer/positions.js +48 -0
  22. package/test/audio-inventory.test.js +176 -176
  23. package/test/auto-quality-step.test.js +514 -514
  24. package/test/concurrent-cost.test.js +138 -138
  25. package/test/coverage-follows-the-disk.test.js +191 -187
  26. package/test/coverage-map.test.js +195 -195
  27. package/test/declared-tracks.test.js +35 -35
  28. package/test/disk-space.test.js +138 -0
  29. package/test/encode-orchestrator.test.js +0 -3
  30. package/test/encode-run.test.js +5 -12
  31. package/test/held-request-width.test.js +155 -155
  32. package/test/helpers/encode-run.js +2 -2
  33. package/test/matroska-blocks.test.js +0 -0
  34. package/test/matroska-cues-track.test.js +192 -192
  35. package/test/mp4-composition-times.test.js +0 -0
  36. package/test/mp4-subtitles.test.js +173 -173
  37. package/test/one-authority.test.js +281 -220
  38. package/test/orchestrator-wired.test.js +199 -195
  39. package/test/packet-witness-ring.test.js +236 -236
  40. package/test/packet-witness.test.js +148 -148
  41. package/test/piece-disk-store.test.js +267 -0
  42. package/test/piece-reader.test.js +4 -4
  43. package/test/piece-store-eviction.test.js +17 -17
  44. package/test/piece-store-reservations.test.js +20 -1
  45. package/test/piece-store-slow-disk.test.js +16 -1
  46. package/test/produced-copy-choice.test.js +258 -358
  47. package/test/read-window.test.js +6 -6
  48. package/test/run-intervals.test.js +100 -100
  49. package/test/seek-landing.test.js +109 -109
  50. package/test/segment-serve-wiring.test.js +8 -9
  51. package/test/segment-store-eviction.test.js +232 -0
  52. package/test/segment-store.test.js +238 -216
  53. package/test/segments-are-shared.test.js +1 -1
  54. package/test/shared-piece-store.test.js +12 -12
  55. package/test/sidecar-naming.test.js +142 -142
  56. package/test/subtitle-cue-framing.test.js +200 -200
  57. package/test/subtitle-cue-walk.test.js +369 -369
  58. package/test/subtitle-defaults.test.js +97 -97
  59. package/test/subtitle-track-numbering.test.js +370 -370
  60. package/test/tail-duplication.test.js +167 -167
  61. package/test/tracks-begin-together.test.js +195 -195
  62. package/test/two-viewers-one-picture.test.js +374 -374
  63. package/test/video-facts.test.js +102 -102
  64. package/test/wedge-certainty.test.js +131 -131
  65. package/services/encode/open-piece.js +0 -135
  66. package/services/piece-store/disk-tier.js +0 -151
  67. package/test/open-piece.test.js +0 -152
@@ -1,370 +1,370 @@
1
- /**
2
- * @file The number a subtitle track is asked for by, and one walk per file.
3
- *
4
- * Two rules, both found by reading on 2026-08-26 after the report that embedded
5
- * subtitles appear late (`research/subtitle-delay-2026-08-26.md`):
6
- *
7
- * 1. The browser names a track by ffmpeg's `0:s:N`, which counts EVERY subtitle
8
- * stream. The container plan drops the picture-based ones — PGS, VobSub —
9
- * because they cannot become WebVTT, so counting the kept ones is a
10
- * different numbering as soon as a file carries one of each. What that cost:
11
- * the push landed on a track the browser does not know, and the browser's
12
- * own request found no track at all and fell through to the ffmpeg
13
- * extraction, which reads the whole film (752 s measured, 2026-08-19).
14
- * 2. A file is walked once at a time. The walk marks a cluster as read only
15
- * after two suspension points, and it is started both on every verified
16
- * piece and on a 3 s timer, so two passes could read and parse the same
17
- * cluster and push the same line twice.
18
- */
19
-
20
- import test from "node:test";
21
- import assert from "node:assert/strict";
22
- import { Readable } from "node:stream";
23
- import { MatroskaContainer } from "../services/container/MatroskaContainer.js";
24
- import { cuesHeldFor, forgetSubtitles } from "../services/torrent-worker/subtitle-cues.js";
25
-
26
- const ID_EBML = 0x1a45dfa3;
27
- const ID_SEGMENT = 0x18538067;
28
- const ID_SEEK_HEAD = 0x114d9b74;
29
- const ID_SEEK = 0x4dbb;
30
- const ID_SEEK_ID = 0x53ab;
31
- const ID_SEEK_POSITION = 0x53ac;
32
- const ID_INFO = 0x1549a966;
33
- const ID_TIMESTAMP_SCALE = 0x2ad7b1;
34
- const ID_TRACKS = 0x1654ae6b;
35
- const ID_TRACK_ENTRY = 0xae;
36
- const ID_TRACK_NUMBER = 0xd7;
37
- const ID_TRACK_TYPE = 0x83;
38
- const ID_CODEC_ID = 0x86;
39
- const ID_LANGUAGE = 0x22b59c;
40
- const ID_LANGUAGE_BCP47 = 0x22b59d;
41
- const ID_FLAG_ENABLED = 0xb9;
42
- const ID_FLAG_FORCED = 0x55aa;
43
- const ID_FLAG_HEARING_IMPAIRED = 0x55ab;
44
- const ID_CUES = 0x1c53bb6b;
45
- const ID_CUE_POINT = 0xbb;
46
- const ID_CUE_TIME = 0xb3;
47
- const ID_CUE_TRACK_POSITIONS = 0xb7;
48
- const ID_CUE_TRACK = 0xf7;
49
- const ID_CUE_CLUSTER_POSITION = 0xf1;
50
- const ID_CLUSTER = 0x1f43b675;
51
- const ID_TIMESTAMP = 0xe7;
52
-
53
- /** An element id, as the bytes the specification gives it. */
54
- function idBytes(id) {
55
- const bytes = [];
56
- let rest = id;
57
- while (rest > 0) {
58
- bytes.unshift(rest & 0xff);
59
- rest = Math.floor(rest / 256);
60
- }
61
- return Buffer.from(bytes);
62
- }
63
-
64
- /** A size, as a four-byte EBML variable-length integer. */
65
- function sizeBytes(size) {
66
- const buffer = Buffer.alloc(4);
67
- buffer.writeUInt32BE(size, 0);
68
- buffer[0] |= 0x10;
69
- return buffer;
70
- }
71
-
72
- function element(id, payload) {
73
- return Buffer.concat([idBytes(id), sizeBytes(payload.length), payload]);
74
- }
75
-
76
- function uintElement(id, value) {
77
- const bytes = [];
78
- let rest = value;
79
- do {
80
- bytes.unshift(rest & 0xff);
81
- rest = Math.floor(rest / 256);
82
- } while (rest > 0);
83
- return element(id, Buffer.from(bytes));
84
- }
85
-
86
- function stringElement(id, value) {
87
- return element(id, Buffer.from(value, "utf8"));
88
- }
89
-
90
- /**
91
- * An unsigned value at a FIXED four bytes. A cue's cluster position has to be
92
- * written twice — once to measure the table, once with the position that
93
- * measurement produced — and a value-sized element would make the second table
94
- * a different length from the first, moving the very cluster it names.
95
- */
96
- function uint32Element(id, value) {
97
- const payload = Buffer.alloc(4);
98
- payload.writeUInt32BE(value, 0);
99
- return element(id, payload);
100
- }
101
-
102
- function trackEntry({ number, type, codecId, language, flags = {}, languageBcp47 = null }) {
103
- const parts = [
104
- uintElement(ID_TRACK_NUMBER, number),
105
- uintElement(ID_TRACK_TYPE, type),
106
- stringElement(ID_CODEC_ID, codecId),
107
- stringElement(ID_LANGUAGE, language)
108
- ];
109
- if (languageBcp47 !== null) {
110
- parts.push(stringElement(ID_LANGUAGE_BCP47, languageBcp47));
111
- }
112
- for (const [id, value] of [
113
- [ID_FLAG_ENABLED, flags.enabled],
114
- [ID_FLAG_FORCED, flags.forced],
115
- [ID_FLAG_HEARING_IMPAIRED, flags.hearingImpaired]
116
- ]) {
117
- if (value !== undefined) {
118
- parts.push(uintElement(id, value ? 1 : 0));
119
- }
120
- }
121
- return element(ID_TRACK_ENTRY, Buffer.concat(parts));
122
- }
123
-
124
- /**
125
- * A file whose subtitle tracks are, in the container's own order: a picture
126
- * one, then two text ones. ffmpeg numbers those `0:s:0`, `0:s:1`, `0:s:2`;
127
- * the plan can only read the last two.
128
- *
129
- * The Cues table points both text tracks at one cluster, which is written after
130
- * the table so its position can be stated.
131
- *
132
- * @returns {{ file: Buffer, clusterAt: number }}
133
- */
134
- function buildFile() {
135
- const info = element(ID_INFO, uintElement(ID_TIMESTAMP_SCALE, 1_000_000));
136
- const tracks = element(ID_TRACKS, Buffer.concat([
137
- trackEntry({ number: 1, type: 1, codecId: "V_MPEG4/ISO/AVC", language: "und" }),
138
- trackEntry({ number: 2, type: 17, codecId: "S_HDMV/PGS", language: "eng" }),
139
- trackEntry({ number: 3, type: 17, codecId: "S_TEXT/UTF8", language: "rus" }),
140
- trackEntry({ number: 4, type: 17, codecId: "S_TEXT/ASS", language: "eng" })
141
- ]));
142
-
143
- // Built twice: the cue points state where the cluster is, and that position
144
- // is only known once everything before it has its final length. Every size
145
- // and position here is written at a fixed width, so the draft and the final
146
- // table are the same length.
147
- const cuesWith = (clusterAt) => element(ID_CUES, Buffer.concat([
148
- element(ID_CUE_POINT, Buffer.concat([
149
- uintElement(ID_CUE_TIME, 1000),
150
- element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
151
- uintElement(ID_CUE_TRACK, 3),
152
- uint32Element(ID_CUE_CLUSTER_POSITION, clusterAt)
153
- ])),
154
- element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
155
- uintElement(ID_CUE_TRACK, 4),
156
- uint32Element(ID_CUE_CLUSTER_POSITION, clusterAt)
157
- ]))
158
- ]))
159
- ]));
160
-
161
- const seekEntry = (targetId, position) => element(ID_SEEK, Buffer.concat([
162
- element(ID_SEEK_ID, idBytes(targetId)),
163
- element(ID_SEEK_POSITION, (() => {
164
- const buffer = Buffer.alloc(4);
165
- buffer.writeUInt32BE(position, 0);
166
- return buffer;
167
- })())
168
- ]));
169
- const seekHeadWith = (infoAt, tracksAt, cuesAt) => element(ID_SEEK_HEAD, Buffer.concat([
170
- seekEntry(ID_INFO, infoAt),
171
- seekEntry(ID_TRACKS, tracksAt),
172
- seekEntry(ID_CUES, cuesAt)
173
- ]));
174
-
175
- const headLength = seekHeadWith(0, 0, 0).length;
176
- const infoAt = headLength;
177
- const tracksAt = infoAt + info.length;
178
- const cuesAt = tracksAt + tracks.length;
179
- // A position in the Cues table is measured from the Segment's payload, and so
180
- // is the one the reader turns it into.
181
- const clusterRelative = cuesAt + cuesWith(0).length;
182
-
183
- // Enough of a cluster to be read and recognised: its own header and a
184
- // timestamp. No blocks, so it yields no cues — what the walk test counts is
185
- // that its bytes are fetched once, and that does not depend on their content.
186
- const cluster = element(ID_CLUSTER, uintElement(ID_TIMESTAMP, 1000));
187
-
188
- const segmentPayload = Buffer.concat([
189
- seekHeadWith(infoAt, tracksAt, cuesAt),
190
- info,
191
- tracks,
192
- cuesWith(clusterRelative),
193
- cluster
194
- ]);
195
- const ebml = element(ID_EBML, Buffer.from([0x42, 0x86, 0x81, 0x01]));
196
- const segment = element(ID_SEGMENT, segmentPayload);
197
- const segmentDataOffset = ebml.length + segment.length - segmentPayload.length;
198
- return {
199
- file: Buffer.concat([ebml, segment]),
200
- clusterAt: segmentDataOffset + clusterRelative
201
- };
202
- }
203
-
204
- function readerOver(file) {
205
- return async (start, end) => {
206
- const last = Math.min(end, file.length - 1);
207
- return start > last ? null : file.subarray(start, last + 1);
208
- };
209
- }
210
-
211
- test("a text track is numbered as ffmpeg numbers it, past the picture ones", async () => {
212
- const { file } = buildFile();
213
-
214
- const plan = await MatroskaContainer.readSubtitlePlan(readerOver(file), file.length);
215
-
216
- assert.equal(plan.declared.length, 3, "all three subtitle tracks are declared");
217
- assert.deepEqual(plan.tracks.map((track) => track.trackNumber), [3, 4], "only the text ones are readable");
218
- assert.deepEqual(
219
- plan.tracks.map((track) => track.declaredIndex),
220
- [1, 2],
221
- "the PGS track is 0:s:0, so the text tracks are 0:s:1 and 0:s:2 — not 0 and 1"
222
- );
223
- });
224
-
225
- /**
226
- * A torrent holding one file entirely, counting the byte ranges read from it.
227
- *
228
- * @param {Buffer} bytes
229
- * @returns {{ torrent: object, reads: Array<{ start: number, end: number }> }}
230
- */
231
- function torrentOver(bytes) {
232
- const reads = [];
233
- const file = {
234
- name: "film.mkv",
235
- length: bytes.length,
236
- offset: 0,
237
- createReadStream({ start = 0, end = bytes.length - 1 } = {}) {
238
- reads.push({ start, end });
239
- // Asynchronous on purpose: a read that resolves in the same tick would
240
- // hide exactly the interleaving this test is about.
241
- return Readable.from((async function* chunks() {
242
- await new Promise((resolve) => setImmediate(resolve));
243
- yield bytes.subarray(start, end + 1);
244
- })());
245
- }
246
- };
247
- return {
248
- reads,
249
- torrent: {
250
- pieceLength: 1024,
251
- bitfield: { get: () => true },
252
- files: [file]
253
- }
254
- };
255
- }
256
-
257
- test("two walks of one file at the same time read each cluster once", async () => {
258
- const { file, clusterAt } = buildFile();
259
- const { torrent, reads } = torrentOver(file);
260
- const sourceKey = "torrent:numbering-test";
261
- forgetSubtitles(sourceKey);
262
-
263
- // Both text tracks at once, which is what the warmup does on every verified
264
- // piece and every three seconds.
265
- const [first, second] = await Promise.all([
266
- cuesHeldFor(torrent, 0, sourceKey, 3),
267
- cuesHeldFor(torrent, 0, sourceKey, 4)
268
- ]);
269
-
270
- assert.equal(first.coveredClusters, 1, "the cluster the table names was walked");
271
- assert.equal(second.coveredClusters, 1, "and the second track sees the same walk, not its own");
272
- const clusterReads = reads.filter((range) => range.start === clusterAt);
273
- assert.equal(
274
- clusterReads.length,
275
- 2,
276
- "one probe of the cluster's header and one read of its body — not two of each"
277
- );
278
- forgetSubtitles(sourceKey);
279
- });
280
-
281
- /**
282
- * A file whose subtitle tracks carry the flags RFC 9559 defines for them: one
283
- * forced, one for viewers who cannot hear, one the file marks unusable, and one
284
- * writing its language as RFC 5646 alongside the three-letter code.
285
- *
286
- * @returns {Buffer}
287
- */
288
- function fileWithFlags() {
289
- const info = element(ID_INFO, uintElement(ID_TIMESTAMP_SCALE, 1_000_000));
290
- const tracks = element(ID_TRACKS, Buffer.concat([
291
- trackEntry({ number: 1, type: 1, codecId: "V_MPEG4/ISO/AVC", language: "und" }),
292
- trackEntry({ number: 2, type: 17, codecId: "S_TEXT/UTF8", language: "rus", flags: { forced: true } }),
293
- trackEntry({ number: 3, type: 17, codecId: "S_TEXT/UTF8", language: "eng", flags: { hearingImpaired: true } }),
294
- trackEntry({ number: 4, type: 17, codecId: "S_TEXT/UTF8", language: "fre", flags: { enabled: false } }),
295
- trackEntry({ number: 5, type: 17, codecId: "S_TEXT/ASS", language: "por", languageBcp47: "pt-BR" })
296
- ]));
297
- const seekEntry = (targetId, position) => element(ID_SEEK, Buffer.concat([
298
- element(ID_SEEK_ID, idBytes(targetId)),
299
- uint32Element(ID_SEEK_POSITION, position)
300
- ]));
301
- const seekHeadWith = (infoAt, tracksAt) => element(ID_SEEK_HEAD, Buffer.concat([
302
- seekEntry(ID_INFO, infoAt),
303
- seekEntry(ID_TRACKS, tracksAt)
304
- ]));
305
- const headLength = seekHeadWith(0, 0).length;
306
- const segmentPayload = Buffer.concat([
307
- seekHeadWith(headLength, headLength + info.length),
308
- info,
309
- tracks
310
- ]);
311
- const ebml = element(ID_EBML, Buffer.from([0x42, 0x86, 0x81, 0x01]));
312
- return Buffer.concat([ebml, element(ID_SEGMENT, segmentPayload)]);
313
- }
314
-
315
- test("the flags the file states about a track are read, not guessed from its name", async () => {
316
- const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
317
-
318
- const forced = plan.tracks.find((track) => track.trackNumber === 2);
319
- assert.equal(forced.isForced, true, "FlagForced 0x55AA");
320
- assert.equal(forced.isHearingImpaired, false);
321
-
322
- const sdh = plan.tracks.find((track) => track.trackNumber === 3);
323
- assert.equal(sdh.isHearingImpaired, true, "FlagHearingImpaired 0x55AB");
324
- assert.equal(sdh.isForced, false);
325
- });
326
-
327
- test("a track the file marks unusable is not offered, but is still counted", async () => {
328
- // FlagEnabled (0xB9): "Set to 1 if the track is usable." Track 4 says 0, so
329
- // it is not offered — but it KEEPS its place in the numbering, because ffmpeg
330
- // keeps it: `matroskadec.c` parses MATROSKA_ID_TRACKFLAGENABLED as EBML_NONE,
331
- // reading the element and storing nothing, so the stream is created and gets
332
- // its own `0:s:N`. Dropping it here would shift every track after it.
333
- const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
334
-
335
- assert.equal(plan.tracks.some((track) => track.trackNumber === 4), false, "not offered for extraction");
336
- const counted = plan.declared.find((track) => track.trackNumber === 4);
337
- assert.ok(counted, "still declared, so the numbering does not move");
338
- assert.equal(counted.isEnabled, false);
339
- });
340
-
341
- test("an unusable track keeps its place, so the tracks after it keep theirs", async () => {
342
- const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
343
-
344
- // s:0 forced, s:1 SDH, s:2 the unusable one, s:3 the Brazilian track.
345
- assert.deepEqual(
346
- plan.tracks.map((track) => [track.trackNumber, track.declaredIndex]),
347
- [[2, 0], [3, 1], [5, 3]]
348
- );
349
- });
350
-
351
- test("the list ffmpeg is lined up against still speaks ffmpeg's language codes", async () => {
352
- // `declared` exists to be paired with the `-i` banner, which prints the
353
- // three-letter code; reporting "pt-BR" there would break the pairing and cost
354
- // the FlagDefault reading with it.
355
- const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
356
-
357
- const declared = plan.declared.find((track) => track.trackNumber === 5);
358
- assert.equal(declared.language, "por");
359
- assert.equal(declared.languageBcp47, "pt-BR");
360
- });
361
-
362
- test("where the file writes RFC 5646, that is the language", async () => {
363
- // "If this element is used, then any Language elements used in the same
364
- // TrackEntry MUST be ignored."
365
- const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
366
-
367
- const track = plan.tracks.find((entry) => entry.trackNumber === 5);
368
- assert.equal(track.language, "pt-BR", "not the three-letter por");
369
- assert.equal(track.languageBcp47, "pt-BR");
370
- });
1
+ /**
2
+ * @file The number a subtitle track is asked for by, and one walk per file.
3
+ *
4
+ * Two rules, both found by reading on 2026-08-26 after the report that embedded
5
+ * subtitles appear late (`research/subtitle-delay-2026-08-26.md`):
6
+ *
7
+ * 1. The browser names a track by ffmpeg's `0:s:N`, which counts EVERY subtitle
8
+ * stream. The container plan drops the picture-based ones — PGS, VobSub —
9
+ * because they cannot become WebVTT, so counting the kept ones is a
10
+ * different numbering as soon as a file carries one of each. What that cost:
11
+ * the push landed on a track the browser does not know, and the browser's
12
+ * own request found no track at all and fell through to the ffmpeg
13
+ * extraction, which reads the whole film (752 s measured, 2026-08-19).
14
+ * 2. A file is walked once at a time. The walk marks a cluster as read only
15
+ * after two suspension points, and it is started both on every verified
16
+ * piece and on a 3 s timer, so two passes could read and parse the same
17
+ * cluster and push the same line twice.
18
+ */
19
+
20
+ import test from "node:test";
21
+ import assert from "node:assert/strict";
22
+ import { Readable } from "node:stream";
23
+ import { MatroskaContainer } from "../services/container/MatroskaContainer.js";
24
+ import { cuesHeldFor, forgetSubtitles } from "../services/torrent-worker/subtitle-cues.js";
25
+
26
+ const ID_EBML = 0x1a45dfa3;
27
+ const ID_SEGMENT = 0x18538067;
28
+ const ID_SEEK_HEAD = 0x114d9b74;
29
+ const ID_SEEK = 0x4dbb;
30
+ const ID_SEEK_ID = 0x53ab;
31
+ const ID_SEEK_POSITION = 0x53ac;
32
+ const ID_INFO = 0x1549a966;
33
+ const ID_TIMESTAMP_SCALE = 0x2ad7b1;
34
+ const ID_TRACKS = 0x1654ae6b;
35
+ const ID_TRACK_ENTRY = 0xae;
36
+ const ID_TRACK_NUMBER = 0xd7;
37
+ const ID_TRACK_TYPE = 0x83;
38
+ const ID_CODEC_ID = 0x86;
39
+ const ID_LANGUAGE = 0x22b59c;
40
+ const ID_LANGUAGE_BCP47 = 0x22b59d;
41
+ const ID_FLAG_ENABLED = 0xb9;
42
+ const ID_FLAG_FORCED = 0x55aa;
43
+ const ID_FLAG_HEARING_IMPAIRED = 0x55ab;
44
+ const ID_CUES = 0x1c53bb6b;
45
+ const ID_CUE_POINT = 0xbb;
46
+ const ID_CUE_TIME = 0xb3;
47
+ const ID_CUE_TRACK_POSITIONS = 0xb7;
48
+ const ID_CUE_TRACK = 0xf7;
49
+ const ID_CUE_CLUSTER_POSITION = 0xf1;
50
+ const ID_CLUSTER = 0x1f43b675;
51
+ const ID_TIMESTAMP = 0xe7;
52
+
53
+ /** An element id, as the bytes the specification gives it. */
54
+ function idBytes(id) {
55
+ const bytes = [];
56
+ let rest = id;
57
+ while (rest > 0) {
58
+ bytes.unshift(rest & 0xff);
59
+ rest = Math.floor(rest / 256);
60
+ }
61
+ return Buffer.from(bytes);
62
+ }
63
+
64
+ /** A size, as a four-byte EBML variable-length integer. */
65
+ function sizeBytes(size) {
66
+ const buffer = Buffer.alloc(4);
67
+ buffer.writeUInt32BE(size, 0);
68
+ buffer[0] |= 0x10;
69
+ return buffer;
70
+ }
71
+
72
+ function element(id, payload) {
73
+ return Buffer.concat([idBytes(id), sizeBytes(payload.length), payload]);
74
+ }
75
+
76
+ function uintElement(id, value) {
77
+ const bytes = [];
78
+ let rest = value;
79
+ do {
80
+ bytes.unshift(rest & 0xff);
81
+ rest = Math.floor(rest / 256);
82
+ } while (rest > 0);
83
+ return element(id, Buffer.from(bytes));
84
+ }
85
+
86
+ function stringElement(id, value) {
87
+ return element(id, Buffer.from(value, "utf8"));
88
+ }
89
+
90
+ /**
91
+ * An unsigned value at a FIXED four bytes. A cue's cluster position has to be
92
+ * written twice — once to measure the table, once with the position that
93
+ * measurement produced — and a value-sized element would make the second table
94
+ * a different length from the first, moving the very cluster it names.
95
+ */
96
+ function uint32Element(id, value) {
97
+ const payload = Buffer.alloc(4);
98
+ payload.writeUInt32BE(value, 0);
99
+ return element(id, payload);
100
+ }
101
+
102
+ function trackEntry({ number, type, codecId, language, flags = {}, languageBcp47 = null }) {
103
+ const parts = [
104
+ uintElement(ID_TRACK_NUMBER, number),
105
+ uintElement(ID_TRACK_TYPE, type),
106
+ stringElement(ID_CODEC_ID, codecId),
107
+ stringElement(ID_LANGUAGE, language)
108
+ ];
109
+ if (languageBcp47 !== null) {
110
+ parts.push(stringElement(ID_LANGUAGE_BCP47, languageBcp47));
111
+ }
112
+ for (const [id, value] of [
113
+ [ID_FLAG_ENABLED, flags.enabled],
114
+ [ID_FLAG_FORCED, flags.forced],
115
+ [ID_FLAG_HEARING_IMPAIRED, flags.hearingImpaired]
116
+ ]) {
117
+ if (value !== undefined) {
118
+ parts.push(uintElement(id, value ? 1 : 0));
119
+ }
120
+ }
121
+ return element(ID_TRACK_ENTRY, Buffer.concat(parts));
122
+ }
123
+
124
+ /**
125
+ * A file whose subtitle tracks are, in the container's own order: a picture
126
+ * one, then two text ones. ffmpeg numbers those `0:s:0`, `0:s:1`, `0:s:2`;
127
+ * the plan can only read the last two.
128
+ *
129
+ * The Cues table points both text tracks at one cluster, which is written after
130
+ * the table so its position can be stated.
131
+ *
132
+ * @returns {{ file: Buffer, clusterAt: number }}
133
+ */
134
+ function buildFile() {
135
+ const info = element(ID_INFO, uintElement(ID_TIMESTAMP_SCALE, 1_000_000));
136
+ const tracks = element(ID_TRACKS, Buffer.concat([
137
+ trackEntry({ number: 1, type: 1, codecId: "V_MPEG4/ISO/AVC", language: "und" }),
138
+ trackEntry({ number: 2, type: 17, codecId: "S_HDMV/PGS", language: "eng" }),
139
+ trackEntry({ number: 3, type: 17, codecId: "S_TEXT/UTF8", language: "rus" }),
140
+ trackEntry({ number: 4, type: 17, codecId: "S_TEXT/ASS", language: "eng" })
141
+ ]));
142
+
143
+ // Built twice: the cue points state where the cluster is, and that position
144
+ // is only known once everything before it has its final length. Every size
145
+ // and position here is written at a fixed width, so the draft and the final
146
+ // table are the same length.
147
+ const cuesWith = (clusterAt) => element(ID_CUES, Buffer.concat([
148
+ element(ID_CUE_POINT, Buffer.concat([
149
+ uintElement(ID_CUE_TIME, 1000),
150
+ element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
151
+ uintElement(ID_CUE_TRACK, 3),
152
+ uint32Element(ID_CUE_CLUSTER_POSITION, clusterAt)
153
+ ])),
154
+ element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
155
+ uintElement(ID_CUE_TRACK, 4),
156
+ uint32Element(ID_CUE_CLUSTER_POSITION, clusterAt)
157
+ ]))
158
+ ]))
159
+ ]));
160
+
161
+ const seekEntry = (targetId, position) => element(ID_SEEK, Buffer.concat([
162
+ element(ID_SEEK_ID, idBytes(targetId)),
163
+ element(ID_SEEK_POSITION, (() => {
164
+ const buffer = Buffer.alloc(4);
165
+ buffer.writeUInt32BE(position, 0);
166
+ return buffer;
167
+ })())
168
+ ]));
169
+ const seekHeadWith = (infoAt, tracksAt, cuesAt) => element(ID_SEEK_HEAD, Buffer.concat([
170
+ seekEntry(ID_INFO, infoAt),
171
+ seekEntry(ID_TRACKS, tracksAt),
172
+ seekEntry(ID_CUES, cuesAt)
173
+ ]));
174
+
175
+ const headLength = seekHeadWith(0, 0, 0).length;
176
+ const infoAt = headLength;
177
+ const tracksAt = infoAt + info.length;
178
+ const cuesAt = tracksAt + tracks.length;
179
+ // A position in the Cues table is measured from the Segment's payload, and so
180
+ // is the one the reader turns it into.
181
+ const clusterRelative = cuesAt + cuesWith(0).length;
182
+
183
+ // Enough of a cluster to be read and recognised: its own header and a
184
+ // timestamp. No blocks, so it yields no cues — what the walk test counts is
185
+ // that its bytes are fetched once, and that does not depend on their content.
186
+ const cluster = element(ID_CLUSTER, uintElement(ID_TIMESTAMP, 1000));
187
+
188
+ const segmentPayload = Buffer.concat([
189
+ seekHeadWith(infoAt, tracksAt, cuesAt),
190
+ info,
191
+ tracks,
192
+ cuesWith(clusterRelative),
193
+ cluster
194
+ ]);
195
+ const ebml = element(ID_EBML, Buffer.from([0x42, 0x86, 0x81, 0x01]));
196
+ const segment = element(ID_SEGMENT, segmentPayload);
197
+ const segmentDataOffset = ebml.length + segment.length - segmentPayload.length;
198
+ return {
199
+ file: Buffer.concat([ebml, segment]),
200
+ clusterAt: segmentDataOffset + clusterRelative
201
+ };
202
+ }
203
+
204
+ function readerOver(file) {
205
+ return async (start, end) => {
206
+ const last = Math.min(end, file.length - 1);
207
+ return start > last ? null : file.subarray(start, last + 1);
208
+ };
209
+ }
210
+
211
+ test("a text track is numbered as ffmpeg numbers it, past the picture ones", async () => {
212
+ const { file } = buildFile();
213
+
214
+ const plan = await MatroskaContainer.readSubtitlePlan(readerOver(file), file.length);
215
+
216
+ assert.equal(plan.declared.length, 3, "all three subtitle tracks are declared");
217
+ assert.deepEqual(plan.tracks.map((track) => track.trackNumber), [3, 4], "only the text ones are readable");
218
+ assert.deepEqual(
219
+ plan.tracks.map((track) => track.declaredIndex),
220
+ [1, 2],
221
+ "the PGS track is 0:s:0, so the text tracks are 0:s:1 and 0:s:2 — not 0 and 1"
222
+ );
223
+ });
224
+
225
+ /**
226
+ * A torrent holding one file entirely, counting the byte ranges read from it.
227
+ *
228
+ * @param {Buffer} bytes
229
+ * @returns {{ torrent: object, reads: Array<{ start: number, end: number }> }}
230
+ */
231
+ function torrentOver(bytes) {
232
+ const reads = [];
233
+ const file = {
234
+ name: "film.mkv",
235
+ length: bytes.length,
236
+ offset: 0,
237
+ createReadStream({ start = 0, end = bytes.length - 1 } = {}) {
238
+ reads.push({ start, end });
239
+ // Asynchronous on purpose: a read that resolves in the same tick would
240
+ // hide exactly the interleaving this test is about.
241
+ return Readable.from((async function* chunks() {
242
+ await new Promise((resolve) => setImmediate(resolve));
243
+ yield bytes.subarray(start, end + 1);
244
+ })());
245
+ }
246
+ };
247
+ return {
248
+ reads,
249
+ torrent: {
250
+ pieceLength: 1024,
251
+ bitfield: { get: () => true },
252
+ files: [file]
253
+ }
254
+ };
255
+ }
256
+
257
+ test("two walks of one file at the same time read each cluster once", async () => {
258
+ const { file, clusterAt } = buildFile();
259
+ const { torrent, reads } = torrentOver(file);
260
+ const sourceKey = "torrent:numbering-test";
261
+ forgetSubtitles(sourceKey);
262
+
263
+ // Both text tracks at once, which is what the warmup does on every verified
264
+ // piece and every three seconds.
265
+ const [first, second] = await Promise.all([
266
+ cuesHeldFor(torrent, 0, sourceKey, 3),
267
+ cuesHeldFor(torrent, 0, sourceKey, 4)
268
+ ]);
269
+
270
+ assert.equal(first.coveredClusters, 1, "the cluster the table names was walked");
271
+ assert.equal(second.coveredClusters, 1, "and the second track sees the same walk, not its own");
272
+ const clusterReads = reads.filter((range) => range.start === clusterAt);
273
+ assert.equal(
274
+ clusterReads.length,
275
+ 2,
276
+ "one probe of the cluster's header and one read of its body — not two of each"
277
+ );
278
+ forgetSubtitles(sourceKey);
279
+ });
280
+
281
+ /**
282
+ * A file whose subtitle tracks carry the flags RFC 9559 defines for them: one
283
+ * forced, one for viewers who cannot hear, one the file marks unusable, and one
284
+ * writing its language as RFC 5646 alongside the three-letter code.
285
+ *
286
+ * @returns {Buffer}
287
+ */
288
+ function fileWithFlags() {
289
+ const info = element(ID_INFO, uintElement(ID_TIMESTAMP_SCALE, 1_000_000));
290
+ const tracks = element(ID_TRACKS, Buffer.concat([
291
+ trackEntry({ number: 1, type: 1, codecId: "V_MPEG4/ISO/AVC", language: "und" }),
292
+ trackEntry({ number: 2, type: 17, codecId: "S_TEXT/UTF8", language: "rus", flags: { forced: true } }),
293
+ trackEntry({ number: 3, type: 17, codecId: "S_TEXT/UTF8", language: "eng", flags: { hearingImpaired: true } }),
294
+ trackEntry({ number: 4, type: 17, codecId: "S_TEXT/UTF8", language: "fre", flags: { enabled: false } }),
295
+ trackEntry({ number: 5, type: 17, codecId: "S_TEXT/ASS", language: "por", languageBcp47: "pt-BR" })
296
+ ]));
297
+ const seekEntry = (targetId, position) => element(ID_SEEK, Buffer.concat([
298
+ element(ID_SEEK_ID, idBytes(targetId)),
299
+ uint32Element(ID_SEEK_POSITION, position)
300
+ ]));
301
+ const seekHeadWith = (infoAt, tracksAt) => element(ID_SEEK_HEAD, Buffer.concat([
302
+ seekEntry(ID_INFO, infoAt),
303
+ seekEntry(ID_TRACKS, tracksAt)
304
+ ]));
305
+ const headLength = seekHeadWith(0, 0).length;
306
+ const segmentPayload = Buffer.concat([
307
+ seekHeadWith(headLength, headLength + info.length),
308
+ info,
309
+ tracks
310
+ ]);
311
+ const ebml = element(ID_EBML, Buffer.from([0x42, 0x86, 0x81, 0x01]));
312
+ return Buffer.concat([ebml, element(ID_SEGMENT, segmentPayload)]);
313
+ }
314
+
315
+ test("the flags the file states about a track are read, not guessed from its name", async () => {
316
+ const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
317
+
318
+ const forced = plan.tracks.find((track) => track.trackNumber === 2);
319
+ assert.equal(forced.isForced, true, "FlagForced 0x55AA");
320
+ assert.equal(forced.isHearingImpaired, false);
321
+
322
+ const sdh = plan.tracks.find((track) => track.trackNumber === 3);
323
+ assert.equal(sdh.isHearingImpaired, true, "FlagHearingImpaired 0x55AB");
324
+ assert.equal(sdh.isForced, false);
325
+ });
326
+
327
+ test("a track the file marks unusable is not offered, but is still counted", async () => {
328
+ // FlagEnabled (0xB9): "Set to 1 if the track is usable." Track 4 says 0, so
329
+ // it is not offered — but it KEEPS its place in the numbering, because ffmpeg
330
+ // keeps it: `matroskadec.c` parses MATROSKA_ID_TRACKFLAGENABLED as EBML_NONE,
331
+ // reading the element and storing nothing, so the stream is created and gets
332
+ // its own `0:s:N`. Dropping it here would shift every track after it.
333
+ const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
334
+
335
+ assert.equal(plan.tracks.some((track) => track.trackNumber === 4), false, "not offered for extraction");
336
+ const counted = plan.declared.find((track) => track.trackNumber === 4);
337
+ assert.ok(counted, "still declared, so the numbering does not move");
338
+ assert.equal(counted.isEnabled, false);
339
+ });
340
+
341
+ test("an unusable track keeps its place, so the tracks after it keep theirs", async () => {
342
+ const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
343
+
344
+ // s:0 forced, s:1 SDH, s:2 the unusable one, s:3 the Brazilian track.
345
+ assert.deepEqual(
346
+ plan.tracks.map((track) => [track.trackNumber, track.declaredIndex]),
347
+ [[2, 0], [3, 1], [5, 3]]
348
+ );
349
+ });
350
+
351
+ test("the list ffmpeg is lined up against still speaks ffmpeg's language codes", async () => {
352
+ // `declared` exists to be paired with the `-i` banner, which prints the
353
+ // three-letter code; reporting "pt-BR" there would break the pairing and cost
354
+ // the FlagDefault reading with it.
355
+ const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
356
+
357
+ const declared = plan.declared.find((track) => track.trackNumber === 5);
358
+ assert.equal(declared.language, "por");
359
+ assert.equal(declared.languageBcp47, "pt-BR");
360
+ });
361
+
362
+ test("where the file writes RFC 5646, that is the language", async () => {
363
+ // "If this element is used, then any Language elements used in the same
364
+ // TrackEntry MUST be ignored."
365
+ const plan = await MatroskaContainer.readSubtitlePlan(readerOver(fileWithFlags()), fileWithFlags().length);
366
+
367
+ const track = plan.tracks.find((entry) => entry.trackNumber === 5);
368
+ assert.equal(track.language, "pt-BR", "not the three-letter por");
369
+ assert.equal(track.languageBcp47, "pt-BR");
370
+ });