@torrent-tv/proxy 2.80.18 → 2.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/docs/encode-architecture.md +51 -2
- package/package.json +1 -1
- package/research/double-spawn-2026-09-10.md +171 -0
- package/services/disk/DiskSpace.js +146 -0
- package/services/disk/wire.js +60 -0
- package/services/encode/EncodeRun.js +37 -9
- package/services/encode/SegmentStore.js +284 -232
- package/services/encode/run-command.js +16 -1
- package/services/hls-session-manager.js +31 -128
- package/services/orchestrators/EncodeOrchestrator.js +52 -38
- package/services/piece-store/allowance.js +107 -0
- package/services/piece-store/piece-disk-store.js +365 -0
- package/services/piece-store/shared-piece-store.js +1549 -1535
- package/services/segment-formats/fmp4.js +54 -0
- package/services/segment-formats/mpegts.js +54 -0
- package/services/torrent-worker/client.js +32 -0
- package/services/torrent-worker/pool-adapter.js +15 -0
- package/services/torrent-worker/protocol.js +9 -0
- package/services/torrent-worker/worker.js +8 -1
- package/services/viewer/positions.js +48 -0
- package/test/audio-inventory.test.js +176 -176
- package/test/auto-quality-step.test.js +514 -514
- package/test/concurrent-cost.test.js +138 -138
- package/test/coverage-follows-the-disk.test.js +191 -187
- package/test/coverage-map.test.js +195 -195
- package/test/declared-tracks.test.js +35 -35
- package/test/disk-space.test.js +138 -0
- package/test/encode-orchestrator.test.js +0 -3
- package/test/encode-run.test.js +5 -12
- package/test/held-request-width.test.js +155 -155
- package/test/helpers/encode-run.js +2 -2
- package/test/matroska-blocks.test.js +0 -0
- package/test/matroska-cues-track.test.js +192 -192
- package/test/mp4-composition-times.test.js +0 -0
- package/test/mp4-subtitles.test.js +173 -173
- package/test/one-authority.test.js +281 -220
- package/test/orchestrator-wired.test.js +199 -195
- package/test/packet-witness-ring.test.js +236 -236
- package/test/packet-witness.test.js +148 -148
- package/test/piece-disk-store.test.js +267 -0
- package/test/piece-reader.test.js +4 -4
- package/test/piece-store-eviction.test.js +17 -17
- package/test/piece-store-reservations.test.js +20 -1
- package/test/piece-store-slow-disk.test.js +16 -1
- package/test/produced-copy-choice.test.js +258 -358
- package/test/read-window.test.js +6 -6
- package/test/run-intervals.test.js +100 -100
- package/test/seek-landing.test.js +109 -109
- package/test/segment-serve-wiring.test.js +8 -9
- package/test/segment-store-eviction.test.js +232 -0
- package/test/segment-store.test.js +238 -216
- package/test/segments-are-shared.test.js +1 -1
- package/test/shared-piece-store.test.js +12 -12
- package/test/sidecar-naming.test.js +142 -142
- package/test/subtitle-cue-framing.test.js +200 -200
- package/test/subtitle-cue-walk.test.js +369 -369
- package/test/subtitle-defaults.test.js +97 -97
- package/test/subtitle-track-numbering.test.js +370 -370
- package/test/tail-duplication.test.js +167 -167
- package/test/tracks-begin-together.test.js +195 -195
- package/test/two-viewers-one-picture.test.js +374 -374
- package/test/video-facts.test.js +102 -102
- package/test/wedge-certainty.test.js +131 -131
- package/services/encode/open-piece.js +0 -135
- package/services/piece-store/disk-tier.js +0 -151
- package/test/open-piece.test.js +0 -152
|
@@ -1,369 +1,369 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file What comes out of a real cluster walk, end to end.
|
|
3
|
-
*
|
|
4
|
-
* This is the check the walk never had. Everything around it was covered — the
|
|
5
|
-
* numbering a track is asked for by, the framing one cue's bytes carry, the
|
|
6
|
-
* thread a pull is answered on — and the walk itself, six hundred lines of it,
|
|
7
|
-
* had none: a file is built here with real clusters carrying real blocks, and
|
|
8
|
-
* the cues that come back out are stated exactly. Two field defects landed on
|
|
9
|
-
* this path in one week; a move of this code without a check that would catch a
|
|
10
|
-
* regression is a change nobody can verify.
|
|
11
|
-
*
|
|
12
|
-
* The four things asserted are the four the walk exists for: the text of each
|
|
13
|
-
* cue with its times, the found-order cursor a browser follows, the rule that
|
|
14
|
-
* only already-downloaded clusters are read, and the rule that one walk fills
|
|
15
|
-
* every track of the file.
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
import test from "node:test";
|
|
19
|
-
import assert from "node:assert/strict";
|
|
20
|
-
import { Readable } from "node:stream";
|
|
21
|
-
|
|
22
|
-
import { cuesHeldFor, warmSubtitleCues, forgetSubtitles } from "../services/torrent-worker/subtitle-cues.js";
|
|
23
|
-
|
|
24
|
-
const ID_EBML = 0x1a45dfa3;
|
|
25
|
-
const ID_SEGMENT = 0x18538067;
|
|
26
|
-
const ID_SEEK_HEAD = 0x114d9b74;
|
|
27
|
-
const ID_SEEK = 0x4dbb;
|
|
28
|
-
const ID_SEEK_ID = 0x53ab;
|
|
29
|
-
const ID_SEEK_POSITION = 0x53ac;
|
|
30
|
-
const ID_INFO = 0x1549a966;
|
|
31
|
-
const ID_TIMESTAMP_SCALE = 0x2ad7b1;
|
|
32
|
-
const ID_TRACKS = 0x1654ae6b;
|
|
33
|
-
const ID_TRACK_ENTRY = 0xae;
|
|
34
|
-
const ID_TRACK_NUMBER = 0xd7;
|
|
35
|
-
const ID_TRACK_TYPE = 0x83;
|
|
36
|
-
const ID_CODEC_ID = 0x86;
|
|
37
|
-
const ID_LANGUAGE = 0x22b59c;
|
|
38
|
-
const ID_CUES = 0x1c53bb6b;
|
|
39
|
-
const ID_CUE_POINT = 0xbb;
|
|
40
|
-
const ID_CUE_TIME = 0xb3;
|
|
41
|
-
const ID_CUE_TRACK_POSITIONS = 0xb7;
|
|
42
|
-
const ID_CUE_TRACK = 0xf7;
|
|
43
|
-
const ID_CUE_CLUSTER_POSITION = 0xf1;
|
|
44
|
-
const ID_CLUSTER = 0x1f43b675;
|
|
45
|
-
const ID_TIMESTAMP = 0xe7;
|
|
46
|
-
const ID_BLOCK_GROUP = 0xa0;
|
|
47
|
-
const ID_BLOCK = 0xa1;
|
|
48
|
-
const ID_BLOCK_DURATION = 0x9b;
|
|
49
|
-
|
|
50
|
-
function idBytes(id) {
|
|
51
|
-
const bytes = [];
|
|
52
|
-
let rest = id;
|
|
53
|
-
while (rest > 0) {
|
|
54
|
-
bytes.unshift(rest & 0xff);
|
|
55
|
-
rest = Math.floor(rest / 256);
|
|
56
|
-
}
|
|
57
|
-
return Buffer.from(bytes);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
/** A four-byte size: a valid variable-length integer whatever the payload. */
|
|
61
|
-
function sizeBytes(size) {
|
|
62
|
-
const buffer = Buffer.alloc(4);
|
|
63
|
-
buffer.writeUInt32BE(size, 0);
|
|
64
|
-
buffer[0] |= 0x10;
|
|
65
|
-
return buffer;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function element(id, payload) {
|
|
69
|
-
return Buffer.concat([idBytes(id), sizeBytes(payload.length), payload]);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function uintElement(id, value) {
|
|
73
|
-
const bytes = [];
|
|
74
|
-
let rest = value;
|
|
75
|
-
do {
|
|
76
|
-
bytes.unshift(rest & 0xff);
|
|
77
|
-
rest = Math.floor(rest / 256);
|
|
78
|
-
} while (rest > 0);
|
|
79
|
-
return element(id, Buffer.from(bytes));
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function stringElement(id, value) {
|
|
83
|
-
return element(id, Buffer.from(value, "utf8"));
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/** A fixed four bytes, so a table measured twice comes out the same length. */
|
|
87
|
-
function uint32Element(id, value) {
|
|
88
|
-
const payload = Buffer.alloc(4);
|
|
89
|
-
payload.writeUInt32BE(value, 0);
|
|
90
|
-
return element(id, payload);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function trackEntry({ number, type, codecId, language }) {
|
|
94
|
-
return element(ID_TRACK_ENTRY, Buffer.concat([
|
|
95
|
-
uintElement(ID_TRACK_NUMBER, number),
|
|
96
|
-
uintElement(ID_TRACK_TYPE, type),
|
|
97
|
-
stringElement(ID_CODEC_ID, codecId),
|
|
98
|
-
stringElement(ID_LANGUAGE, language)
|
|
99
|
-
]));
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/**
|
|
103
|
-
* One cue, as a BlockGroup so it can carry its own duration.
|
|
104
|
-
*
|
|
105
|
-
* @param {{ track: number, relativeTicks: number, durationTicks: number, text: string }} params
|
|
106
|
-
* @returns {Buffer}
|
|
107
|
-
*/
|
|
108
|
-
function cueBlock({ track, relativeTicks, durationTicks, text }) {
|
|
109
|
-
const header = Buffer.alloc(4);
|
|
110
|
-
header[0] = 0x80 | track; // track numbers under 128 are one byte, marker set
|
|
111
|
-
header.writeInt16BE(relativeTicks, 1);
|
|
112
|
-
header[3] = 0;
|
|
113
|
-
const block = element(ID_BLOCK, Buffer.concat([header, Buffer.from(text, "utf8")]));
|
|
114
|
-
return element(ID_BLOCK_GROUP, Buffer.concat([block, uintElement(ID_BLOCK_DURATION, durationTicks)]));
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* A file with two text tracks and two clusters, each carrying a line for each
|
|
119
|
-
* track.
|
|
120
|
-
*
|
|
121
|
-
* Track 2 is `S_TEXT/UTF8`, whose block payload IS the text. Track 3 is
|
|
122
|
-
* `S_TEXT/ASS`, whose payload is the dialogue row without its `Dialogue:`
|
|
123
|
-
* header — eight comma-separated fields and then the words, which is what
|
|
124
|
-
* Matroska states and what the framing rule has to strip.
|
|
125
|
-
*
|
|
126
|
-
* @returns {{ file: Buffer, clusterAt: number[] }}
|
|
127
|
-
*/
|
|
128
|
-
function buildFile() {
|
|
129
|
-
const info = element(ID_INFO, uintElement(ID_TIMESTAMP_SCALE, 1_000_000));
|
|
130
|
-
const tracks = element(ID_TRACKS, Buffer.concat([
|
|
131
|
-
trackEntry({ number: 1, type: 1, codecId: "V_MPEG4/ISO/AVC", language: "und" }),
|
|
132
|
-
trackEntry({ number: 2, type: 17, codecId: "S_TEXT/UTF8", language: "eng" }),
|
|
133
|
-
trackEntry({ number: 3, type: 17, codecId: "S_TEXT/ASS", language: "rus" })
|
|
134
|
-
]));
|
|
135
|
-
|
|
136
|
-
const clusterOne = element(ID_CLUSTER, Buffer.concat([
|
|
137
|
-
uintElement(ID_TIMESTAMP, 1000),
|
|
138
|
-
cueBlock({ track: 2, relativeTicks: 0, durationTicks: 2000, text: "First English line" }),
|
|
139
|
-
cueBlock({
|
|
140
|
-
track: 3,
|
|
141
|
-
relativeTicks: 500,
|
|
142
|
-
durationTicks: 1500,
|
|
143
|
-
// ReadOrder, Layer, Style, Name, MarginL, MarginR, MarginV, Effect,
|
|
144
|
-
// then the text — the Dialogue row without its header and its timestamps.
|
|
145
|
-
text: "1,0,Default,,0000,0000,0000,,Первая русская строка"
|
|
146
|
-
})
|
|
147
|
-
]));
|
|
148
|
-
const clusterTwo = element(ID_CLUSTER, Buffer.concat([
|
|
149
|
-
uintElement(ID_TIMESTAMP, 10_000),
|
|
150
|
-
cueBlock({ track: 2, relativeTicks: 0, durationTicks: 1000, text: "Second English line" }),
|
|
151
|
-
cueBlock({
|
|
152
|
-
track: 3,
|
|
153
|
-
relativeTicks: 250,
|
|
154
|
-
durationTicks: 1000,
|
|
155
|
-
text: "2,0,Default,,0000,0000,0000,,Вторая, с запятой"
|
|
156
|
-
})
|
|
157
|
-
]));
|
|
158
|
-
|
|
159
|
-
const cuesWith = (firstAt, secondAt) => element(ID_CUES, Buffer.concat([
|
|
160
|
-
element(ID_CUE_POINT, Buffer.concat([
|
|
161
|
-
uintElement(ID_CUE_TIME, 1000),
|
|
162
|
-
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
163
|
-
uintElement(ID_CUE_TRACK, 2),
|
|
164
|
-
uint32Element(ID_CUE_CLUSTER_POSITION, firstAt)
|
|
165
|
-
])),
|
|
166
|
-
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
167
|
-
uintElement(ID_CUE_TRACK, 3),
|
|
168
|
-
uint32Element(ID_CUE_CLUSTER_POSITION, firstAt)
|
|
169
|
-
]))
|
|
170
|
-
])),
|
|
171
|
-
element(ID_CUE_POINT, Buffer.concat([
|
|
172
|
-
uintElement(ID_CUE_TIME, 10_000),
|
|
173
|
-
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
174
|
-
uintElement(ID_CUE_TRACK, 2),
|
|
175
|
-
uint32Element(ID_CUE_CLUSTER_POSITION, secondAt)
|
|
176
|
-
])),
|
|
177
|
-
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
178
|
-
uintElement(ID_CUE_TRACK, 3),
|
|
179
|
-
uint32Element(ID_CUE_CLUSTER_POSITION, secondAt)
|
|
180
|
-
]))
|
|
181
|
-
]))
|
|
182
|
-
]));
|
|
183
|
-
|
|
184
|
-
const seekEntry = (targetId, position) => element(ID_SEEK, Buffer.concat([
|
|
185
|
-
element(ID_SEEK_ID, idBytes(targetId)),
|
|
186
|
-
uint32Element(ID_SEEK_POSITION, position)
|
|
187
|
-
]));
|
|
188
|
-
const seekHeadWith = (infoAt, tracksAt, cuesAt) => element(ID_SEEK_HEAD, Buffer.concat([
|
|
189
|
-
seekEntry(ID_INFO, infoAt),
|
|
190
|
-
seekEntry(ID_TRACKS, tracksAt),
|
|
191
|
-
seekEntry(ID_CUES, cuesAt)
|
|
192
|
-
]));
|
|
193
|
-
|
|
194
|
-
const headLength = seekHeadWith(0, 0, 0).length;
|
|
195
|
-
const infoAt = headLength;
|
|
196
|
-
const tracksAt = infoAt + info.length;
|
|
197
|
-
const cuesAt = tracksAt + tracks.length;
|
|
198
|
-
const firstClusterAt = cuesAt + cuesWith(0, 0).length;
|
|
199
|
-
const secondClusterAt = firstClusterAt + clusterOne.length;
|
|
200
|
-
|
|
201
|
-
const segmentPayload = Buffer.concat([
|
|
202
|
-
seekHeadWith(infoAt, tracksAt, cuesAt),
|
|
203
|
-
info,
|
|
204
|
-
tracks,
|
|
205
|
-
cuesWith(firstClusterAt, secondClusterAt),
|
|
206
|
-
clusterOne,
|
|
207
|
-
clusterTwo
|
|
208
|
-
]);
|
|
209
|
-
const ebml = element(ID_EBML, Buffer.from([0x42, 0x86, 0x81, 0x01]));
|
|
210
|
-
const segment = element(ID_SEGMENT, segmentPayload);
|
|
211
|
-
const segmentDataOffset = ebml.length + segment.length - segmentPayload.length;
|
|
212
|
-
return {
|
|
213
|
-
file: Buffer.concat([ebml, segment]),
|
|
214
|
-
clusterAt: [segmentDataOffset + firstClusterAt, segmentDataOffset + secondClusterAt]
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
/**
|
|
219
|
-
* A torrent over one file, holding whichever pieces the caller says.
|
|
220
|
-
*
|
|
221
|
-
* @param {Buffer} bytes
|
|
222
|
-
* @param {(index: number) => boolean} [holds] - Whether a piece is downloaded.
|
|
223
|
-
* @returns {{ torrent: object, reads: Array<{ start: number, end: number }>, pieceLength: number }}
|
|
224
|
-
*/
|
|
225
|
-
function torrentOver(bytes, holds = () => true) {
|
|
226
|
-
const pieceLength = 64;
|
|
227
|
-
const reads = [];
|
|
228
|
-
const file = {
|
|
229
|
-
name: "film.mkv",
|
|
230
|
-
length: bytes.length,
|
|
231
|
-
offset: 0,
|
|
232
|
-
createReadStream({ start = 0, end = bytes.length - 1 } = {}) {
|
|
233
|
-
reads.push({ start, end });
|
|
234
|
-
return Readable.from((async function* chunks() {
|
|
235
|
-
await new Promise((resolve) => setImmediate(resolve));
|
|
236
|
-
yield bytes.subarray(start, end + 1);
|
|
237
|
-
})());
|
|
238
|
-
}
|
|
239
|
-
};
|
|
240
|
-
return {
|
|
241
|
-
reads,
|
|
242
|
-
pieceLength,
|
|
243
|
-
torrent: {
|
|
244
|
-
name: "film",
|
|
245
|
-
pieceLength,
|
|
246
|
-
bitfield: { get: (index) => holds(index) },
|
|
247
|
-
files: [file]
|
|
248
|
-
}
|
|
249
|
-
};
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
test("a walk of a downloaded file gives up every cue, unframed, with its times", async () => {
|
|
253
|
-
const { file } = buildFile();
|
|
254
|
-
const { torrent } = torrentOver(file);
|
|
255
|
-
const sourceKey = "a".repeat(40);
|
|
256
|
-
forgetSubtitles(sourceKey);
|
|
257
|
-
try {
|
|
258
|
-
const plain = await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
259
|
-
assert.deepEqual(
|
|
260
|
-
plain.cues.map((cue) => [cue.startSeconds, cue.endSeconds, cue.text]),
|
|
261
|
-
[
|
|
262
|
-
[1, 3, "First English line"],
|
|
263
|
-
[10, 11, "Second English line"]
|
|
264
|
-
]
|
|
265
|
-
);
|
|
266
|
-
|
|
267
|
-
const ass = await cuesHeldFor(torrent, 0, sourceKey, 3);
|
|
268
|
-
assert.deepEqual(
|
|
269
|
-
ass.cues.map((cue) => [cue.startSeconds, cue.endSeconds, cue.text]),
|
|
270
|
-
[
|
|
271
|
-
[1.5, 3, "Первая русская строка"],
|
|
272
|
-
[10.25, 11.25, "Вторая, с запятой"]
|
|
273
|
-
],
|
|
274
|
-
"the eight fields Matroska puts before the text are the container's framing, not the words"
|
|
275
|
-
);
|
|
276
|
-
} finally {
|
|
277
|
-
forgetSubtitles(sourceKey);
|
|
278
|
-
}
|
|
279
|
-
});
|
|
280
|
-
|
|
281
|
-
test("the cursor is the order cues were FOUND, and it never repeats", async () => {
|
|
282
|
-
const { file } = buildFile();
|
|
283
|
-
const { torrent } = torrentOver(file);
|
|
284
|
-
const sourceKey = "b".repeat(40);
|
|
285
|
-
forgetSubtitles(sourceKey);
|
|
286
|
-
try {
|
|
287
|
-
const held = await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
288
|
-
const cursors = held.cues.map((cue) => cue.seq);
|
|
289
|
-
assert.deepEqual(cursors, [1, 2], "two cues found, in the order they were read");
|
|
290
|
-
assert.equal(new Set(cursors).size, cursors.length, "no cue shares a cursor with another");
|
|
291
|
-
} finally {
|
|
292
|
-
forgetSubtitles(sourceKey);
|
|
293
|
-
}
|
|
294
|
-
});
|
|
295
|
-
|
|
296
|
-
test("a cluster whose bytes are not downloaded is left for next time", async () => {
|
|
297
|
-
const { file, clusterAt } = buildFile();
|
|
298
|
-
// Everything except the pieces the SECOND cluster sits in.
|
|
299
|
-
const { torrent, pieceLength } = torrentOver(file, (index) => index * pieceLength < clusterAt[1]);
|
|
300
|
-
const sourceKey = "c".repeat(40);
|
|
301
|
-
forgetSubtitles(sourceKey);
|
|
302
|
-
try {
|
|
303
|
-
const held = await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
304
|
-
assert.deepEqual(
|
|
305
|
-
held.cues.map((cue) => cue.text),
|
|
306
|
-
["First English line"],
|
|
307
|
-
"the downloaded cluster is read and the other is not — nothing is asked of the swarm"
|
|
308
|
-
);
|
|
309
|
-
assert.equal(held.coveredClusters, 1);
|
|
310
|
-
assert.equal(held.indexedClusters, 2, "the file states two, and one is not here yet");
|
|
311
|
-
} finally {
|
|
312
|
-
forgetSubtitles(sourceKey);
|
|
313
|
-
}
|
|
314
|
-
});
|
|
315
|
-
|
|
316
|
-
test("one walk fills every track, and a second call reads no cluster again", async () => {
|
|
317
|
-
const { file, clusterAt } = buildFile();
|
|
318
|
-
const { torrent, reads } = torrentOver(file);
|
|
319
|
-
const sourceKey = "d".repeat(40);
|
|
320
|
-
forgetSubtitles(sourceKey);
|
|
321
|
-
try {
|
|
322
|
-
await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
323
|
-
const clusterReads = reads.filter((range) => clusterAt.includes(range.start)).length;
|
|
324
|
-
|
|
325
|
-
const other = await cuesHeldFor(torrent, 0, sourceKey, 3);
|
|
326
|
-
assert.equal(other.cues.length, 2, "the other track was filled by the same walk");
|
|
327
|
-
assert.equal(
|
|
328
|
-
reads.filter((range) => clusterAt.includes(range.start)).length,
|
|
329
|
-
clusterReads,
|
|
330
|
-
"asking for the second track reads no cluster a second time"
|
|
331
|
-
);
|
|
332
|
-
} finally {
|
|
333
|
-
forgetSubtitles(sourceKey);
|
|
334
|
-
}
|
|
335
|
-
});
|
|
336
|
-
|
|
337
|
-
test("the warm pass reports what is new, by the number the browser knows", async () => {
|
|
338
|
-
const { file } = buildFile();
|
|
339
|
-
const { torrent } = torrentOver(file);
|
|
340
|
-
const sourceKey = "e".repeat(40);
|
|
341
|
-
forgetSubtitles(sourceKey);
|
|
342
|
-
try {
|
|
343
|
-
const first = await warmSubtitleCues(torrent, 0, sourceKey);
|
|
344
|
-
assert.deepEqual(
|
|
345
|
-
first.map((entry) => [entry.trackIndex, entry.cues.length, entry.language]).sort(),
|
|
346
|
-
[[0, 2, "eng"], [1, 2, "rus"]].sort(),
|
|
347
|
-
"both text tracks gained two cues, numbered as ffmpeg numbers them"
|
|
348
|
-
);
|
|
349
|
-
|
|
350
|
-
const again = await warmSubtitleCues(torrent, 0, sourceKey);
|
|
351
|
-
assert.deepEqual(again, [], "nothing is new the second time round");
|
|
352
|
-
} finally {
|
|
353
|
-
forgetSubtitles(sourceKey);
|
|
354
|
-
}
|
|
355
|
-
});
|
|
356
|
-
|
|
357
|
-
test("a torrent that cannot say which pieces it holds is refused, not answered emptily", async () => {
|
|
358
|
-
const { file } = buildFile();
|
|
359
|
-
const { torrent } = torrentOver(file);
|
|
360
|
-
const sourceKey = "f".repeat(40);
|
|
361
|
-
forgetSubtitles(sourceKey);
|
|
362
|
-
try {
|
|
363
|
-
const held = await cuesHeldFor({ ...torrent, bitfield: null }, 0, sourceKey, 2);
|
|
364
|
-
assert.deepEqual(held.cues, []);
|
|
365
|
-
assert.equal(held.track, null, "the answer says nothing was read, not that there is nothing");
|
|
366
|
-
} finally {
|
|
367
|
-
forgetSubtitles(sourceKey);
|
|
368
|
-
}
|
|
369
|
-
});
|
|
1
|
+
/**
|
|
2
|
+
* @file What comes out of a real cluster walk, end to end.
|
|
3
|
+
*
|
|
4
|
+
* This is the check the walk never had. Everything around it was covered — the
|
|
5
|
+
* numbering a track is asked for by, the framing one cue's bytes carry, the
|
|
6
|
+
* thread a pull is answered on — and the walk itself, six hundred lines of it,
|
|
7
|
+
* had none: a file is built here with real clusters carrying real blocks, and
|
|
8
|
+
* the cues that come back out are stated exactly. Two field defects landed on
|
|
9
|
+
* this path in one week; a move of this code without a check that would catch a
|
|
10
|
+
* regression is a change nobody can verify.
|
|
11
|
+
*
|
|
12
|
+
* The four things asserted are the four the walk exists for: the text of each
|
|
13
|
+
* cue with its times, the found-order cursor a browser follows, the rule that
|
|
14
|
+
* only already-downloaded clusters are read, and the rule that one walk fills
|
|
15
|
+
* every track of the file.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import test from "node:test";
|
|
19
|
+
import assert from "node:assert/strict";
|
|
20
|
+
import { Readable } from "node:stream";
|
|
21
|
+
|
|
22
|
+
import { cuesHeldFor, warmSubtitleCues, forgetSubtitles } from "../services/torrent-worker/subtitle-cues.js";
|
|
23
|
+
|
|
24
|
+
const ID_EBML = 0x1a45dfa3;
|
|
25
|
+
const ID_SEGMENT = 0x18538067;
|
|
26
|
+
const ID_SEEK_HEAD = 0x114d9b74;
|
|
27
|
+
const ID_SEEK = 0x4dbb;
|
|
28
|
+
const ID_SEEK_ID = 0x53ab;
|
|
29
|
+
const ID_SEEK_POSITION = 0x53ac;
|
|
30
|
+
const ID_INFO = 0x1549a966;
|
|
31
|
+
const ID_TIMESTAMP_SCALE = 0x2ad7b1;
|
|
32
|
+
const ID_TRACKS = 0x1654ae6b;
|
|
33
|
+
const ID_TRACK_ENTRY = 0xae;
|
|
34
|
+
const ID_TRACK_NUMBER = 0xd7;
|
|
35
|
+
const ID_TRACK_TYPE = 0x83;
|
|
36
|
+
const ID_CODEC_ID = 0x86;
|
|
37
|
+
const ID_LANGUAGE = 0x22b59c;
|
|
38
|
+
const ID_CUES = 0x1c53bb6b;
|
|
39
|
+
const ID_CUE_POINT = 0xbb;
|
|
40
|
+
const ID_CUE_TIME = 0xb3;
|
|
41
|
+
const ID_CUE_TRACK_POSITIONS = 0xb7;
|
|
42
|
+
const ID_CUE_TRACK = 0xf7;
|
|
43
|
+
const ID_CUE_CLUSTER_POSITION = 0xf1;
|
|
44
|
+
const ID_CLUSTER = 0x1f43b675;
|
|
45
|
+
const ID_TIMESTAMP = 0xe7;
|
|
46
|
+
const ID_BLOCK_GROUP = 0xa0;
|
|
47
|
+
const ID_BLOCK = 0xa1;
|
|
48
|
+
const ID_BLOCK_DURATION = 0x9b;
|
|
49
|
+
|
|
50
|
+
function idBytes(id) {
|
|
51
|
+
const bytes = [];
|
|
52
|
+
let rest = id;
|
|
53
|
+
while (rest > 0) {
|
|
54
|
+
bytes.unshift(rest & 0xff);
|
|
55
|
+
rest = Math.floor(rest / 256);
|
|
56
|
+
}
|
|
57
|
+
return Buffer.from(bytes);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** A four-byte size: a valid variable-length integer whatever the payload. */
|
|
61
|
+
function sizeBytes(size) {
|
|
62
|
+
const buffer = Buffer.alloc(4);
|
|
63
|
+
buffer.writeUInt32BE(size, 0);
|
|
64
|
+
buffer[0] |= 0x10;
|
|
65
|
+
return buffer;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function element(id, payload) {
|
|
69
|
+
return Buffer.concat([idBytes(id), sizeBytes(payload.length), payload]);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function uintElement(id, value) {
|
|
73
|
+
const bytes = [];
|
|
74
|
+
let rest = value;
|
|
75
|
+
do {
|
|
76
|
+
bytes.unshift(rest & 0xff);
|
|
77
|
+
rest = Math.floor(rest / 256);
|
|
78
|
+
} while (rest > 0);
|
|
79
|
+
return element(id, Buffer.from(bytes));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function stringElement(id, value) {
|
|
83
|
+
return element(id, Buffer.from(value, "utf8"));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** A fixed four bytes, so a table measured twice comes out the same length. */
|
|
87
|
+
function uint32Element(id, value) {
|
|
88
|
+
const payload = Buffer.alloc(4);
|
|
89
|
+
payload.writeUInt32BE(value, 0);
|
|
90
|
+
return element(id, payload);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function trackEntry({ number, type, codecId, language }) {
|
|
94
|
+
return element(ID_TRACK_ENTRY, Buffer.concat([
|
|
95
|
+
uintElement(ID_TRACK_NUMBER, number),
|
|
96
|
+
uintElement(ID_TRACK_TYPE, type),
|
|
97
|
+
stringElement(ID_CODEC_ID, codecId),
|
|
98
|
+
stringElement(ID_LANGUAGE, language)
|
|
99
|
+
]));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* One cue, as a BlockGroup so it can carry its own duration.
|
|
104
|
+
*
|
|
105
|
+
* @param {{ track: number, relativeTicks: number, durationTicks: number, text: string }} params
|
|
106
|
+
* @returns {Buffer}
|
|
107
|
+
*/
|
|
108
|
+
function cueBlock({ track, relativeTicks, durationTicks, text }) {
|
|
109
|
+
const header = Buffer.alloc(4);
|
|
110
|
+
header[0] = 0x80 | track; // track numbers under 128 are one byte, marker set
|
|
111
|
+
header.writeInt16BE(relativeTicks, 1);
|
|
112
|
+
header[3] = 0;
|
|
113
|
+
const block = element(ID_BLOCK, Buffer.concat([header, Buffer.from(text, "utf8")]));
|
|
114
|
+
return element(ID_BLOCK_GROUP, Buffer.concat([block, uintElement(ID_BLOCK_DURATION, durationTicks)]));
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* A file with two text tracks and two clusters, each carrying a line for each
|
|
119
|
+
* track.
|
|
120
|
+
*
|
|
121
|
+
* Track 2 is `S_TEXT/UTF8`, whose block payload IS the text. Track 3 is
|
|
122
|
+
* `S_TEXT/ASS`, whose payload is the dialogue row without its `Dialogue:`
|
|
123
|
+
* header — eight comma-separated fields and then the words, which is what
|
|
124
|
+
* Matroska states and what the framing rule has to strip.
|
|
125
|
+
*
|
|
126
|
+
* @returns {{ file: Buffer, clusterAt: number[] }}
|
|
127
|
+
*/
|
|
128
|
+
function buildFile() {
|
|
129
|
+
const info = element(ID_INFO, uintElement(ID_TIMESTAMP_SCALE, 1_000_000));
|
|
130
|
+
const tracks = element(ID_TRACKS, Buffer.concat([
|
|
131
|
+
trackEntry({ number: 1, type: 1, codecId: "V_MPEG4/ISO/AVC", language: "und" }),
|
|
132
|
+
trackEntry({ number: 2, type: 17, codecId: "S_TEXT/UTF8", language: "eng" }),
|
|
133
|
+
trackEntry({ number: 3, type: 17, codecId: "S_TEXT/ASS", language: "rus" })
|
|
134
|
+
]));
|
|
135
|
+
|
|
136
|
+
const clusterOne = element(ID_CLUSTER, Buffer.concat([
|
|
137
|
+
uintElement(ID_TIMESTAMP, 1000),
|
|
138
|
+
cueBlock({ track: 2, relativeTicks: 0, durationTicks: 2000, text: "First English line" }),
|
|
139
|
+
cueBlock({
|
|
140
|
+
track: 3,
|
|
141
|
+
relativeTicks: 500,
|
|
142
|
+
durationTicks: 1500,
|
|
143
|
+
// ReadOrder, Layer, Style, Name, MarginL, MarginR, MarginV, Effect,
|
|
144
|
+
// then the text — the Dialogue row without its header and its timestamps.
|
|
145
|
+
text: "1,0,Default,,0000,0000,0000,,Первая русская строка"
|
|
146
|
+
})
|
|
147
|
+
]));
|
|
148
|
+
const clusterTwo = element(ID_CLUSTER, Buffer.concat([
|
|
149
|
+
uintElement(ID_TIMESTAMP, 10_000),
|
|
150
|
+
cueBlock({ track: 2, relativeTicks: 0, durationTicks: 1000, text: "Second English line" }),
|
|
151
|
+
cueBlock({
|
|
152
|
+
track: 3,
|
|
153
|
+
relativeTicks: 250,
|
|
154
|
+
durationTicks: 1000,
|
|
155
|
+
text: "2,0,Default,,0000,0000,0000,,Вторая, с запятой"
|
|
156
|
+
})
|
|
157
|
+
]));
|
|
158
|
+
|
|
159
|
+
const cuesWith = (firstAt, secondAt) => element(ID_CUES, Buffer.concat([
|
|
160
|
+
element(ID_CUE_POINT, Buffer.concat([
|
|
161
|
+
uintElement(ID_CUE_TIME, 1000),
|
|
162
|
+
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
163
|
+
uintElement(ID_CUE_TRACK, 2),
|
|
164
|
+
uint32Element(ID_CUE_CLUSTER_POSITION, firstAt)
|
|
165
|
+
])),
|
|
166
|
+
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
167
|
+
uintElement(ID_CUE_TRACK, 3),
|
|
168
|
+
uint32Element(ID_CUE_CLUSTER_POSITION, firstAt)
|
|
169
|
+
]))
|
|
170
|
+
])),
|
|
171
|
+
element(ID_CUE_POINT, Buffer.concat([
|
|
172
|
+
uintElement(ID_CUE_TIME, 10_000),
|
|
173
|
+
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
174
|
+
uintElement(ID_CUE_TRACK, 2),
|
|
175
|
+
uint32Element(ID_CUE_CLUSTER_POSITION, secondAt)
|
|
176
|
+
])),
|
|
177
|
+
element(ID_CUE_TRACK_POSITIONS, Buffer.concat([
|
|
178
|
+
uintElement(ID_CUE_TRACK, 3),
|
|
179
|
+
uint32Element(ID_CUE_CLUSTER_POSITION, secondAt)
|
|
180
|
+
]))
|
|
181
|
+
]))
|
|
182
|
+
]));
|
|
183
|
+
|
|
184
|
+
const seekEntry = (targetId, position) => element(ID_SEEK, Buffer.concat([
|
|
185
|
+
element(ID_SEEK_ID, idBytes(targetId)),
|
|
186
|
+
uint32Element(ID_SEEK_POSITION, position)
|
|
187
|
+
]));
|
|
188
|
+
const seekHeadWith = (infoAt, tracksAt, cuesAt) => element(ID_SEEK_HEAD, Buffer.concat([
|
|
189
|
+
seekEntry(ID_INFO, infoAt),
|
|
190
|
+
seekEntry(ID_TRACKS, tracksAt),
|
|
191
|
+
seekEntry(ID_CUES, cuesAt)
|
|
192
|
+
]));
|
|
193
|
+
|
|
194
|
+
const headLength = seekHeadWith(0, 0, 0).length;
|
|
195
|
+
const infoAt = headLength;
|
|
196
|
+
const tracksAt = infoAt + info.length;
|
|
197
|
+
const cuesAt = tracksAt + tracks.length;
|
|
198
|
+
const firstClusterAt = cuesAt + cuesWith(0, 0).length;
|
|
199
|
+
const secondClusterAt = firstClusterAt + clusterOne.length;
|
|
200
|
+
|
|
201
|
+
const segmentPayload = Buffer.concat([
|
|
202
|
+
seekHeadWith(infoAt, tracksAt, cuesAt),
|
|
203
|
+
info,
|
|
204
|
+
tracks,
|
|
205
|
+
cuesWith(firstClusterAt, secondClusterAt),
|
|
206
|
+
clusterOne,
|
|
207
|
+
clusterTwo
|
|
208
|
+
]);
|
|
209
|
+
const ebml = element(ID_EBML, Buffer.from([0x42, 0x86, 0x81, 0x01]));
|
|
210
|
+
const segment = element(ID_SEGMENT, segmentPayload);
|
|
211
|
+
const segmentDataOffset = ebml.length + segment.length - segmentPayload.length;
|
|
212
|
+
return {
|
|
213
|
+
file: Buffer.concat([ebml, segment]),
|
|
214
|
+
clusterAt: [segmentDataOffset + firstClusterAt, segmentDataOffset + secondClusterAt]
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* A torrent over one file, holding whichever pieces the caller says.
|
|
220
|
+
*
|
|
221
|
+
* @param {Buffer} bytes
|
|
222
|
+
* @param {(index: number) => boolean} [holds] - Whether a piece is downloaded.
|
|
223
|
+
* @returns {{ torrent: object, reads: Array<{ start: number, end: number }>, pieceLength: number }}
|
|
224
|
+
*/
|
|
225
|
+
function torrentOver(bytes, holds = () => true) {
|
|
226
|
+
const pieceLength = 64;
|
|
227
|
+
const reads = [];
|
|
228
|
+
const file = {
|
|
229
|
+
name: "film.mkv",
|
|
230
|
+
length: bytes.length,
|
|
231
|
+
offset: 0,
|
|
232
|
+
createReadStream({ start = 0, end = bytes.length - 1 } = {}) {
|
|
233
|
+
reads.push({ start, end });
|
|
234
|
+
return Readable.from((async function* chunks() {
|
|
235
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
236
|
+
yield bytes.subarray(start, end + 1);
|
|
237
|
+
})());
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
return {
|
|
241
|
+
reads,
|
|
242
|
+
pieceLength,
|
|
243
|
+
torrent: {
|
|
244
|
+
name: "film",
|
|
245
|
+
pieceLength,
|
|
246
|
+
bitfield: { get: (index) => holds(index) },
|
|
247
|
+
files: [file]
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
test("a walk of a downloaded file gives up every cue, unframed, with its times", async () => {
|
|
253
|
+
const { file } = buildFile();
|
|
254
|
+
const { torrent } = torrentOver(file);
|
|
255
|
+
const sourceKey = "a".repeat(40);
|
|
256
|
+
forgetSubtitles(sourceKey);
|
|
257
|
+
try {
|
|
258
|
+
const plain = await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
259
|
+
assert.deepEqual(
|
|
260
|
+
plain.cues.map((cue) => [cue.startSeconds, cue.endSeconds, cue.text]),
|
|
261
|
+
[
|
|
262
|
+
[1, 3, "First English line"],
|
|
263
|
+
[10, 11, "Second English line"]
|
|
264
|
+
]
|
|
265
|
+
);
|
|
266
|
+
|
|
267
|
+
const ass = await cuesHeldFor(torrent, 0, sourceKey, 3);
|
|
268
|
+
assert.deepEqual(
|
|
269
|
+
ass.cues.map((cue) => [cue.startSeconds, cue.endSeconds, cue.text]),
|
|
270
|
+
[
|
|
271
|
+
[1.5, 3, "Первая русская строка"],
|
|
272
|
+
[10.25, 11.25, "Вторая, с запятой"]
|
|
273
|
+
],
|
|
274
|
+
"the eight fields Matroska puts before the text are the container's framing, not the words"
|
|
275
|
+
);
|
|
276
|
+
} finally {
|
|
277
|
+
forgetSubtitles(sourceKey);
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
test("the cursor is the order cues were FOUND, and it never repeats", async () => {
|
|
282
|
+
const { file } = buildFile();
|
|
283
|
+
const { torrent } = torrentOver(file);
|
|
284
|
+
const sourceKey = "b".repeat(40);
|
|
285
|
+
forgetSubtitles(sourceKey);
|
|
286
|
+
try {
|
|
287
|
+
const held = await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
288
|
+
const cursors = held.cues.map((cue) => cue.seq);
|
|
289
|
+
assert.deepEqual(cursors, [1, 2], "two cues found, in the order they were read");
|
|
290
|
+
assert.equal(new Set(cursors).size, cursors.length, "no cue shares a cursor with another");
|
|
291
|
+
} finally {
|
|
292
|
+
forgetSubtitles(sourceKey);
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
test("a cluster whose bytes are not downloaded is left for next time", async () => {
|
|
297
|
+
const { file, clusterAt } = buildFile();
|
|
298
|
+
// Everything except the pieces the SECOND cluster sits in.
|
|
299
|
+
const { torrent, pieceLength } = torrentOver(file, (index) => index * pieceLength < clusterAt[1]);
|
|
300
|
+
const sourceKey = "c".repeat(40);
|
|
301
|
+
forgetSubtitles(sourceKey);
|
|
302
|
+
try {
|
|
303
|
+
const held = await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
304
|
+
assert.deepEqual(
|
|
305
|
+
held.cues.map((cue) => cue.text),
|
|
306
|
+
["First English line"],
|
|
307
|
+
"the downloaded cluster is read and the other is not — nothing is asked of the swarm"
|
|
308
|
+
);
|
|
309
|
+
assert.equal(held.coveredClusters, 1);
|
|
310
|
+
assert.equal(held.indexedClusters, 2, "the file states two, and one is not here yet");
|
|
311
|
+
} finally {
|
|
312
|
+
forgetSubtitles(sourceKey);
|
|
313
|
+
}
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test("one walk fills every track, and a second call reads no cluster again", async () => {
|
|
317
|
+
const { file, clusterAt } = buildFile();
|
|
318
|
+
const { torrent, reads } = torrentOver(file);
|
|
319
|
+
const sourceKey = "d".repeat(40);
|
|
320
|
+
forgetSubtitles(sourceKey);
|
|
321
|
+
try {
|
|
322
|
+
await cuesHeldFor(torrent, 0, sourceKey, 2);
|
|
323
|
+
const clusterReads = reads.filter((range) => clusterAt.includes(range.start)).length;
|
|
324
|
+
|
|
325
|
+
const other = await cuesHeldFor(torrent, 0, sourceKey, 3);
|
|
326
|
+
assert.equal(other.cues.length, 2, "the other track was filled by the same walk");
|
|
327
|
+
assert.equal(
|
|
328
|
+
reads.filter((range) => clusterAt.includes(range.start)).length,
|
|
329
|
+
clusterReads,
|
|
330
|
+
"asking for the second track reads no cluster a second time"
|
|
331
|
+
);
|
|
332
|
+
} finally {
|
|
333
|
+
forgetSubtitles(sourceKey);
|
|
334
|
+
}
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
test("the warm pass reports what is new, by the number the browser knows", async () => {
|
|
338
|
+
const { file } = buildFile();
|
|
339
|
+
const { torrent } = torrentOver(file);
|
|
340
|
+
const sourceKey = "e".repeat(40);
|
|
341
|
+
forgetSubtitles(sourceKey);
|
|
342
|
+
try {
|
|
343
|
+
const first = await warmSubtitleCues(torrent, 0, sourceKey);
|
|
344
|
+
assert.deepEqual(
|
|
345
|
+
first.map((entry) => [entry.trackIndex, entry.cues.length, entry.language]).sort(),
|
|
346
|
+
[[0, 2, "eng"], [1, 2, "rus"]].sort(),
|
|
347
|
+
"both text tracks gained two cues, numbered as ffmpeg numbers them"
|
|
348
|
+
);
|
|
349
|
+
|
|
350
|
+
const again = await warmSubtitleCues(torrent, 0, sourceKey);
|
|
351
|
+
assert.deepEqual(again, [], "nothing is new the second time round");
|
|
352
|
+
} finally {
|
|
353
|
+
forgetSubtitles(sourceKey);
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
test("a torrent that cannot say which pieces it holds is refused, not answered emptily", async () => {
|
|
358
|
+
const { file } = buildFile();
|
|
359
|
+
const { torrent } = torrentOver(file);
|
|
360
|
+
const sourceKey = "f".repeat(40);
|
|
361
|
+
forgetSubtitles(sourceKey);
|
|
362
|
+
try {
|
|
363
|
+
const held = await cuesHeldFor({ ...torrent, bitfield: null }, 0, sourceKey, 2);
|
|
364
|
+
assert.deepEqual(held.cues, []);
|
|
365
|
+
assert.equal(held.track, null, "the answer says nothing was read, not that there is nothing");
|
|
366
|
+
} finally {
|
|
367
|
+
forgetSubtitles(sourceKey);
|
|
368
|
+
}
|
|
369
|
+
});
|