@torrent-tv/proxy 2.74.1 → 2.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1504 -1453
- package/CLAUDE.md +19 -6
- package/biome.json +182 -1
- package/docs/container-architecture.md +27 -6
- package/docs/encode-run-state.md +1 -1
- package/knip.json +14 -0
- package/package.json +1 -1
- package/routes/api/sources/warm/post.js +1 -1
- package/routes/api/transcode-sessions/post.js +185 -185
- package/routes/api/transcode-sessions/progress/get.js +5 -1
- package/routes/transcode/audio-file/get.js +11 -1
- package/routes/transcode/audio-warm/get.js +11 -1
- package/routes/transcode/session-file/get.js +1 -1
- package/routes/transcode/variant-file/get.js +10 -1
- package/scripts/render-run-graph.js +2 -2
- package/server.js +25 -0
- package/services/audio-inventory.js +9 -214
- package/services/container/AviContainer.js +266 -81
- package/services/container/Container.js +281 -80
- package/services/container/ContainerFactory.js +67 -0
- package/services/container/MatroskaContainer.js +327 -8
- package/services/container/Mp4Container.js +373 -27
- package/services/container/SubtitleFileContainer.js +0 -1
- package/services/controllers/SubtitleController.js +128 -128
- package/services/demand/index.js +7 -10
- package/services/download/registry.js +0 -14
- package/services/encode/CoverageMap.js +281 -0
- package/services/encode/EncodePlan.js +255 -0
- package/services/encode/EncodeRun.js +587 -0
- package/services/encode/Encoder.js +84 -0
- package/services/encode/NvencEncoder.js +45 -0
- package/services/encode/QsvEncoder.js +47 -0
- package/services/encode/SegmentDemand.js +0 -0
- package/services/encode/SegmentStore.js +529 -0
- package/services/encode/SoftwareEncoder.js +111 -0
- package/services/encode/V4l2m2mEncoder.js +53 -0
- package/services/encode/VaapiEncoder.js +53 -0
- package/services/encode/args.js +200 -0
- package/services/{encode-exit.js → encode/encode-exit.js} +17 -0
- package/services/encode/index.js +9 -0
- package/services/encode/run-command.js +647 -0
- package/services/hls-session-manager.js +11073 -10711
- package/services/hwaccel.js +1688 -1992
- package/services/orchestrators/EncodeOrchestrator.js +359 -0
- package/services/output/LiveOutputs.js +213 -0
- package/services/output/Output.js +94 -0
- package/services/output/OutputSpec.js +195 -0
- package/services/output/Timeline.js +220 -0
- package/services/output/index.js +1 -0
- package/services/output/ladder.js +26 -0
- package/services/playback-planner.js +806 -747
- package/services/produced-index.js +222 -300
- package/services/source/SourceFile.js +346 -0
- package/services/{sidecar-files.js → torrent/files.js} +107 -11
- package/services/torrent/naming.js +619 -0
- package/services/torrent-worker/client.js +10 -0
- package/services/torrent-worker/container-tracks.js +71 -43
- package/services/torrent-worker/pool-adapter.js +370 -333
- package/services/torrent-worker/protocol.js +7 -0
- package/services/torrent-worker/subtitle-cues.js +549 -549
- package/services/torrent-worker/worker.js +18 -0
- package/services/tracks/AudioTrack.js +131 -40
- package/services/tracks/TextSubtitleTrack.js +287 -287
- package/services/tracks/index.js +15 -14
- package/services/viewer/Viewer.js +145 -0
- package/services/viewer/Viewers.js +124 -0
- package/test/audio-inventory.test.js +176 -177
- package/test/auto-quality-step.test.js +508 -506
- package/test/behind-head-repair.test.js +17 -7
- package/test/coverage-map.test.js +153 -0
- package/test/cut-times-timeline.test.js +6 -5
- package/test/cuts-follow-published-grid.test.js +4 -4
- package/test/decode-cost.test.js +31 -12
- package/test/encode-exit.test.js +1 -1
- package/test/encode-orchestrator.test.js +196 -0
- package/test/encode-plan.test.js +245 -0
- package/test/encode-run-state.test.js +2 -2
- package/test/encode-run.test.js +168 -0
- package/test/encoder-kinds.test.js +122 -0
- package/test/held-request-width.test.js +9 -3
- package/test/helpers/encode-run.js +128 -0
- package/test/keyframe-index-accuracy.test.js +19 -12
- package/test/keyframes-belong-to-the-file.test.js +132 -0
- package/test/matroska-cues-track.test.js +192 -192
- package/test/mp4-composition-times.test.js +0 -0
- package/test/orchestrator-wired.test.js +164 -0
- package/test/output-shape.test.js +68 -0
- package/test/output-spec.test.js +157 -0
- package/test/produced-copy-choice.test.js +58 -92
- package/test/produced-index.test.js +142 -188
- package/test/quality-variants.test.js +1079 -1075
- package/test/run-graph-drift.test.js +1 -1
- package/test/run-intervals.test.js +329 -0
- package/test/run-position-follows-published-grid.test.js +4 -4
- package/test/seek-landing.test.js +8 -8
- package/test/seek-target-not-superseded.test.js +21 -9
- package/test/segment-demand.test.js +82 -0
- package/test/segment-serve-wiring.test.js +47 -52
- package/test/segment-store.test.js +187 -0
- package/test/segments-are-shared.test.js +175 -0
- package/test/sidecar-naming.test.js +142 -0
- package/test/source-file.test.js +133 -0
- package/test/stale-request-after-seek.test.js +18 -12
- package/test/subtitle-language.test.js +252 -252
- package/test/timeline.test.js +95 -0
- package/test/{sidecar-files.test.js → torrent-files.test.js} +44 -1
- package/test/torrent-naming.test.js +255 -0
- package/test/tracks-begin-together.test.js +44 -32
- package/test/two-viewers-one-picture.test.js +347 -0
- package/test/video-facts.test.js +102 -0
- package/test/viewer-outputs.test.js +273 -0
- package/test/viewer.test.js +91 -0
- package/utils/perf.js +1 -63
- package/services/container/index.js +0 -6
- package/services/container-index/avi.js +0 -167
- package/services/container-index/index.js +0 -118
- package/services/container-index/matroska.js +0 -336
- package/services/container-index/mp4.js +0 -358
- package/services/controllers/index.js +0 -2
- package/services/download/index.js +0 -8
- package/services/orchestrators/index.js +0 -2
- /package/services/{container-index → container}/ebml-reader.js +0 -0
- /package/services/{encode-run-state.js → encode/encode-run-state.js} +0 -0
- /package/services/{language-detect.js → tracks/language-detect.js} +0 -0
|
@@ -17,11 +17,14 @@
|
|
|
17
17
|
|
|
18
18
|
import test from "node:test";
|
|
19
19
|
import assert from "node:assert/strict";
|
|
20
|
+
import { SourceFile } from "../services/source/SourceFile.js";
|
|
21
|
+
import { Timeline } from "../services/output/Timeline.js";
|
|
20
22
|
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
21
23
|
import os from "node:os";
|
|
22
24
|
import path from "node:path";
|
|
23
25
|
import { HlsSessionManager } from "../services/hls-session-manager.js";
|
|
24
|
-
import { ENCODE_RUN_STATE } from "../services/encode-run-state.js";
|
|
26
|
+
import { ENCODE_RUN_STATE } from "../services/encode/encode-run-state.js";
|
|
27
|
+
import { startRunOn } from "./helpers/encode-run.js";
|
|
25
28
|
import { fmp4Format } from "../services/segment-formats/fmp4.js";
|
|
26
29
|
|
|
27
30
|
const MOVIE_TIMESCALE = 1000;
|
|
@@ -132,25 +135,35 @@ async function managerWithReadySegment(overrides = {}) {
|
|
|
132
135
|
const session = {
|
|
133
136
|
id: SESSION_ID,
|
|
134
137
|
dirPath,
|
|
138
|
+
// Where this file is cut, held by the file. A fixture that stated it
|
|
139
|
+
// on the session was describing what production no longer does.
|
|
140
|
+
timeline: new Timeline({
|
|
141
|
+
boundaries: [0, SEGMENT_START_SECONDS, 25],
|
|
142
|
+
cutGrid: "uniform"
|
|
143
|
+
}),
|
|
135
144
|
state: "ready",
|
|
136
|
-
|
|
145
|
+
file: new SourceFile({ sourceKey: "source-1", fileIndex: 0, name: "video.mkv" }),
|
|
146
|
+
// An ordinary session reads its own file, and its sound is inside it. The
|
|
147
|
+
// three differ only for a soundtrack shipped as a file of its own.
|
|
148
|
+
get inputFile() { return this.file; },
|
|
149
|
+
get audioFile() { return this.file; },
|
|
137
150
|
startedAt: Date.now(),
|
|
138
151
|
createEntryMs: Date.now(),
|
|
139
152
|
lastAccessedAt: Date.now(),
|
|
140
|
-
|
|
153
|
+
runs: new Set(),
|
|
141
154
|
lastError: "",
|
|
142
155
|
consumers: new Set(),
|
|
143
156
|
segmentFormat: overrides.segmentFormat ?? fmp4Format,
|
|
144
|
-
usesExplicitCuts: true,
|
|
145
157
|
useSyntheticPlaylist: true,
|
|
146
158
|
playlistText: "#EXTM3U\n",
|
|
147
|
-
segmentBoundaries: [0, SEGMENT_START_SECONDS, 25],
|
|
148
159
|
initBytes: fmp4Format.extractInit(piece),
|
|
149
|
-
encodeStartIndex: 0,
|
|
150
160
|
firstSegmentLogged: false,
|
|
151
161
|
waitEpoch: 0
|
|
152
162
|
};
|
|
153
163
|
manager.sessionsById.set(SESSION_ID, session);
|
|
164
|
+
// The run in force. The fixture's segments live directly in the session
|
|
165
|
+
// directory, which is exactly what one run's directory is here.
|
|
166
|
+
startRunOn(session, { from: 0, usesExplicitCuts: true });
|
|
154
167
|
return { manager, session, dirPath };
|
|
155
168
|
}
|
|
156
169
|
|
|
@@ -163,11 +176,11 @@ test("serving a segment records what its real start says about the container's i
|
|
|
163
176
|
// The tally is counted in the module tests; what this pins is that serving a
|
|
164
177
|
// segment reaches it at all. A counter nothing increments reports a clean
|
|
165
178
|
// index for every file forever, which is worse than no measurement.
|
|
166
|
-
session.indexCheck = { checked: 0, disagreed: 0, maxDeviationSec: 0, firstDisagreementIndex: -1, seen: new Set() };
|
|
179
|
+
session.timeline.indexCheck = { checked: 0, disagreed: 0, maxDeviationSec: 0, firstDisagreementIndex: -1, seen: new Set() };
|
|
167
180
|
|
|
168
181
|
await manager.getFileStream(SESSION_ID, "segment-00000.mp4", { requestSeq: 1 });
|
|
169
182
|
|
|
170
|
-
assert.equal(session.indexCheck.checked, 1, "the boundary that was just produced must have been examined");
|
|
183
|
+
assert.equal(session.timeline.indexCheck.checked, 1, "the boundary that was just produced must have been examined");
|
|
171
184
|
});
|
|
172
185
|
|
|
173
186
|
test("a segment that exists is served, not reported as still being produced", async (t) => {
|
|
@@ -202,7 +215,7 @@ test("a segment that exists is served, not reported as still being produced", as
|
|
|
202
215
|
"the segment must be stamped with where it really begins"
|
|
203
216
|
);
|
|
204
217
|
assert.equal(
|
|
205
|
-
session.indexCheck.checked,
|
|
218
|
+
session.timeline.indexCheck.checked,
|
|
206
219
|
1,
|
|
207
220
|
"and the piece's own position must still have been read, or nothing measures the index"
|
|
208
221
|
);
|
|
@@ -241,8 +254,6 @@ test("a run's FIRST segment is served once the encoder has passed it, without wa
|
|
|
241
254
|
// successor and nothing is producing one. Waiting for that successor is what
|
|
242
255
|
// held #317 for 46 s and then answered 404 to a browser that had given up.
|
|
243
256
|
await rm(path.join(dirPath, "segment-00001.mp4"));
|
|
244
|
-
session.encodeStartIndex = 0;
|
|
245
|
-
session.ffmpeg = { killed: true, kill() {} };
|
|
246
257
|
session.progress = { processedSeconds: SEGMENT_START_SECONDS + 10 };
|
|
247
258
|
|
|
248
259
|
const result = await manager.getFileStream(SESSION_ID, "segment-00000.mp4", { requestSeq: 1 });
|
|
@@ -254,38 +265,26 @@ test("a run's FIRST segment is served once the encoder has passed it, without wa
|
|
|
254
265
|
);
|
|
255
266
|
});
|
|
256
267
|
|
|
257
|
-
test("a segment
|
|
268
|
+
test("a segment an earlier run made is served by whatever run is going now", async (t) => {
|
|
258
269
|
const { manager, session, dirPath } = await managerWithReadySegment();
|
|
259
270
|
t.after(async () => {
|
|
260
271
|
await manager.disposeAll();
|
|
261
272
|
await rm(dirPath, { recursive: true, force: true });
|
|
262
273
|
});
|
|
263
|
-
//
|
|
264
|
-
//
|
|
265
|
-
//
|
|
266
|
-
//
|
|
267
|
-
|
|
268
|
-
const piece = selfContainedPiece(SEGMENT_START_SECONDS);
|
|
269
|
-
await mkdir(path.join(dirPath, "run-1"), { recursive: true });
|
|
270
|
-
await mkdir(path.join(dirPath, "run-2"), { recursive: true });
|
|
271
|
-
await writeFile(path.join(dirPath, "run-1", "segment-00000.mp4"), Buffer.alloc(8));
|
|
272
|
-
await writeFile(path.join(dirPath, "run-2", "segment-00000.mp4"), piece);
|
|
273
|
-
await writeFile(path.join(dirPath, "run-2", "segment-00001.mp4"), piece);
|
|
274
|
-
await rm(path.join(dirPath, "segment-00000.mp4"));
|
|
275
|
-
await rm(path.join(dirPath, "segment-00001.mp4"));
|
|
276
|
-
session.encodeStartIndex = 0;
|
|
274
|
+
// Every run of an output writes into that output's own directory — there is
|
|
275
|
+
// one, and runs are kept apart by their stretches instead. So a viewer who
|
|
276
|
+
// seeks back is served what an earlier run made, with no restart and no
|
|
277
|
+
// search: the run going now began at #40 and #0 is simply there.
|
|
278
|
+
startRunOn(session, { from: 40, usesExplicitCuts: true });
|
|
277
279
|
|
|
278
280
|
const result = await manager.getFileStream(SESSION_ID, "segment-00000.mp4", { requestSeq: 1 });
|
|
279
281
|
|
|
280
|
-
assert.equal(result.kind, "file", "
|
|
282
|
+
assert.equal(result.kind, "file", "what any run of this output made is what this output holds");
|
|
281
283
|
const chunks = [];
|
|
282
284
|
for await (const chunk of result.stream) {
|
|
283
285
|
chunks.push(chunk);
|
|
284
286
|
}
|
|
285
|
-
assert.ok(
|
|
286
|
-
Buffer.concat(chunks).length > 8,
|
|
287
|
-
"the newest run's output must win — the older file here is the 8-byte stub a killed run leaves"
|
|
288
|
-
);
|
|
287
|
+
assert.ok(Buffer.concat(chunks).length > 0);
|
|
289
288
|
});
|
|
290
289
|
|
|
291
290
|
test("serving a run's own segment moves the run out of STARTING", async (t) => {
|
|
@@ -294,41 +293,37 @@ test("serving a run's own segment moves the run out of STARTING", async (t) => {
|
|
|
294
293
|
await manager.disposeAll();
|
|
295
294
|
await rm(dirPath, { recursive: true, force: true });
|
|
296
295
|
});
|
|
297
|
-
// The table lives in `encode-run-state.js` and is tested there as a graph.
|
|
296
|
+
// The table lives in `encode/encode-run-state.js` and is tested there as a graph.
|
|
298
297
|
// What this pins is that a real serve REACHES it: a state nothing writes
|
|
299
298
|
// describes every run as still starting, for ever, and the log built on it
|
|
300
299
|
// would say so too.
|
|
301
|
-
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
|
|
305
|
-
|
|
300
|
+
// A run that has not yet made anything: starting. The fixture's own run is
|
|
301
|
+
// let go first, because this test is about ONE run and what serving does to
|
|
302
|
+
// it.
|
|
303
|
+
session.runs.clear();
|
|
304
|
+
const run = startRunOn(session, { from: 0, producing: false, usesExplicitCuts: true });
|
|
305
|
+
assert.equal(run.state, ENCODE_RUN_STATE.STARTING);
|
|
306
306
|
|
|
307
307
|
await manager.getFileStream(SESSION_ID, "segment-00000.mp4", { requestSeq: 1 });
|
|
308
308
|
|
|
309
|
-
assert.equal(
|
|
309
|
+
assert.equal(run.state, ENCODE_RUN_STATE.PRODUCING);
|
|
310
310
|
});
|
|
311
311
|
|
|
312
|
-
test("a segment
|
|
312
|
+
test("a segment behind a run's own start does not claim that run has produced", async (t) => {
|
|
313
313
|
const { manager, session, dirPath } = await managerWithReadySegment();
|
|
314
314
|
t.after(async () => {
|
|
315
315
|
await manager.disposeAll();
|
|
316
316
|
await rm(dirPath, { recursive: true, force: true });
|
|
317
317
|
});
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
//
|
|
321
|
-
//
|
|
322
|
-
//
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
// case an index comparison gets wrong: after a backward seek the old run's
|
|
326
|
-
// output sits ahead of the new run's beginning.
|
|
327
|
-
session.runState = ENCODE_RUN_STATE.STARTING;
|
|
328
|
-
session.encodeStartIndex = 0;
|
|
329
|
-
session.runDirPath = path.join(dirPath, "run-7");
|
|
318
|
+
// What moves a run out of starting is ITS OWN first segment. An earlier run's
|
|
319
|
+
// output sits in the same directory, and after a backward seek some of it
|
|
320
|
+
// lies ahead of the new run's beginning — so the question cannot be asked of
|
|
321
|
+
// the number alone, and a run believed to be producing is one the look-ahead
|
|
322
|
+
// may suspend and the seek path may wave through as "already covered".
|
|
323
|
+
session.runs.clear();
|
|
324
|
+
const run = startRunOn(session, { from: 5, producing: false, usesExplicitCuts: true });
|
|
330
325
|
|
|
331
326
|
await manager.getFileStream(SESSION_ID, "segment-00000.mp4", { requestSeq: 1 });
|
|
332
327
|
|
|
333
|
-
assert.equal(
|
|
328
|
+
assert.equal(run.state, ENCODE_RUN_STATE.STARTING);
|
|
334
329
|
});
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file One store of segments, addressed by what they are, and what it makes of
|
|
3
|
+
* what a killed process left behind.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import test from "node:test";
|
|
7
|
+
import assert from "node:assert/strict";
|
|
8
|
+
import { mkdtempSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
|
9
|
+
import os from "node:os";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { directoryNameFor, SegmentStore } from "../services/encode/SegmentStore.js";
|
|
12
|
+
import { fmp4Format } from "../services/segment-formats/fmp4.js";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @returns {{ store: SegmentStore, root: string, lines: string[] }}
|
|
16
|
+
*/
|
|
17
|
+
function storeInATempRoot() {
|
|
18
|
+
const root = mkdtempSync(path.join(os.tmpdir(), "segment-store-"));
|
|
19
|
+
const lines = [];
|
|
20
|
+
const store = new SegmentStore({
|
|
21
|
+
root,
|
|
22
|
+
logger: { info: (line) => lines.push(line), warn: (line) => lines.push(line) }
|
|
23
|
+
});
|
|
24
|
+
return { store, root, lines };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @param {string} dir
|
|
29
|
+
* @param {number} index
|
|
30
|
+
* @param {number} bytes
|
|
31
|
+
*/
|
|
32
|
+
function writeSegment(dir, index, bytes = 16) {
|
|
33
|
+
writeFileSync(path.join(dir, fmp4Format.segmentFileName(index)), Buffer.alloc(bytes));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const KEY = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
|
|
37
|
+
|
|
38
|
+
test("two viewers of one output are given the same directory", (t) => {
|
|
39
|
+
const { store, root } = storeInATempRoot();
|
|
40
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
41
|
+
|
|
42
|
+
// Which viewer asked never enters it — the address is what the segments are.
|
|
43
|
+
assert.equal(store.directoryFor(KEY), store.directoryFor(KEY));
|
|
44
|
+
assert.notEqual(store.directoryFor(KEY), store.directoryFor(`${KEY}:other`));
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test("the directory says what it holds, so a later process can tell", (t) => {
|
|
48
|
+
const { store, root } = storeInATempRoot();
|
|
49
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
50
|
+
|
|
51
|
+
const dir = store.directoryFor(KEY);
|
|
52
|
+
const names = readdirSync(dir);
|
|
53
|
+
assert.deepEqual(names, ["key.txt"]);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("a segment is proven closed by the existence of the next one", (t) => {
|
|
57
|
+
const { store, root } = storeInATempRoot();
|
|
58
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
59
|
+
|
|
60
|
+
const dir = store.directoryFor(KEY);
|
|
61
|
+
store.useFormat(KEY, fmp4Format);
|
|
62
|
+
writeSegment(dir, 0);
|
|
63
|
+
writeSegment(dir, 1);
|
|
64
|
+
writeSegment(dir, 2);
|
|
65
|
+
|
|
66
|
+
// The `segment` muxer writes no temporary file, so a file that exists may
|
|
67
|
+
// still be growing; only a run that has moved past it proves otherwise.
|
|
68
|
+
assert.deepEqual(store.provenNumbers(KEY), [0, 1]);
|
|
69
|
+
assert.equal(store.unprovenNumber(KEY), 2);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test("a file of no bytes is not a segment, whatever it is called", (t) => {
|
|
73
|
+
const { store, root } = storeInATempRoot();
|
|
74
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
75
|
+
|
|
76
|
+
const dir = store.directoryFor(KEY);
|
|
77
|
+
store.useFormat(KEY, fmp4Format);
|
|
78
|
+
writeSegment(dir, 0);
|
|
79
|
+
writeSegment(dir, 1);
|
|
80
|
+
writeSegment(dir, 2, 0);
|
|
81
|
+
|
|
82
|
+
// What a run killed the moment after opening its next piece leaves behind.
|
|
83
|
+
// Counted as a segment once, it closed the only hole in the numbering and
|
|
84
|
+
// convinced the look-ahead the encoder had produced it.
|
|
85
|
+
assert.equal(store.pathOf(KEY, 2), null);
|
|
86
|
+
assert.deepEqual(store.provenNumbers(KEY), [0]);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("a directory that has not moved is not read again", (t) => {
|
|
90
|
+
const { store, root } = storeInATempRoot();
|
|
91
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
92
|
+
|
|
93
|
+
const dir = store.directoryFor(KEY);
|
|
94
|
+
store.useFormat(KEY, fmp4Format);
|
|
95
|
+
writeSegment(dir, 0);
|
|
96
|
+
writeSegment(dir, 1);
|
|
97
|
+
|
|
98
|
+
const first = store.refresh(KEY);
|
|
99
|
+
const second = store.refresh(KEY);
|
|
100
|
+
assert.equal(first, second, "the same reading is handed back, not a fresh listing");
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("what a killed process left is found, named and counted", (t) => {
|
|
104
|
+
const { store, root, lines } = storeInATempRoot();
|
|
105
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
106
|
+
|
|
107
|
+
const dir = path.join(root, directoryNameFor(KEY));
|
|
108
|
+
mkdirSync(dir, { recursive: true });
|
|
109
|
+
writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
|
|
110
|
+
writeSegment(dir, 0);
|
|
111
|
+
writeSegment(dir, 1);
|
|
112
|
+
const orphan = path.join(root, "0123456789abcdef");
|
|
113
|
+
mkdirSync(orphan, { recursive: true });
|
|
114
|
+
writeSegment(orphan, 0);
|
|
115
|
+
|
|
116
|
+
const swept = store.sweep();
|
|
117
|
+
assert.equal(swept.directories, 2);
|
|
118
|
+
assert.equal(swept.unidentified, 1, "the one with no key file cannot be matched to a request");
|
|
119
|
+
assert.equal(swept.segments, 3);
|
|
120
|
+
assert.match(lines.join("\n"), /startup sweep/);
|
|
121
|
+
assert.match(lines.join("\n"), /ended\s+without anything recording why/);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test("what survived a kill is taken back, minus the piece nothing proves", (t) => {
|
|
125
|
+
const { store, root } = storeInATempRoot();
|
|
126
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
127
|
+
|
|
128
|
+
const dir = path.join(root, directoryNameFor(KEY));
|
|
129
|
+
mkdirSync(dir, { recursive: true });
|
|
130
|
+
writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
|
|
131
|
+
for (let index = 0; index <= 5; index += 1) {
|
|
132
|
+
writeSegment(dir, index);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const taken = store.adoptWhatSurvived(() => fmp4Format);
|
|
136
|
+
|
|
137
|
+
assert.equal(taken.adopted, 1);
|
|
138
|
+
assert.equal(taken.unprovenRemoved, 1, "the highest number was being written when the run died");
|
|
139
|
+
// Five kept rather than six thrown away: a copied segment's bytes depend only
|
|
140
|
+
// on the source, and re-encoding them costs a machine already short of it.
|
|
141
|
+
assert.deepEqual(store.provenNumbers(KEY), [0, 1, 2, 3, 4]);
|
|
142
|
+
assert.equal(store.pathOf(KEY, 5), null);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test("a directory that cannot name itself is thrown away rather than served", (t) => {
|
|
146
|
+
const { store, root } = storeInATempRoot();
|
|
147
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
148
|
+
|
|
149
|
+
const orphan = path.join(root, "0123456789abcdef");
|
|
150
|
+
mkdirSync(orphan, { recursive: true });
|
|
151
|
+
writeSegment(orphan, 0);
|
|
152
|
+
|
|
153
|
+
const taken = store.adoptWhatSurvived(() => fmp4Format);
|
|
154
|
+
|
|
155
|
+
assert.equal(taken.adopted, 0);
|
|
156
|
+
assert.equal(taken.dropped, 1);
|
|
157
|
+
assert.equal(readdirSync(root).length, 0);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("an output this proxy can no longer serve is thrown away too", (t) => {
|
|
161
|
+
const { store, root } = storeInATempRoot();
|
|
162
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
163
|
+
|
|
164
|
+
const dir = path.join(root, directoryNameFor(KEY));
|
|
165
|
+
mkdirSync(dir, { recursive: true });
|
|
166
|
+
writeFileSync(path.join(dir, "key.txt"), `${KEY}\n`);
|
|
167
|
+
writeSegment(dir, 0);
|
|
168
|
+
|
|
169
|
+
const taken = store.adoptWhatSurvived(() => null);
|
|
170
|
+
|
|
171
|
+
assert.equal(taken.dropped, 1);
|
|
172
|
+
assert.equal(readdirSync(root).length, 0);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test("what the store weighs is reported, and dropping one output frees it", (t) => {
|
|
176
|
+
const { store, root } = storeInATempRoot();
|
|
177
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
178
|
+
|
|
179
|
+
const dir = store.directoryFor(KEY);
|
|
180
|
+
store.useFormat(KEY, fmp4Format);
|
|
181
|
+
writeSegment(dir, 0, 1000);
|
|
182
|
+
writeSegment(dir, 1, 1000);
|
|
183
|
+
|
|
184
|
+
assert.equal(store.stats().bytes, 2000);
|
|
185
|
+
store.drop(KEY, "nobody is watching it");
|
|
186
|
+
assert.equal(store.stats().outputs, 0);
|
|
187
|
+
});
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Two sessions of one output write into one place, and each serves what
|
|
3
|
+
* the other has made.
|
|
4
|
+
*
|
|
5
|
+
* This is what the whole address change is for. Measured 2026-09-03: two
|
|
6
|
+
* viewers of one copied file got two sessions with byte-identical output —
|
|
7
|
+
* `segment-00000.mp4` of 4141899 bytes, twice — and neither could see the
|
|
8
|
+
* other's work, because the directory was named after the session's own random
|
|
9
|
+
* id (`research/two-viewers-one-file-2026-09-03.md`).
|
|
10
|
+
*
|
|
11
|
+
* The start position is deliberately NOT part of the address: segment 42 covers
|
|
12
|
+
* the same span whoever began where, so two viewers who opened the same film at
|
|
13
|
+
* different places produce interchangeable segments and must share them.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import test from "node:test";
|
|
17
|
+
import assert from "node:assert/strict";
|
|
18
|
+
import { SourceFile } from "../services/source/SourceFile.js";
|
|
19
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
20
|
+
import os from "node:os";
|
|
21
|
+
import path from "node:path";
|
|
22
|
+
import { HlsSessionManager } from "../services/hls-session-manager.js";
|
|
23
|
+
import { SegmentStore } from "../services/encode/SegmentStore.js";
|
|
24
|
+
import { Timeline } from "../services/output/Timeline.js";
|
|
25
|
+
import { fmp4Format } from "../services/segment-formats/fmp4.js";
|
|
26
|
+
|
|
27
|
+
const OUTPUT_KEY = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @returns {{ manager: HlsSessionManager, store: SegmentStore, root: string }}
|
|
31
|
+
*/
|
|
32
|
+
function managerOverATempStore() {
|
|
33
|
+
const root = mkdtempSync(path.join(os.tmpdir(), "shared-segments-"));
|
|
34
|
+
const store = new SegmentStore({ root });
|
|
35
|
+
const manager = new HlsSessionManager({
|
|
36
|
+
enabled: true,
|
|
37
|
+
ffmpegBin: "ffmpeg",
|
|
38
|
+
localBindHost: "127.0.0.1",
|
|
39
|
+
localPort: 9090,
|
|
40
|
+
segmentStore: store
|
|
41
|
+
});
|
|
42
|
+
return { manager, store, root };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* A session shaped like a live one, sharing an output with any other built
|
|
47
|
+
* from the same key.
|
|
48
|
+
*
|
|
49
|
+
* @param {{ id: string, dirPath: string, outputKey: string }} params
|
|
50
|
+
* @returns {object}
|
|
51
|
+
*/
|
|
52
|
+
function sessionOn({ id, dirPath, outputKey }) {
|
|
53
|
+
return {
|
|
54
|
+
id,
|
|
55
|
+
outputKey,
|
|
56
|
+
dirPath,
|
|
57
|
+
state: "ready",
|
|
58
|
+
file: new SourceFile({ sourceKey: "source-1", fileIndex: 0, name: "video.mkv" }),
|
|
59
|
+
// An ordinary session reads its own file, and its sound is inside it. The
|
|
60
|
+
// three differ only for a soundtrack shipped as a file of its own.
|
|
61
|
+
get inputFile() { return this.file; },
|
|
62
|
+
get audioFile() { return this.file; },
|
|
63
|
+
segmentFormat: fmp4Format,
|
|
64
|
+
// Where the file is cut, which every live session holds.
|
|
65
|
+
timeline: new Timeline({ boundaries: [0, 4, 8], cutGrid: "keyframe" }),
|
|
66
|
+
consumers: new Set(),
|
|
67
|
+
lastAccessedAt: Date.now()
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
test("a second session of the same output is given the same directory", (t) => {
|
|
72
|
+
const { store, root } = managerOverATempStore();
|
|
73
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
74
|
+
|
|
75
|
+
// Two viewers who opened one film ten minutes apart. Their sessions differ —
|
|
76
|
+
// the start position is still what keeps their encoders apart — but what they
|
|
77
|
+
// produce is the same material.
|
|
78
|
+
assert.equal(store.pathFor(OUTPUT_KEY), store.pathFor(OUTPUT_KEY));
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test("each session serves the segments the other one's encoder made", (t) => {
|
|
82
|
+
const { manager, store, root } = managerOverATempStore();
|
|
83
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
84
|
+
|
|
85
|
+
const dirPath = store.directoryFor(OUTPUT_KEY);
|
|
86
|
+
store.useFormat(OUTPUT_KEY, fmp4Format);
|
|
87
|
+
|
|
88
|
+
// Two runs writing into the ONE directory this output has. They are kept
|
|
89
|
+
// apart by the stretches they were given, not by a directory each, so neither
|
|
90
|
+
// can reach the other's numbers and there is nothing to separate.
|
|
91
|
+
writeFileSync(path.join(dirPath, "segment-00000.mp4"), Buffer.alloc(32, 1));
|
|
92
|
+
writeFileSync(path.join(dirPath, "segment-00100.mp4"), Buffer.alloc(32, 2));
|
|
93
|
+
|
|
94
|
+
const viewerOne = sessionOn({ id: "s-one", dirPath, outputKey: OUTPUT_KEY });
|
|
95
|
+
const viewerTwo = sessionOn({ id: "s-two", dirPath, outputKey: OUTPUT_KEY });
|
|
96
|
+
manager.sessionsById.set(viewerOne.id, viewerOne);
|
|
97
|
+
manager.sessionsById.set(viewerTwo.id, viewerTwo);
|
|
98
|
+
|
|
99
|
+
const sorted = (numbers) => [...numbers].sort((left, right) => left - right);
|
|
100
|
+
const heldByOne = sorted(manager.producedSegmentNumbers(viewerOne));
|
|
101
|
+
const heldByTwo = sorted(manager.producedSegmentNumbers(viewerTwo));
|
|
102
|
+
|
|
103
|
+
assert.deepEqual(heldByOne, [0, 100]);
|
|
104
|
+
assert.deepEqual(heldByTwo, [0, 100], "neither viewer's session is the owner of either run");
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("a session leaving does not take the segments with it", async (t) => {
|
|
108
|
+
const { manager, store, root } = managerOverATempStore();
|
|
109
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
110
|
+
|
|
111
|
+
const dirPath = store.directoryFor(OUTPUT_KEY);
|
|
112
|
+
store.useFormat(OUTPUT_KEY, fmp4Format);
|
|
113
|
+
writeFileSync(path.join(dirPath, "segment-00000.mp4"), Buffer.alloc(32, 1));
|
|
114
|
+
|
|
115
|
+
const viewerOne = sessionOn({ id: "s-one", dirPath, outputKey: OUTPUT_KEY });
|
|
116
|
+
const viewerTwo = sessionOn({ id: "s-two", dirPath, outputKey: OUTPUT_KEY });
|
|
117
|
+
manager.sessionsById.set(viewerOne.id, viewerOne);
|
|
118
|
+
manager.sessionsById.set(viewerTwo.id, viewerTwo);
|
|
119
|
+
|
|
120
|
+
await manager.disposeSession(viewerOne.id);
|
|
121
|
+
await manager.disposeSession(viewerTwo.id);
|
|
122
|
+
|
|
123
|
+
// Both sessions are gone and the material is still here. That is the whole
|
|
124
|
+
// decoupling: a session ending says nothing about whether anybody will ask
|
|
125
|
+
// for these segments again — the viewer who left may come back, and a viewer
|
|
126
|
+
// who never had a session here may open the same film in a minute.
|
|
127
|
+
assert.notEqual(store.pathOf(OUTPUT_KEY, 0), null);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("what nobody has read for long enough goes, and time is the only judge", (t) => {
|
|
131
|
+
const root = mkdtempSync(path.join(os.tmpdir(), "shared-segments-"));
|
|
132
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
133
|
+
let clock = 1_000_000;
|
|
134
|
+
const store = new SegmentStore({ root, now: () => clock });
|
|
135
|
+
|
|
136
|
+
const dirPath = store.directoryFor(OUTPUT_KEY);
|
|
137
|
+
store.useFormat(OUTPUT_KEY, fmp4Format);
|
|
138
|
+
writeFileSync(path.join(dirPath, "segment-00000.mp4"), Buffer.alloc(32, 1));
|
|
139
|
+
|
|
140
|
+
clock += 60_000;
|
|
141
|
+
assert.deepEqual(
|
|
142
|
+
store.enforce({ idleMs: 3_600_000, maxBytes: 1e9 }),
|
|
143
|
+
{ droppedIdle: 0, droppedForRoom: 0, bytes: 32 }
|
|
144
|
+
);
|
|
145
|
+
|
|
146
|
+
clock += 7_200_000;
|
|
147
|
+
const swept = store.enforce({ idleMs: 3_600_000, maxBytes: 1e9 });
|
|
148
|
+
assert.equal(swept.droppedIdle, 1);
|
|
149
|
+
assert.equal(store.pathOf(OUTPUT_KEY, 0), null);
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
test("over the allowance, what was read longest ago goes first", (t) => {
|
|
153
|
+
const root = mkdtempSync(path.join(os.tmpdir(), "shared-segments-"));
|
|
154
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
155
|
+
let clock = 1_000_000;
|
|
156
|
+
const store = new SegmentStore({ root, now: () => clock });
|
|
157
|
+
|
|
158
|
+
const older = OUTPUT_KEY;
|
|
159
|
+
const newer = `${OUTPUT_KEY}:second`;
|
|
160
|
+
for (const key of [older, newer]) {
|
|
161
|
+
const dir = store.directoryFor(key);
|
|
162
|
+
store.useFormat(key, fmp4Format);
|
|
163
|
+
writeFileSync(path.join(dir, "segment-00000.mp4"), Buffer.alloc(1000, 1));
|
|
164
|
+
clock += 1000;
|
|
165
|
+
}
|
|
166
|
+
// Reading the newer one again is what puts the older one at the front of the
|
|
167
|
+
// queue: the judge is the last REQUEST, not when the file was written.
|
|
168
|
+
clock += 1000;
|
|
169
|
+
store.pathOf(newer, 0);
|
|
170
|
+
|
|
171
|
+
const swept = store.enforce({ idleMs: 3_600_000, maxBytes: 1500 });
|
|
172
|
+
assert.equal(swept.droppedForRoom, 1);
|
|
173
|
+
assert.equal(store.pathOf(older, 0), null);
|
|
174
|
+
assert.notEqual(store.pathOf(newer, 0), null);
|
|
175
|
+
});
|