@torrent-tv/proxy 2.80.19 → 2.81.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/package.json +1 -1
  3. package/research/double-spawn-2026-09-10.md +171 -0
  4. package/services/disk/DiskSpace.js +146 -0
  5. package/services/disk/wire.js +60 -0
  6. package/services/encode/EncodeRun.js +25 -3
  7. package/services/encode/SegmentStore.js +137 -9
  8. package/services/hls-session-manager.js +26 -45
  9. package/services/orchestrators/EncodeOrchestrator.js +4 -1
  10. package/services/piece-store/allowance.js +107 -0
  11. package/services/piece-store/piece-disk-store.js +365 -0
  12. package/services/piece-store/shared-piece-store.js +1549 -1535
  13. package/services/torrent-worker/client.js +32 -0
  14. package/services/torrent-worker/pool-adapter.js +15 -0
  15. package/services/torrent-worker/protocol.js +9 -0
  16. package/services/torrent-worker/worker.js +8 -1
  17. package/services/viewer/positions.js +48 -0
  18. package/test/audio-inventory.test.js +176 -176
  19. package/test/auto-quality-step.test.js +514 -514
  20. package/test/concurrent-cost.test.js +138 -138
  21. package/test/coverage-follows-the-disk.test.js +191 -191
  22. package/test/coverage-map.test.js +195 -195
  23. package/test/declared-tracks.test.js +35 -35
  24. package/test/disk-space.test.js +138 -0
  25. package/test/encode-orchestrator.test.js +0 -3
  26. package/test/encode-run.test.js +5 -12
  27. package/test/held-request-width.test.js +155 -155
  28. package/test/helpers/encode-run.js +2 -2
  29. package/test/matroska-blocks.test.js +0 -0
  30. package/test/matroska-cues-track.test.js +192 -192
  31. package/test/mp4-composition-times.test.js +0 -0
  32. package/test/mp4-subtitles.test.js +173 -173
  33. package/test/one-authority.test.js +281 -220
  34. package/test/orchestrator-wired.test.js +199 -199
  35. package/test/packet-witness-ring.test.js +236 -236
  36. package/test/packet-witness.test.js +148 -148
  37. package/test/piece-disk-store.test.js +267 -0
  38. package/test/piece-reader.test.js +4 -4
  39. package/test/piece-store-eviction.test.js +17 -17
  40. package/test/piece-store-reservations.test.js +20 -1
  41. package/test/piece-store-slow-disk.test.js +16 -1
  42. package/test/read-window.test.js +6 -6
  43. package/test/run-intervals.test.js +100 -100
  44. package/test/seek-landing.test.js +109 -109
  45. package/test/segment-store-eviction.test.js +232 -0
  46. package/test/segments-are-shared.test.js +1 -1
  47. package/test/shared-piece-store.test.js +12 -12
  48. package/test/sidecar-naming.test.js +142 -142
  49. package/test/subtitle-cue-framing.test.js +200 -200
  50. package/test/subtitle-cue-walk.test.js +369 -369
  51. package/test/subtitle-defaults.test.js +97 -97
  52. package/test/subtitle-track-numbering.test.js +370 -370
  53. package/test/tail-duplication.test.js +167 -167
  54. package/test/tracks-begin-together.test.js +195 -195
  55. package/test/two-viewers-one-picture.test.js +374 -374
  56. package/test/video-facts.test.js +102 -102
  57. package/test/wedge-certainty.test.js +131 -131
  58. package/services/piece-store/disk-tier.js +0 -151
@@ -1,199 +1,199 @@
1
- /**
2
- * @file The plan is what decides the encoders, and it is asked of real sessions.
3
- *
4
- * Three rules written into the session manager become one here: where a run
5
- * belongs, when it has been overtaken, and how many the machine affords. The
6
- * point of wiring rather than duplicating is that the arithmetic has sixteen
7
- * checks of its own and no ffmpeg behind it.
8
- */
9
-
10
- import test from "node:test";
11
- import assert from "node:assert/strict";
12
- import { SourceFile } from "../services/source/SourceFile.js";
13
- import { Timeline } from "../services/output/Timeline.js";
14
- import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
15
- import os from "node:os";
16
- import path from "node:path";
17
- import { HlsSessionManager } from "../services/hls-session-manager.js";
18
- import { SegmentStore } from "../services/encode/SegmentStore.js";
19
- import { fmp4Format } from "../services/segment-formats/fmp4.js";
20
- import { viewerOf } from "../services/viewer/Viewer.js";
21
- import { startRunOn } from "./helpers/encode-run.js";
22
-
23
- const KEY = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
24
-
25
- function managerWithAnOutput() {
26
- const root = mkdtempSync(path.join(os.tmpdir(), "orchestrator-wired-"));
27
- const store = new SegmentStore({ root });
28
- const manager = new HlsSessionManager({
29
- enabled: true,
30
- ffmpegBin: "ffmpeg",
31
- localBindHost: "127.0.0.1",
32
- localPort: 9090,
33
- segmentStore: store
34
- });
35
- const dirPath = store.directoryFor(KEY);
36
- store.useFormat(KEY, fmp4Format);
37
- return { manager, store, root, dirPath };
38
- }
39
-
40
- function sessionOn({ id, dirPath, encodeStartIndex = 0, runEndIndex = -1, speed = 0, running = true }) {
41
- const session = {
42
- id,
43
- outputKey: KEY,
44
- dirPath,
45
- // Where this file is cut, held by the file. A fixture that stated it
46
- // on the session was describing what production no longer does.
47
- timeline: new Timeline({
48
- boundaries: Array.from({ length: 1001 }, (_, index) => index * 4),
49
- cutGrid: "uniform"
50
- }),
51
- state: "ready",
52
- file: new SourceFile({ sourceKey: "source-1", fileIndex: 0, name: "video.mkv" }).learn({ durationSeconds: 4000 }),
53
- // An ordinary session reads its own file, and its sound is inside it. The
54
- // three differ only for a soundtrack shipped as a file of its own.
55
- get inputFile() { return this.file; },
56
- get audioFile() { return this.file; },
57
- segmentFormat: fmp4Format,
58
- segmentCount: 1000,
59
- recentSpeed: null,
60
- consumers: new Set(),
61
- viewers: new Map(),
62
- runs: new Set(),
63
- lastAccessedAt: Date.now()
64
- };
65
- if (running) {
66
- const run = startRunOn(session, { from: encodeStartIndex, to: runEndIndex });
67
- // A speed is a reading taken FROM a run, so it names the run it came from.
68
- session.recentSpeed = speed > 0 ? { speed, at: Date.now(), run } : null;
69
- run.noteSpeed(speed);
70
- }
71
- return session;
72
- }
73
-
74
- test("a session is handed to the plan as the run it is", (t) => {
75
- const { manager, root, dirPath } = managerWithAnOutput();
76
- t.after(() => rmSync(root, { recursive: true, force: true }));
77
-
78
- const session = sessionOn({ id: "one", dirPath, encodeStartIndex: 10, runEndIndex: 40 });
79
- manager.sessionsById.set(session.id, session);
80
- viewerOf(session, "watching").position = { segment: 12, seconds: 48, at: Date.now() };
81
-
82
- manager.runQualityBudgetOnce;
83
- manager.planEncodersNow();
84
-
85
- const runs = manager.encodeOrchestrator.runsOn(KEY);
86
- assert.equal(runs.length, 1, "the session the browser already has is a run like any other");
87
- assert.equal(runs[0], [...session.runs][0], "and it is the run the session actually holds");
88
- assert.equal(runs[0].from, 10);
89
- assert.equal(runs[0].to, 40);
90
- });
91
-
92
- test("what a viewer waits for reaches the plan without their name", (t) => {
93
- const { manager, root, dirPath } = managerWithAnOutput();
94
- t.after(() => rmSync(root, { recursive: true, force: true }));
95
-
96
- const session = sessionOn({ id: "one", dirPath, encodeStartIndex: 0, runEndIndex: -1 });
97
- manager.sessionsById.set(session.id, session);
98
- viewerOf(session, "someone").position = { segment: 5, seconds: 20, at: Date.now() };
99
-
100
- manager.planEncodersNow();
101
-
102
- // THE MAP IS BANDS NOW, not one window per viewer: what a viewer is stopped
103
- // on, then what is in front of them band by band, then the rest of the track.
104
- // The check is the same question asked of that shape — where the most urgent
105
- // band begins, and that there is film in front of it.
106
- const wanted = manager.encodeOrchestrator.demand.mapOn(KEY);
107
- const first = [...wanted].sort((left, right) => right.priority - left.priority)[0];
108
- assert.equal(first.from, 5, "where they are");
109
- assert.ok(wanted.some((zone) => zone.to > 5), "and the cushion in front of them");
110
- });
111
-
112
- test("a viewer nothing has been heard from at all stops being waited for", (t) => {
113
- const { manager, root, dirPath } = managerWithAnOutput();
114
- t.after(() => rmSync(root, { recursive: true, force: true }));
115
-
116
- const session = sessionOn({ id: "one", dirPath });
117
- manager.sessionsById.set(session.id, session);
118
- const gone = viewerOf(session, "gone");
119
- gone.position = { segment: 5, seconds: 20, at: Date.now() - 10 * 60 * 1000 };
120
- // What decides it is when they were last HEARD FROM, not when they last
121
- // asked for a segment: a viewer holding a full cushion asks for nothing for
122
- // as long as it takes to drain, and is no less present for it.
123
- gone.seen(Date.now() - 10 * 60 * 1000);
124
-
125
- manager.planEncodersNow();
126
-
127
- assert.equal(manager.encodeOrchestrator.demand.mapOn(KEY).length, 0);
128
- });
129
-
130
- test("a viewer who has arrived and asked for nothing is waited for", (t) => {
131
- const { manager, root, dirPath } = managerWithAnOutput();
132
- t.after(() => rmSync(root, { recursive: true, force: true }));
133
-
134
- const session = sessionOn({ id: "one", dirPath });
135
- manager.sessionsById.set(session.id, session);
136
- // Nothing places them here on purpose: this is a viewer known to the output
137
- // and nothing more, which is what a viewer IS between arriving and asking
138
- // for their first file. Answered from the position alone, this viewer read
139
- // as absent, and on 2026-09-05 that stopped a soundtrack's encoder 1.25 s
140
- // after it started, with nothing produced.
141
- const fresh = manager.viewers.of(session, "fresh");
142
- assert.equal(fresh.position, null, "nobody has placed them");
143
-
144
- manager.planEncodersNow();
145
-
146
- const wanted = manager.encodeOrchestrator.demand.mapOn(KEY);
147
- assert.ok(wanted.length > 0, "an output with a viewer on it is wanted");
148
- const first = [...wanted].sort((left, right) => right.priority - left.priority)[0];
149
- assert.equal(first.from, 0, "and the beginning is where an unplaced viewer is");
150
- });
151
-
152
- test("how many encoders the machine affords is measured, not chosen", (t) => {
153
- const { manager, root, dirPath } = managerWithAnOutput();
154
- t.after(() => rmSync(root, { recursive: true, force: true }));
155
-
156
- // Nothing measured yet: one is what it has.
157
- const cold = sessionOn({ id: "cold", dirPath });
158
- manager.sessionsById.set(cold.id, cold);
159
- assert.equal(manager.maxRunsForOutput(KEY), 1);
160
-
161
- // Fast, but what a second job costs on THIS machine has not been measured,
162
- // and an unmeasured penalty of 1 is not a statement that it is free.
163
- cold.lastAloneSpeed = 7.12;
164
- assert.equal(manager.maxRunsForOutput(KEY), 1, "no measurement, no second encoder");
165
-
166
- // Measured on the addon host 2026-09-03: at 854x480 one run made 7.12x and
167
- // two made 4.20x and 4.16x, a penalty of 1.70x, and both stayed far above
168
- // realtime.
169
- manager.contentionPenalties = new Map([[1, 1.7]]);
170
- assert.ok(manager.maxRunsForOutput(KEY) > 1, "measured, and a second fits");
171
-
172
- // The same host at 1920x1080: one made 1.96x, two made 0.99x and 0.98x.
173
- manager.contentionPenalties = new Map([[1, 1.98]]);
174
- cold.lastAloneSpeed = 1.96;
175
- assert.equal(manager.maxRunsForOutput(KEY), 1, "the machine is full at one");
176
- });
177
-
178
- test("segments already made are known to the plan, whoever made them", (t) => {
179
- const { manager, store, root, dirPath } = managerWithAnOutput();
180
- t.after(() => rmSync(root, { recursive: true, force: true }));
181
-
182
- for (const index of [0, 1, 2, 3]) {
183
- writeFileSync(path.join(dirPath, fmp4Format.segmentFileName(index)), Buffer.alloc(16, 1));
184
- }
185
- const session = sessionOn({ id: "one", dirPath });
186
- manager.sessionsById.set(session.id, session);
187
-
188
- manager.planEncodersNow();
189
-
190
- const coverage = manager.encodeOrchestrator.coverageOf(KEY);
191
- assert.equal(coverage.isReady(0), true);
192
- assert.equal(coverage.isReady(2), true);
193
- // Including the highest. It used to be excluded for having no successor to
194
- // prove it closed, which left the last piece of every run unprovable for ever
195
- // — and proved a half-written one whenever the number above it was written by
196
- // another run of the same output. A served name is the proof now.
197
- assert.equal(coverage.isReady(3), true, "its name says it is closed");
198
- void store;
199
- });
1
+ /**
2
+ * @file The plan is what decides the encoders, and it is asked of real sessions.
3
+ *
4
+ * Three rules written into the session manager become one here: where a run
5
+ * belongs, when it has been overtaken, and how many the machine affords. The
6
+ * point of wiring rather than duplicating is that the arithmetic has sixteen
7
+ * checks of its own and no ffmpeg behind it.
8
+ */
9
+
10
+ import test from "node:test";
11
+ import assert from "node:assert/strict";
12
+ import { SourceFile } from "../services/source/SourceFile.js";
13
+ import { Timeline } from "../services/output/Timeline.js";
14
+ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
15
+ import os from "node:os";
16
+ import path from "node:path";
17
+ import { HlsSessionManager } from "../services/hls-session-manager.js";
18
+ import { SegmentStore } from "../services/encode/SegmentStore.js";
19
+ import { fmp4Format } from "../services/segment-formats/fmp4.js";
20
+ import { viewerOf } from "../services/viewer/Viewer.js";
21
+ import { startRunOn } from "./helpers/encode-run.js";
22
+
23
+ const KEY = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
24
+
25
+ function managerWithAnOutput() {
26
+ const root = mkdtempSync(path.join(os.tmpdir(), "orchestrator-wired-"));
27
+ const store = new SegmentStore({ root });
28
+ const manager = new HlsSessionManager({
29
+ enabled: true,
30
+ ffmpegBin: "ffmpeg",
31
+ localBindHost: "127.0.0.1",
32
+ localPort: 9090,
33
+ segmentStore: store
34
+ });
35
+ const dirPath = store.directoryFor(KEY);
36
+ store.useFormat(KEY, fmp4Format);
37
+ return { manager, store, root, dirPath };
38
+ }
39
+
40
+ function sessionOn({ id, dirPath, encodeStartIndex = 0, runEndIndex = -1, speed = 0, running = true }) {
41
+ const session = {
42
+ id,
43
+ outputKey: KEY,
44
+ dirPath,
45
+ // Where this file is cut, held by the file. A fixture that stated it
46
+ // on the session was describing what production no longer does.
47
+ timeline: new Timeline({
48
+ boundaries: Array.from({ length: 1001 }, (_, index) => index * 4),
49
+ cutGrid: "uniform"
50
+ }),
51
+ state: "ready",
52
+ file: new SourceFile({ sourceKey: "source-1", fileIndex: 0, name: "video.mkv" }).learn({ durationSeconds: 4000 }),
53
+ // An ordinary session reads its own file, and its sound is inside it. The
54
+ // three differ only for a soundtrack shipped as a file of its own.
55
+ get inputFile() { return this.file; },
56
+ get audioFile() { return this.file; },
57
+ segmentFormat: fmp4Format,
58
+ segmentCount: 1000,
59
+ recentSpeed: null,
60
+ consumers: new Set(),
61
+ viewers: new Map(),
62
+ runs: new Set(),
63
+ lastAccessedAt: Date.now()
64
+ };
65
+ if (running) {
66
+ const run = startRunOn(session, { from: encodeStartIndex, to: runEndIndex });
67
+ // A speed is a reading taken FROM a run, so it names the run it came from.
68
+ session.recentSpeed = speed > 0 ? { speed, at: Date.now(), run } : null;
69
+ run.noteSpeed(speed);
70
+ }
71
+ return session;
72
+ }
73
+
74
+ test("a session is handed to the plan as the run it is", (t) => {
75
+ const { manager, root, dirPath } = managerWithAnOutput();
76
+ t.after(() => rmSync(root, { recursive: true, force: true }));
77
+
78
+ const session = sessionOn({ id: "one", dirPath, encodeStartIndex: 10, runEndIndex: 40 });
79
+ manager.sessionsById.set(session.id, session);
80
+ viewerOf(session, "watching").position = { segment: 12, seconds: 48, at: Date.now() };
81
+
82
+ manager.runQualityBudgetOnce;
83
+ manager.planEncodersNow();
84
+
85
+ const runs = manager.encodeOrchestrator.runsOn(KEY);
86
+ assert.equal(runs.length, 1, "the session the browser already has is a run like any other");
87
+ assert.equal(runs[0], [...session.runs][0], "and it is the run the session actually holds");
88
+ assert.equal(runs[0].from, 10);
89
+ assert.equal(runs[0].to, 40);
90
+ });
91
+
92
+ test("what a viewer waits for reaches the plan without their name", (t) => {
93
+ const { manager, root, dirPath } = managerWithAnOutput();
94
+ t.after(() => rmSync(root, { recursive: true, force: true }));
95
+
96
+ const session = sessionOn({ id: "one", dirPath, encodeStartIndex: 0, runEndIndex: -1 });
97
+ manager.sessionsById.set(session.id, session);
98
+ viewerOf(session, "someone").position = { segment: 5, seconds: 20, at: Date.now() };
99
+
100
+ manager.planEncodersNow();
101
+
102
+ // THE MAP IS BANDS NOW, not one window per viewer: what a viewer is stopped
103
+ // on, then what is in front of them band by band, then the rest of the track.
104
+ // The check is the same question asked of that shape — where the most urgent
105
+ // band begins, and that there is film in front of it.
106
+ const wanted = manager.encodeOrchestrator.demand.mapOn(KEY);
107
+ const first = [...wanted].sort((left, right) => right.priority - left.priority)[0];
108
+ assert.equal(first.from, 5, "where they are");
109
+ assert.ok(wanted.some((zone) => zone.to > 5), "and the cushion in front of them");
110
+ });
111
+
112
+ test("a viewer nothing has been heard from at all stops being waited for", (t) => {
113
+ const { manager, root, dirPath } = managerWithAnOutput();
114
+ t.after(() => rmSync(root, { recursive: true, force: true }));
115
+
116
+ const session = sessionOn({ id: "one", dirPath });
117
+ manager.sessionsById.set(session.id, session);
118
+ const gone = viewerOf(session, "gone");
119
+ gone.position = { segment: 5, seconds: 20, at: Date.now() - 10 * 60 * 1000 };
120
+ // What decides it is when they were last HEARD FROM, not when they last
121
+ // asked for a segment: a viewer holding a full cushion asks for nothing for
122
+ // as long as it takes to drain, and is no less present for it.
123
+ gone.seen(Date.now() - 10 * 60 * 1000);
124
+
125
+ manager.planEncodersNow();
126
+
127
+ assert.equal(manager.encodeOrchestrator.demand.mapOn(KEY).length, 0);
128
+ });
129
+
130
+ test("a viewer who has arrived and asked for nothing is waited for", (t) => {
131
+ const { manager, root, dirPath } = managerWithAnOutput();
132
+ t.after(() => rmSync(root, { recursive: true, force: true }));
133
+
134
+ const session = sessionOn({ id: "one", dirPath });
135
+ manager.sessionsById.set(session.id, session);
136
+ // Nothing places them here on purpose: this is a viewer known to the output
137
+ // and nothing more, which is what a viewer IS between arriving and asking
138
+ // for their first file. Answered from the position alone, this viewer read
139
+ // as absent, and on 2026-09-05 that stopped a soundtrack's encoder 1.25 s
140
+ // after it started, with nothing produced.
141
+ const fresh = manager.viewers.of(session, "fresh");
142
+ assert.equal(fresh.position, null, "nobody has placed them");
143
+
144
+ manager.planEncodersNow();
145
+
146
+ const wanted = manager.encodeOrchestrator.demand.mapOn(KEY);
147
+ assert.ok(wanted.length > 0, "an output with a viewer on it is wanted");
148
+ const first = [...wanted].sort((left, right) => right.priority - left.priority)[0];
149
+ assert.equal(first.from, 0, "and the beginning is where an unplaced viewer is");
150
+ });
151
+
152
+ test("how many encoders the machine affords is measured, not chosen", (t) => {
153
+ const { manager, root, dirPath } = managerWithAnOutput();
154
+ t.after(() => rmSync(root, { recursive: true, force: true }));
155
+
156
+ // Nothing measured yet: one is what it has.
157
+ const cold = sessionOn({ id: "cold", dirPath });
158
+ manager.sessionsById.set(cold.id, cold);
159
+ assert.equal(manager.maxRunsForOutput(KEY), 1);
160
+
161
+ // Fast, but what a second job costs on THIS machine has not been measured,
162
+ // and an unmeasured penalty of 1 is not a statement that it is free.
163
+ cold.lastAloneSpeed = 7.12;
164
+ assert.equal(manager.maxRunsForOutput(KEY), 1, "no measurement, no second encoder");
165
+
166
+ // Measured on the addon host 2026-09-03: at 854x480 one run made 7.12x and
167
+ // two made 4.20x and 4.16x, a penalty of 1.70x, and both stayed far above
168
+ // realtime.
169
+ manager.contentionPenalties = new Map([[1, 1.7]]);
170
+ assert.ok(manager.maxRunsForOutput(KEY) > 1, "measured, and a second fits");
171
+
172
+ // The same host at 1920x1080: one made 1.96x, two made 0.99x and 0.98x.
173
+ manager.contentionPenalties = new Map([[1, 1.98]]);
174
+ cold.lastAloneSpeed = 1.96;
175
+ assert.equal(manager.maxRunsForOutput(KEY), 1, "the machine is full at one");
176
+ });
177
+
178
+ test("segments already made are known to the plan, whoever made them", (t) => {
179
+ const { manager, store, root, dirPath } = managerWithAnOutput();
180
+ t.after(() => rmSync(root, { recursive: true, force: true }));
181
+
182
+ for (const index of [0, 1, 2, 3]) {
183
+ writeFileSync(path.join(dirPath, fmp4Format.segmentFileName(index)), Buffer.alloc(16, 1));
184
+ }
185
+ const session = sessionOn({ id: "one", dirPath });
186
+ manager.sessionsById.set(session.id, session);
187
+
188
+ manager.planEncodersNow();
189
+
190
+ const coverage = manager.encodeOrchestrator.coverageOf(KEY);
191
+ assert.equal(coverage.isReady(0), true);
192
+ assert.equal(coverage.isReady(2), true);
193
+ // Including the highest. It used to be excluded for having no successor to
194
+ // prove it closed, which left the last piece of every run unprovable for ever
195
+ // — and proved a half-written one whenever the number above it was written by
196
+ // another run of the same output. A served name is the proof now.
197
+ assert.equal(coverage.isReady(3), true, "its name says it is closed");
198
+ void store;
199
+ });