@torrent-tv/proxy 2.80.18 → 2.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/docs/encode-architecture.md +51 -2
- package/package.json +1 -1
- package/research/double-spawn-2026-09-10.md +171 -0
- package/services/disk/DiskSpace.js +146 -0
- package/services/disk/wire.js +60 -0
- package/services/encode/EncodeRun.js +37 -9
- package/services/encode/SegmentStore.js +284 -232
- package/services/encode/run-command.js +16 -1
- package/services/hls-session-manager.js +31 -128
- package/services/orchestrators/EncodeOrchestrator.js +52 -38
- package/services/piece-store/allowance.js +107 -0
- package/services/piece-store/piece-disk-store.js +365 -0
- package/services/piece-store/shared-piece-store.js +1549 -1535
- package/services/segment-formats/fmp4.js +54 -0
- package/services/segment-formats/mpegts.js +54 -0
- package/services/torrent-worker/client.js +32 -0
- package/services/torrent-worker/pool-adapter.js +15 -0
- package/services/torrent-worker/protocol.js +9 -0
- package/services/torrent-worker/worker.js +8 -1
- package/services/viewer/positions.js +48 -0
- package/test/audio-inventory.test.js +176 -176
- package/test/auto-quality-step.test.js +514 -514
- package/test/concurrent-cost.test.js +138 -138
- package/test/coverage-follows-the-disk.test.js +191 -187
- package/test/coverage-map.test.js +195 -195
- package/test/declared-tracks.test.js +35 -35
- package/test/disk-space.test.js +138 -0
- package/test/encode-orchestrator.test.js +0 -3
- package/test/encode-run.test.js +5 -12
- package/test/held-request-width.test.js +155 -155
- package/test/helpers/encode-run.js +2 -2
- package/test/matroska-blocks.test.js +0 -0
- package/test/matroska-cues-track.test.js +192 -192
- package/test/mp4-composition-times.test.js +0 -0
- package/test/mp4-subtitles.test.js +173 -173
- package/test/one-authority.test.js +281 -220
- package/test/orchestrator-wired.test.js +199 -195
- package/test/packet-witness-ring.test.js +236 -236
- package/test/packet-witness.test.js +148 -148
- package/test/piece-disk-store.test.js +267 -0
- package/test/piece-reader.test.js +4 -4
- package/test/piece-store-eviction.test.js +17 -17
- package/test/piece-store-reservations.test.js +20 -1
- package/test/piece-store-slow-disk.test.js +16 -1
- package/test/produced-copy-choice.test.js +258 -358
- package/test/read-window.test.js +6 -6
- package/test/run-intervals.test.js +100 -100
- package/test/seek-landing.test.js +109 -109
- package/test/segment-serve-wiring.test.js +8 -9
- package/test/segment-store-eviction.test.js +232 -0
- package/test/segment-store.test.js +238 -216
- package/test/segments-are-shared.test.js +1 -1
- package/test/shared-piece-store.test.js +12 -12
- package/test/sidecar-naming.test.js +142 -142
- package/test/subtitle-cue-framing.test.js +200 -200
- package/test/subtitle-cue-walk.test.js +369 -369
- package/test/subtitle-defaults.test.js +97 -97
- package/test/subtitle-track-numbering.test.js +370 -370
- package/test/tail-duplication.test.js +167 -167
- package/test/tracks-begin-together.test.js +195 -195
- package/test/two-viewers-one-picture.test.js +374 -374
- package/test/video-facts.test.js +102 -102
- package/test/wedge-certainty.test.js +131 -131
- package/services/encode/open-piece.js +0 -135
- package/services/piece-store/disk-tier.js +0 -151
- package/test/open-piece.test.js +0 -152
|
@@ -1,187 +1,191 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file What the plan believes exists is what the disk holds — with a real
|
|
3
|
-
* store, a real directory, and no ffmpeg.
|
|
4
|
-
*
|
|
5
|
-
* The failure these are written against, field 2026-09-07, twice in one evening
|
|
6
|
-
* on `Star.Trek.Strange.New.Worlds.S04E03`:
|
|
7
|
-
*
|
|
8
|
-
* > `encode: … ready=482 zones=[p100:#0..#0 …] runs=[#0..#0@0.0x] waiting=nobody`
|
|
9
|
-
* > `encode-run stopped #0..#0 …, reached #-1 (0 segment(s)) after 2778ms,`
|
|
10
|
-
* > `signal SIGTERM: the film is no worse off without it`
|
|
11
|
-
* > `encode: … ready=482 … runs=[] waiting=nobody`
|
|
12
|
-
*
|
|
13
|
-
* The map claimed every one of the film's 482 segments while the directory held
|
|
14
|
-
* nothing a header could be lifted out of, so every arrangement scored perfect,
|
|
15
|
-
* the only encoder was taken away as unnecessary and none was placed again. The
|
|
16
|
-
* viewer's browser asked for the init segment, was never answered, and gave up.
|
|
17
|
-
*
|
|
18
|
-
* Readiness had been accumulating for the life of the process: the map was told
|
|
19
|
-
* what existed and never told what had stopped existing — the word for that
|
|
20
|
-
* existed and was called from no line of the product. So these check the one
|
|
21
|
-
* property that makes the whole class of failure impossible: what the map says
|
|
22
|
-
* is ready is what the store can prove right now, and nothing older.
|
|
23
|
-
*/
|
|
24
|
-
|
|
25
|
-
import test from "node:test";
|
|
26
|
-
import assert from "node:assert/strict";
|
|
27
|
-
import { EventEmitter } from "node:events";
|
|
28
|
-
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
29
|
-
import os from "node:os";
|
|
30
|
-
import path from "node:path";
|
|
31
|
-
import { EncodeRun } from "../services/encode/EncodeRun.js";
|
|
32
|
-
import { SegmentStore } from "../services/encode/SegmentStore.js";
|
|
33
|
-
import { SoftwareEncoder } from "../services/encode/SoftwareEncoder.js";
|
|
34
|
-
import { EncodeOrchestrator } from "../services/orchestrators/EncodeOrchestrator.js";
|
|
35
|
-
import { fmp4Format } from "../services/segment-formats/fmp4.js";
|
|
36
|
-
|
|
37
|
-
const PICTURE = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
|
|
38
|
-
const SEGMENTS = 482;
|
|
39
|
-
|
|
40
|
-
class FakeProcess extends EventEmitter {
|
|
41
|
-
constructor() {
|
|
42
|
-
super();
|
|
43
|
-
this.pid = 1;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
kill(signal) {
|
|
47
|
-
this.emit("exit", null, signal);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* A store on a real directory, and an orchestrator that owns it.
|
|
53
|
-
*
|
|
54
|
-
* @returns {{ made: EncodeOrchestrator, store: SegmentStore, dir: string,
|
|
55
|
-
* root: string, lines: string[] }}
|
|
56
|
-
*/
|
|
57
|
-
function orchestratorWithAStore() {
|
|
58
|
-
const root = mkdtempSync(path.join(os.tmpdir(), "coverage-disk-"));
|
|
59
|
-
const lines = [];
|
|
60
|
-
const logger = { info: (line) => lines.push(line), warn: (line) => lines.push(line) };
|
|
61
|
-
const store = new SegmentStore({ root, logger });
|
|
62
|
-
store.useFormat(PICTURE, fmp4Format);
|
|
63
|
-
const dir = store.directoryFor(PICTURE);
|
|
64
|
-
/** @type {EncodeOrchestrator} */
|
|
65
|
-
let made;
|
|
66
|
-
made = new EncodeOrchestrator({
|
|
67
|
-
maxRunsFor: () => 1,
|
|
68
|
-
segmentSeconds: 6,
|
|
69
|
-
killCostSec: 0,
|
|
70
|
-
firstByteWaitSec: 0.12,
|
|
71
|
-
refetchSecPerFilmSecond: () => 0.25,
|
|
72
|
-
startingSpeedFor: () => 2,
|
|
73
|
-
segmentStore: store,
|
|
74
|
-
now: () => 1000,
|
|
75
|
-
logger,
|
|
76
|
-
makeRun: ({ address, from, to }) => new EncodeRun({
|
|
77
|
-
address,
|
|
78
|
-
encoder: new SoftwareEncoder(),
|
|
79
|
-
from,
|
|
80
|
-
to,
|
|
81
|
-
buildArgs: () => ["-i", "in", "out"],
|
|
82
|
-
spawn: () => new FakeProcess(),
|
|
83
|
-
logger,
|
|
84
|
-
now: () => 1000,
|
|
85
|
-
onEnded: (ended) => made.noteEnded(ended)
|
|
86
|
-
})
|
|
87
|
-
});
|
|
88
|
-
made.setSegmentCount(PICTURE, SEGMENTS);
|
|
89
|
-
return { made, store, dir, root, lines };
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
/** A viewer stopped at the beginning, which is what a fresh session states. */
|
|
93
|
-
function aViewerAtTheStart(made) {
|
|
94
|
-
made.notePriorityMap(PICTURE, [
|
|
95
|
-
{ from: 0, to: 0, priority: 100, withinSeconds: 0 },
|
|
96
|
-
{ from: 1, to: SEGMENTS - 1, priority: 91, withinSeconds: 6 }
|
|
97
|
-
]);
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Every segment of the film, written and reported closed — what the store looked
|
|
102
|
-
* like while the film was being watched.
|
|
103
|
-
*/
|
|
104
|
-
function theWholeFilmIsOnDisk(store, dir) {
|
|
105
|
-
for (let index = 0; index < SEGMENTS; index += 1) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
//
|
|
176
|
-
//
|
|
177
|
-
made
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
assert.
|
|
186
|
-
|
|
187
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @file What the plan believes exists is what the disk holds — with a real
|
|
3
|
+
* store, a real directory, and no ffmpeg.
|
|
4
|
+
*
|
|
5
|
+
* The failure these are written against, field 2026-09-07, twice in one evening
|
|
6
|
+
* on `Star.Trek.Strange.New.Worlds.S04E03`:
|
|
7
|
+
*
|
|
8
|
+
* > `encode: … ready=482 zones=[p100:#0..#0 …] runs=[#0..#0@0.0x] waiting=nobody`
|
|
9
|
+
* > `encode-run stopped #0..#0 …, reached #-1 (0 segment(s)) after 2778ms,`
|
|
10
|
+
* > `signal SIGTERM: the film is no worse off without it`
|
|
11
|
+
* > `encode: … ready=482 … runs=[] waiting=nobody`
|
|
12
|
+
*
|
|
13
|
+
* The map claimed every one of the film's 482 segments while the directory held
|
|
14
|
+
* nothing a header could be lifted out of, so every arrangement scored perfect,
|
|
15
|
+
* the only encoder was taken away as unnecessary and none was placed again. The
|
|
16
|
+
* viewer's browser asked for the init segment, was never answered, and gave up.
|
|
17
|
+
*
|
|
18
|
+
* Readiness had been accumulating for the life of the process: the map was told
|
|
19
|
+
* what existed and never told what had stopped existing — the word for that
|
|
20
|
+
* existed and was called from no line of the product. So these check the one
|
|
21
|
+
* property that makes the whole class of failure impossible: what the map says
|
|
22
|
+
* is ready is what the store can prove right now, and nothing older.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import test from "node:test";
|
|
26
|
+
import assert from "node:assert/strict";
|
|
27
|
+
import { EventEmitter } from "node:events";
|
|
28
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
29
|
+
import os from "node:os";
|
|
30
|
+
import path from "node:path";
|
|
31
|
+
import { EncodeRun } from "../services/encode/EncodeRun.js";
|
|
32
|
+
import { SegmentStore } from "../services/encode/SegmentStore.js";
|
|
33
|
+
import { SoftwareEncoder } from "../services/encode/SoftwareEncoder.js";
|
|
34
|
+
import { EncodeOrchestrator } from "../services/orchestrators/EncodeOrchestrator.js";
|
|
35
|
+
import { fmp4Format } from "../services/segment-formats/fmp4.js";
|
|
36
|
+
|
|
37
|
+
const PICTURE = "torrent:abc:fmt=fmp4:grid=kf@0:video-only:v=0/copy";
|
|
38
|
+
const SEGMENTS = 482;
|
|
39
|
+
|
|
40
|
+
class FakeProcess extends EventEmitter {
|
|
41
|
+
constructor() {
|
|
42
|
+
super();
|
|
43
|
+
this.pid = 1;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
kill(signal) {
|
|
47
|
+
this.emit("exit", null, signal);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* A store on a real directory, and an orchestrator that owns it.
|
|
53
|
+
*
|
|
54
|
+
* @returns {{ made: EncodeOrchestrator, store: SegmentStore, dir: string,
|
|
55
|
+
* root: string, lines: string[] }}
|
|
56
|
+
*/
|
|
57
|
+
function orchestratorWithAStore() {
|
|
58
|
+
const root = mkdtempSync(path.join(os.tmpdir(), "coverage-disk-"));
|
|
59
|
+
const lines = [];
|
|
60
|
+
const logger = { info: (line) => lines.push(line), warn: (line) => lines.push(line) };
|
|
61
|
+
const store = new SegmentStore({ root, logger });
|
|
62
|
+
store.useFormat(PICTURE, fmp4Format);
|
|
63
|
+
const dir = store.directoryFor(PICTURE);
|
|
64
|
+
/** @type {EncodeOrchestrator} */
|
|
65
|
+
let made;
|
|
66
|
+
made = new EncodeOrchestrator({
|
|
67
|
+
maxRunsFor: () => 1,
|
|
68
|
+
segmentSeconds: 6,
|
|
69
|
+
killCostSec: 0,
|
|
70
|
+
firstByteWaitSec: 0.12,
|
|
71
|
+
refetchSecPerFilmSecond: () => 0.25,
|
|
72
|
+
startingSpeedFor: () => 2,
|
|
73
|
+
segmentStore: store,
|
|
74
|
+
now: () => 1000,
|
|
75
|
+
logger,
|
|
76
|
+
makeRun: ({ address, from, to }) => new EncodeRun({
|
|
77
|
+
address,
|
|
78
|
+
encoder: new SoftwareEncoder(),
|
|
79
|
+
from,
|
|
80
|
+
to,
|
|
81
|
+
buildArgs: () => ["-i", "in", "out"],
|
|
82
|
+
spawn: () => new FakeProcess(),
|
|
83
|
+
logger,
|
|
84
|
+
now: () => 1000,
|
|
85
|
+
onEnded: (ended) => made.noteEnded(ended)
|
|
86
|
+
})
|
|
87
|
+
});
|
|
88
|
+
made.setSegmentCount(PICTURE, SEGMENTS);
|
|
89
|
+
return { made, store, dir, root, lines };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** A viewer stopped at the beginning, which is what a fresh session states. */
|
|
93
|
+
function aViewerAtTheStart(made) {
|
|
94
|
+
made.notePriorityMap(PICTURE, [
|
|
95
|
+
{ from: 0, to: 0, priority: 100, withinSeconds: 0 },
|
|
96
|
+
{ from: 1, to: SEGMENTS - 1, priority: 91, withinSeconds: 6 }
|
|
97
|
+
]);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Every segment of the film, written and reported closed — what the store looked
|
|
102
|
+
* like while the film was being watched.
|
|
103
|
+
*/
|
|
104
|
+
function theWholeFilmIsOnDisk(store, dir) {
|
|
105
|
+
for (let index = 0; index < SEGMENTS; index += 1) {
|
|
106
|
+
// Under the served name, which is the whole of what says a piece is closed:
|
|
107
|
+
// it takes that name only when its writer has said so. There used to be a
|
|
108
|
+
// statement kept beside the disk as well, told by whoever noticed a piece
|
|
109
|
+
// being produced — a second owner of one fact, and the fault this file's own
|
|
110
|
+
// subject is.
|
|
111
|
+
writeFileSync(path.join(dir, fmp4Format.segmentFileName(index)), Buffer.alloc(16));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
test("a film that is entirely made needs no encoder", (t) => {
|
|
116
|
+
const { made, store, dir, root } = orchestratorWithAStore();
|
|
117
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
118
|
+
|
|
119
|
+
theWholeFilmIsOnDisk(store, dir);
|
|
120
|
+
aViewerAtTheStart(made);
|
|
121
|
+
made.reconcile();
|
|
122
|
+
|
|
123
|
+
// The other half of the property, and the reason the failure looked
|
|
124
|
+
// reasonable: when the film really is all there, taking the encoder away IS
|
|
125
|
+
// right, and the viewer is served from the disk.
|
|
126
|
+
assert.equal(made.runsOn(PICTURE).length, 0, "nothing left to make");
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("segments the store has lost stop being ready, and an encoder is placed again", (t) => {
|
|
130
|
+
const { made, store, dir, root } = orchestratorWithAStore();
|
|
131
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
132
|
+
|
|
133
|
+
theWholeFilmIsOnDisk(store, dir);
|
|
134
|
+
aViewerAtTheStart(made);
|
|
135
|
+
made.reconcile();
|
|
136
|
+
assert.equal(made.runsOn(PICTURE).length, 0);
|
|
137
|
+
|
|
138
|
+
// THE FIELD FAILURE. The output's directory goes — dropped for room, or by the
|
|
139
|
+
// idle rule, or because a killed process left it and the sweep cleared it. The
|
|
140
|
+
// map used to keep all 482 numbers and the plan went on believing the film was
|
|
141
|
+
// made, for the life of the process.
|
|
142
|
+
store.drop(PICTURE, "the test is taking its files away");
|
|
143
|
+
store.useFormat(PICTURE, fmp4Format);
|
|
144
|
+
store.directoryFor(PICTURE);
|
|
145
|
+
|
|
146
|
+
made.reconcile();
|
|
147
|
+
const runs = made.runsOn(PICTURE);
|
|
148
|
+
assert.equal(runs.length, 1, "the film is unmade again, so somebody makes it");
|
|
149
|
+
assert.equal(runs[0].from, 0, "beginning where the viewer is stopped");
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
test("a piece discarded with the run that had it open is not ready either", (t) => {
|
|
153
|
+
const { made, store, dir, root } = orchestratorWithAStore();
|
|
154
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
155
|
+
|
|
156
|
+
theWholeFilmIsOnDisk(store, dir);
|
|
157
|
+
aViewerAtTheStart(made);
|
|
158
|
+
made.reconcile();
|
|
159
|
+
assert.equal(made.runsOn(PICTURE).length, 0);
|
|
160
|
+
|
|
161
|
+
// A run stopped the instant after opening #0 leaves a file of no bytes under a
|
|
162
|
+
// name that reads as a segment; the store throws it away. Field 2026-09-07,
|
|
163
|
+
// 20:13:13 and 20:13:57 — printed twice while the map went on saying 482.
|
|
164
|
+
rmSync(path.join(dir, fmp4Format.segmentFileName(0)), { force: true });
|
|
165
|
+
|
|
166
|
+
made.reconcile();
|
|
167
|
+
assert.equal(made.runsOn(PICTURE).length, 1, "one number missing is one encoder");
|
|
168
|
+
assert.equal(made.runsOn(PICTURE)[0].from, 0);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("nobody making what somebody waits for is said out loud", (t) => {
|
|
172
|
+
const { made, root, lines } = orchestratorWithAStore();
|
|
173
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
174
|
+
|
|
175
|
+
// A HOST THAT CANNOT BUILD AN ENCODER. Whatever the reason — and there are
|
|
176
|
+
// several — the state it leaves is the one that cost two sessions on
|
|
177
|
+
// 2026-09-07: somebody is waiting, nothing is being made, and every line
|
|
178
|
+
// above reads as a healthy proxy. It ran for 41 seconds in the field with no
|
|
179
|
+
// word about it on either side, and ended at the browser's own timeout with a
|
|
180
|
+
// message naming no cause.
|
|
181
|
+
made.makeRun = () => null;
|
|
182
|
+
aViewerAtTheStart(made);
|
|
183
|
+
made.reconcile();
|
|
184
|
+
|
|
185
|
+
assert.equal(made.runsOn(PICTURE).length, 0);
|
|
186
|
+
const said = lines.find((line) => line.includes("NO ENCODER IS MAKING IT"));
|
|
187
|
+
assert.ok(said, "the state has a line of its own");
|
|
188
|
+
assert.ok(said.includes("#0"), "which number is wanted");
|
|
189
|
+
assert.ok(said.includes(PICTURE), "of which output, whole — not cut to sixty characters");
|
|
190
|
+
assert.ok(/files=\d+/.test(said), "and what the disk holds, beside what is proven");
|
|
191
|
+
});
|