@torrent-tv/proxy 2.77.0 → 2.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/biome.json +1 -0
- package/package.json +1 -1
- package/routes/api/transcode-sessions/net-report/post.js +5 -0
- package/services/encode/EncodePlan.js +309 -272
- package/services/encode/EncodeRun.js +70 -0
- package/services/encode/SegmentDemand.js +0 -0
- package/services/encode/SegmentStore.js +59 -0
- package/services/encode/open-piece.js +88 -0
- package/services/encode/run-budget.js +89 -0
- package/services/encode/run-command.js +15 -0
- package/services/encode/run-costs.js +94 -0
- package/services/hls-session-manager.js +10657 -10846
- package/services/orchestrators/EncodeOrchestrator.js +147 -20
- package/services/priority/PriorityMap.js +252 -0
- package/services/viewer/Viewer.js +83 -0
- package/test/encode-orchestrator.test.js +36 -8
- package/test/encode-plan.test.js +111 -7
- package/test/one-authority.test.js +105 -0
- package/test/priority-map.test.js +139 -0
- package/test/run-intervals.test.js +67 -303
|
@@ -1,336 +1,100 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @file
|
|
2
|
+
* @file How far a run may work, and who decides it.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* These cases used to be asked of `planRunInterval`, a second authority inside
|
|
5
|
+
* the session manager that answered by its own rules: it walked the whole track
|
|
6
|
+
* for the first free number, MOVED the start there, and counted every live run
|
|
7
|
+
* as claiming up to `head + look-ahead`. It contradicted the plan directly, and
|
|
8
|
+
* the two together produced the field oscillation of 2026-09-05 — the plan
|
|
9
|
+
* commanded a start at #46, this moved it to #78, the plan killed the run for
|
|
10
|
+
* standing outside the window it had asked for, and the same start was
|
|
11
|
+
* commanded again, 350-700ms per cycle, no segment ever produced, the viewer's
|
|
12
|
+
* picture stopped for 125 seconds.
|
|
9
13
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
14
|
+
* It is gone. WHERE a run starts is the plan's decision and nothing moves it;
|
|
15
|
+
* HOW FAR it may work is a fact of the one coverage map, and that is what these
|
|
16
|
+
* cases now ask. The map is the same object the plan reads, so there is no
|
|
17
|
+
* second set of rules for the two to disagree about.
|
|
13
18
|
*/
|
|
14
19
|
|
|
15
20
|
import test from "node:test";
|
|
16
21
|
import assert from "node:assert/strict";
|
|
17
|
-
import {
|
|
18
|
-
import { startRunOn } from "./helpers/encode-run.js";
|
|
19
|
-
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
20
|
-
import os from "node:os";
|
|
21
|
-
import path from "node:path";
|
|
22
|
-
import { HlsSessionManager } from "../services/hls-session-manager.js";
|
|
23
|
-
import { SegmentStore } from "../services/encode/SegmentStore.js";
|
|
24
|
-
import { Timeline } from "../services/output/Timeline.js";
|
|
25
|
-
import { fmp4Format } from "../services/segment-formats/fmp4.js";
|
|
26
|
-
import { viewerOf } from "../services/viewer/Viewer.js";
|
|
22
|
+
import { CoverageMap } from "../services/encode/CoverageMap.js";
|
|
27
23
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
* @returns {{ manager: HlsSessionManager, store: SegmentStore, root: string, dirPath: string }}
|
|
32
|
-
*/
|
|
33
|
-
function managerWithAnEmptyOutput() {
|
|
34
|
-
const root = mkdtempSync(path.join(os.tmpdir(), "run-intervals-"));
|
|
35
|
-
const store = new SegmentStore({ root });
|
|
36
|
-
const manager = new HlsSessionManager({
|
|
37
|
-
enabled: true,
|
|
38
|
-
ffmpegBin: "ffmpeg",
|
|
39
|
-
localBindHost: "127.0.0.1",
|
|
40
|
-
localPort: 9090,
|
|
41
|
-
segmentStore: store
|
|
42
|
-
});
|
|
43
|
-
const dirPath = store.directoryFor(KEY);
|
|
44
|
-
store.useFormat(KEY, fmp4Format);
|
|
45
|
-
return { manager, store, root, dirPath };
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* @param {object} params
|
|
50
|
-
* @returns {object}
|
|
51
|
-
*/
|
|
52
|
-
function sessionOn({ id, dirPath, segmentCount = 100, runState = null, encodeStartIndex = 0, runEndIndex = -1 }) {
|
|
53
|
-
const session = {
|
|
54
|
-
id,
|
|
55
|
-
outputKey: KEY,
|
|
56
|
-
dirPath,
|
|
57
|
-
state: "ready",
|
|
58
|
-
file: new SourceFile({ sourceKey: "source-1", fileIndex: 0, name: "video.mkv" }),
|
|
59
|
-
// An ordinary session reads its own file, and its sound is inside it. The
|
|
60
|
-
// three differ only for a soundtrack shipped as a file of its own.
|
|
61
|
-
get inputFile() { return this.file; },
|
|
62
|
-
get audioFile() { return this.file; },
|
|
63
|
-
segmentFormat: fmp4Format,
|
|
64
|
-
// How the file is cut, held by the TIMELINE. A fixture that stated it on the
|
|
65
|
-
// session was describing a shape the product had left, and it kept a defect
|
|
66
|
-
// alive for a release: `session.segmentCount` is undefined on every real
|
|
67
|
-
// session, so runs were given no end and the coverage map had no length.
|
|
68
|
-
timeline: new Timeline({
|
|
69
|
-
boundaries: Array.from({ length: segmentCount + 1 }, (_, index) => index * 4),
|
|
70
|
-
cutGrid: "uniform"
|
|
71
|
-
}),
|
|
72
|
-
runs: new Set(),
|
|
73
|
-
consumers: new Set(),
|
|
74
|
-
lastAccessedAt: Date.now()
|
|
75
|
-
};
|
|
76
|
-
if (runState !== null) {
|
|
77
|
-
const run = startRunOn(session, { from: encodeStartIndex, to: runEndIndex });
|
|
78
|
-
if (runState === "ENDED_FAILED") {
|
|
79
|
-
// A run that failed, said so, and left what it had made behind.
|
|
80
|
-
run.process?.exit(255, null);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
return session;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* @param {string} dirPath
|
|
88
|
-
* @param {number[]} indexes
|
|
89
|
-
*/
|
|
90
|
-
function alreadyMade(dirPath, indexes) {
|
|
91
|
-
for (const index of indexes) {
|
|
92
|
-
writeFileSync(path.join(dirPath, fmp4Format.segmentFileName(index)), Buffer.alloc(16, 1));
|
|
93
|
-
}
|
|
24
|
+
/** A stand-in for a run: the map identifies one by being it. */
|
|
25
|
+
function run(name) {
|
|
26
|
+
return { name };
|
|
94
27
|
}
|
|
95
28
|
|
|
96
|
-
test("with nothing made, a run gets everything from where it was asked", (
|
|
97
|
-
const
|
|
98
|
-
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
99
|
-
|
|
100
|
-
const session = sessionOn({ id: "s", dirPath });
|
|
101
|
-
// -1 for the end means "to the end of the film", which is what every run had
|
|
102
|
-
// before ends existed — and the only case where that is still right.
|
|
103
|
-
assert.deepEqual(manager.planRunInterval(session, 0), { from: 0, to: -1 });
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
test("a run stops before material that is already made", (t) => {
|
|
107
|
-
const { manager, root, dirPath } = managerWithAnEmptyOutput();
|
|
108
|
-
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
109
|
-
|
|
110
|
-
// 10..20 on the disk. Only 10..19 are PROVEN — the highest has no successor,
|
|
111
|
-
// so nothing shows whether it was closed or was being written when its run
|
|
112
|
-
// died.
|
|
113
|
-
alreadyMade(dirPath, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]);
|
|
114
|
-
const session = sessionOn({ id: "s", dirPath });
|
|
115
|
-
|
|
116
|
-
assert.deepEqual(manager.planRunInterval(session, 0), { from: 0, to: 9 });
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
test("a run asked to start inside made material is moved forward to the gap", (t) => {
|
|
120
|
-
const { manager, root, dirPath } = managerWithAnEmptyOutput();
|
|
121
|
-
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
122
|
-
|
|
123
|
-
alreadyMade(dirPath, [10, 11, 12, 13, 14, 15]);
|
|
124
|
-
const session = sessionOn({ id: "s", dirPath });
|
|
125
|
-
|
|
126
|
-
// Asked for 11, which is made, and so is everything to 14. The first thing
|
|
127
|
-
// worth encoding is 15 — unproven, because 16 does not exist.
|
|
128
|
-
assert.deepEqual(manager.planRunInterval(session, 11), { from: 15, to: -1 });
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
test("a run stops before a stretch another live run was given", (t) => {
|
|
132
|
-
const { manager, root, dirPath } = managerWithAnEmptyOutput();
|
|
133
|
-
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
134
|
-
|
|
135
|
-
const other = sessionOn({
|
|
136
|
-
id: "other",
|
|
137
|
-
dirPath,
|
|
138
|
-
runState: "PRODUCING",
|
|
139
|
-
encodeStartIndex: 40,
|
|
140
|
-
runEndIndex: 60
|
|
141
|
-
});
|
|
142
|
-
manager.sessionsById.set(other.id, other);
|
|
143
|
-
const session = sessionOn({ id: "s", dirPath });
|
|
29
|
+
test("with nothing made, a run gets everything from where it was asked", () => {
|
|
30
|
+
const coverage = new CoverageMap({ segmentCount: 100 });
|
|
144
31
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
// neither ever writes a name the other wants.
|
|
148
|
-
assert.deepEqual(manager.planRunInterval(session, 0), { from: 0, to: 39 });
|
|
149
|
-
assert.deepEqual(manager.planRunInterval(session, 45), { from: 61, to: -1 });
|
|
32
|
+
assert.equal(coverage.freeRunFrom(0), 100, "the whole track is free");
|
|
33
|
+
assert.equal(coverage.firstGapFrom(0), 0);
|
|
150
34
|
});
|
|
151
35
|
|
|
152
|
-
test("a run
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
// The same stretch, but the process cannot be signalled — it has ended. What
|
|
157
|
-
// it did not finish is free again, without anything having to release it.
|
|
158
|
-
const dead = sessionOn({
|
|
159
|
-
id: "dead",
|
|
160
|
-
dirPath,
|
|
161
|
-
runState: "ENDED_FAILED",
|
|
162
|
-
encodeStartIndex: 40,
|
|
163
|
-
runEndIndex: 60
|
|
164
|
-
});
|
|
165
|
-
manager.sessionsById.set(dead.id, dead);
|
|
166
|
-
const session = sessionOn({ id: "s", dirPath });
|
|
36
|
+
test("a run stops before material that is already made", () => {
|
|
37
|
+
const coverage = new CoverageMap({ segmentCount: 100 });
|
|
38
|
+
coverage.markReadyAll([10, 11, 12, 13, 14, 15, 16, 17, 18, 19]);
|
|
167
39
|
|
|
168
|
-
assert.
|
|
40
|
+
assert.equal(coverage.freeRunFrom(0), 10, "0..9, and it stops where 10 begins");
|
|
169
41
|
});
|
|
170
42
|
|
|
171
|
-
test("
|
|
172
|
-
const
|
|
173
|
-
|
|
43
|
+
test("a run stops before a stretch another live run was given", () => {
|
|
44
|
+
const coverage = new CoverageMap({ segmentCount: 100 });
|
|
45
|
+
const other = run("other");
|
|
46
|
+
coverage.claim(other, 40, 60);
|
|
174
47
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
const other = sessionOn({
|
|
178
|
-
id: "other",
|
|
179
|
-
dirPath,
|
|
180
|
-
segmentCount: 5,
|
|
181
|
-
runState: "PRODUCING",
|
|
182
|
-
encodeStartIndex: 4,
|
|
183
|
-
runEndIndex: 4
|
|
184
|
-
});
|
|
185
|
-
manager.sessionsById.set(other.id, other);
|
|
186
|
-
|
|
187
|
-
// Starting an encoder here would only repeat somebody else's work, which is
|
|
188
|
-
// what three ffmpeg processes making one identical picture cost on a CM4.
|
|
189
|
-
assert.equal(manager.planRunInterval(session, 0), null);
|
|
48
|
+
assert.equal(coverage.freeRunFrom(0), 40, "0..39, and the other run's claim begins at 40");
|
|
49
|
+
assert.equal(coverage.firstGapFrom(45), 61, "the first thing nobody holds after it");
|
|
190
50
|
});
|
|
191
51
|
|
|
192
|
-
test("a
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
const session = sessionOn({ id: "s", dirPath });
|
|
197
|
-
session.outputKey = "";
|
|
198
|
-
|
|
199
|
-
assert.deepEqual(manager.planRunInterval(session, 7), { from: 7, to: -1 });
|
|
200
|
-
});
|
|
201
|
-
|
|
202
|
-
test("a run without an end does not claim the whole film away from a later viewer", (t) => {
|
|
203
|
-
const { manager, root, dirPath } = managerWithAnEmptyOutput();
|
|
204
|
-
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
205
|
-
|
|
206
|
-
// The first viewer's run, at the beginning, with no end — which is every run
|
|
207
|
-
// whose output was empty when it started.
|
|
208
|
-
const first = sessionOn({
|
|
209
|
-
id: "first",
|
|
210
|
-
dirPath,
|
|
211
|
-
segmentCount: 1000,
|
|
212
|
-
runState: "PRODUCING",
|
|
213
|
-
encodeStartIndex: 0,
|
|
214
|
-
runEndIndex: -1
|
|
215
|
-
});
|
|
216
|
-
manager.sessionsById.set(first.id, first);
|
|
217
|
-
const second = sessionOn({ id: "second", dirPath, segmentCount: 1000 });
|
|
218
|
-
|
|
219
|
-
// A second viewer opens the same film in the middle. Counting the first run's
|
|
220
|
-
// claim as the whole film would leave them with no encoder at all, waiting for
|
|
221
|
-
// it to encode its way there — an hour on a long film. It only claims as far
|
|
222
|
-
// as it will actually get, which is its head plus the look-ahead.
|
|
223
|
-
const planned = manager.planRunInterval(second, 500);
|
|
224
|
-
assert.notEqual(planned, null);
|
|
225
|
-
assert.equal(planned.from, 500);
|
|
226
|
-
});
|
|
52
|
+
test("a run's own claim does not hold it back", () => {
|
|
53
|
+
const coverage = new CoverageMap({ segmentCount: 100 });
|
|
54
|
+
const mine = run("mine");
|
|
55
|
+
coverage.claim(mine, 40, 60);
|
|
227
56
|
|
|
228
|
-
test("a run stops when it reaches a stretch another run was given", async (t) => {
|
|
229
|
-
const { manager, root, dirPath } = managerWithAnEmptyOutput();
|
|
230
|
-
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
231
|
-
|
|
232
|
-
const stopped = [];
|
|
233
|
-
const ahead = sessionOn({
|
|
234
|
-
id: "ahead",
|
|
235
|
-
dirPath,
|
|
236
|
-
runState: "PRODUCING",
|
|
237
|
-
encodeStartIndex: 500,
|
|
238
|
-
runEndIndex: 600
|
|
239
|
-
});
|
|
240
|
-
manager.sessionsById.set(ahead.id, ahead);
|
|
241
|
-
const behind = sessionOn({
|
|
242
|
-
id: "behind",
|
|
243
|
-
dirPath,
|
|
244
|
-
runState: "PRODUCING",
|
|
245
|
-
encodeStartIndex: 0,
|
|
246
|
-
runEndIndex: -1
|
|
247
|
-
});
|
|
248
|
-
manager.sessionsById.set(behind.id, behind);
|
|
249
|
-
|
|
250
|
-
// Its own end was set from the gaps of the moment it began, and the viewer who
|
|
251
|
-
// opened the film further on was not in that picture. Walking into their
|
|
252
|
-
// stretch means writing names they are writing.
|
|
253
|
-
const behindRun = [...behind.runs][0];
|
|
254
|
-
const aheadRun = [...ahead.runs][0];
|
|
255
|
-
assert.equal(manager.runMakingSegment(behind, 550, behindRun), aheadRun);
|
|
256
|
-
assert.equal(manager.runMakingSegment(behind, 499, behindRun), null);
|
|
257
57
|
assert.equal(
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
"
|
|
58
|
+
coverage.freeRunFrom(40, mine),
|
|
59
|
+
60,
|
|
60
|
+
"asked by the run that holds it, the stretch is its own to work through"
|
|
261
61
|
);
|
|
262
|
-
void stopped;
|
|
263
62
|
});
|
|
264
63
|
|
|
265
|
-
test("a
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
id: "shared",
|
|
271
|
-
dirPath,
|
|
272
|
-
segmentCount: 1000,
|
|
273
|
-
runState: "PRODUCING",
|
|
274
|
-
encodeStartIndex: 200,
|
|
275
|
-
runEndIndex: -1
|
|
276
|
-
});
|
|
277
|
-
shared.sourceKey = "torrent:abc";
|
|
278
|
-
shared.fileIndex = 0;
|
|
279
|
-
shared.timeline = new Timeline({ boundaries: Array.from({ length: 1001 }, (_, index) => index * 4), cutGrid: "uniform" });
|
|
280
|
-
shared.acquireSource = null;
|
|
281
|
-
shared.progress = { processedSeconds: 900, startPositionSeconds: 800 };
|
|
282
|
-
manager.sessionsById.set(shared.id, shared);
|
|
64
|
+
test("a run that has ended holds nothing back", () => {
|
|
65
|
+
const coverage = new CoverageMap({ segmentCount: 100 });
|
|
66
|
+
const dead = run("dead");
|
|
67
|
+
coverage.claim(dead, 40, 60);
|
|
68
|
+
coverage.release(dead);
|
|
283
69
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
viewerOf(shared, "back").position = { segment: 250, seconds: 1000, at: Date.now() };
|
|
70
|
+
assert.equal(coverage.freeRunFrom(0), 100, "what it did not finish is free again");
|
|
71
|
+
});
|
|
287
72
|
|
|
288
|
-
|
|
289
|
-
const
|
|
290
|
-
|
|
73
|
+
test("what a dead run DID finish stays made", () => {
|
|
74
|
+
const coverage = new CoverageMap({ segmentCount: 100 });
|
|
75
|
+
const dead = run("dead");
|
|
76
|
+
coverage.claim(dead, 40, 60);
|
|
77
|
+
coverage.markReadyAll([40, 41, 42]);
|
|
78
|
+
coverage.release(dead);
|
|
291
79
|
|
|
292
|
-
assert.equal(
|
|
293
|
-
servingAhead.from,
|
|
294
|
-
startedAt,
|
|
295
|
-
"the run serving the viewer in front is left exactly where it was"
|
|
296
|
-
);
|
|
297
|
-
assert.ok(shared.runs.has(servingAhead), "and it is not stopped");
|
|
298
|
-
// A run start claims its attempt before it awaits anything, so the assertion
|
|
299
|
-
// holds without the test needing an ffmpeg. 40 s is segment #10 on this grid.
|
|
300
|
-
assert.equal(
|
|
301
|
-
shared.pendingRun?.startIndex,
|
|
302
|
-
10,
|
|
303
|
-
"and the one who jumped is given a run of their own, there"
|
|
304
|
-
);
|
|
80
|
+
assert.equal(coverage.firstGapFrom(40), 43, "a closed file is closed whoever made it");
|
|
305
81
|
});
|
|
306
82
|
|
|
307
|
-
test("
|
|
308
|
-
const
|
|
309
|
-
|
|
83
|
+
test("nothing left to make is answered with nothing", () => {
|
|
84
|
+
const coverage = new CoverageMap({ segmentCount: 5 });
|
|
85
|
+
coverage.markReadyAll([0, 1, 2, 3, 4]);
|
|
310
86
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
created += 1;
|
|
314
|
-
return null;
|
|
315
|
-
};
|
|
316
|
-
|
|
317
|
-
const alone = sessionOn({
|
|
318
|
-
id: "alone",
|
|
319
|
-
dirPath,
|
|
320
|
-
segmentCount: 1000,
|
|
321
|
-
runState: "PRODUCING",
|
|
322
|
-
encodeStartIndex: 200,
|
|
323
|
-
runEndIndex: -1
|
|
324
|
-
});
|
|
325
|
-
alone.timeline = new Timeline({ boundaries: Array.from({ length: 1001 }, (_, index) => index * 4), cutGrid: "uniform" });
|
|
326
|
-
alone.progress = { processedSeconds: 900, startPositionSeconds: 800 };
|
|
327
|
-
manager.sessionsById.set(alone.id, alone);
|
|
328
|
-
viewerOf(alone, "only").position = { segment: 250, seconds: 1000, at: Date.now() };
|
|
87
|
+
assert.equal(coverage.firstGapFrom(0), null, "and a run started here would only repeat somebody's work");
|
|
88
|
+
});
|
|
329
89
|
|
|
330
|
-
|
|
90
|
+
test("a run without an end does not take the film away from a viewer further in", () => {
|
|
91
|
+
// The case that used to need a second authority's `head + look-ahead` guess.
|
|
92
|
+
// A run's claim is now the stretch the plan GAVE it, so a viewer opening the
|
|
93
|
+
// same film in the middle finds their own position free.
|
|
94
|
+
const coverage = new CoverageMap({ segmentCount: 1000 });
|
|
95
|
+
const first = run("first");
|
|
96
|
+
coverage.claim(first, 0, 499);
|
|
331
97
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
assert.equal(created, 0);
|
|
335
|
-
assert.equal(alone.seekTarget !== undefined, true, "the run itself is repositioned, as before");
|
|
98
|
+
assert.equal(coverage.firstGapFrom(500), 500, "the second viewer's own position is free");
|
|
99
|
+
assert.equal(coverage.freeRunFrom(500), 500, "and everything from there to the end");
|
|
336
100
|
});
|