@torrent-tv/proxy 2.87.0 → 2.87.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/CLAUDE.md +8 -4
- package/docs/browser-proxy-contract.md +94 -0
- package/package.json +1 -1
- package/routes/api/transcode-sessions/post.js +1 -1
- package/routes/transcode/audio-file/get.js +1 -1
- package/routes/transcode/audio-warm/get.js +1 -1
- package/routes/transcode/session-file/get.js +6 -383
- package/routes/transcode/variant-file/get.js +2 -3
- package/routes/transcode/variant-warm/get.js +2 -1
- package/services/encode/EncodeOrchestrator.js +23 -13
- package/services/encode/EncodeRuns.js +15 -5
- package/services/encode/OutputOpening.js +16 -6
- package/services/encode/quality/QualityController.js +6 -2
- package/services/server/http-responses.js +60 -0
- package/services/server/transcode-session-files.js +325 -0
- package/services/server/wire-outputs.js +29 -4
- package/test/coverage-follows-the-disk.test.js +6 -1
- package/test/encoded-output.test.js +1 -1
|
@@ -33,6 +33,14 @@ import { contentionPenalty } from "./contention.js";
|
|
|
33
33
|
import { SegmentDemand } from "./SegmentDemand.js";
|
|
34
34
|
import { runStateOf } from "./encode-run-state.js";
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* @typedef {Object} SegmentCoverage
|
|
38
|
+
* @property {(address: string) => number[]} provenNumbers
|
|
39
|
+
* @property {(address: string, index: number) => void} announce
|
|
40
|
+
* @property {(address: string) => number} filesHeld
|
|
41
|
+
* @property {(address: string, startedAt: number) => number} clearUpAfter
|
|
42
|
+
*/
|
|
43
|
+
|
|
36
44
|
/**
|
|
37
45
|
* How long nothing is placed on an output whose input has just gone, and the
|
|
38
46
|
* ceiling that doubling reaches.
|
|
@@ -58,6 +66,9 @@ export class EncodeOrchestrator {
|
|
|
58
66
|
/** Output address to what has been made of it. @type {Map<string, CoverageMap>} */
|
|
59
67
|
#coverage = new Map();
|
|
60
68
|
|
|
69
|
+
/** @type {SegmentCoverage | null} */
|
|
70
|
+
#segmentCoverage = null;
|
|
71
|
+
|
|
61
72
|
/** Output address to the runs on it. @type {Map<string, import("../encode/EncodeRun.js").EncodeRun[]>} */
|
|
62
73
|
#runs = new Map();
|
|
63
74
|
|
|
@@ -148,6 +159,7 @@ export class EncodeOrchestrator {
|
|
|
148
159
|
* HANDED IN: what anybody waited for belongs to the priority layer, and
|
|
149
160
|
* this one may be asked about encoders with no ledger, no clock and no
|
|
150
161
|
* viewer anywhere. Absent, the line simply does not carry it.
|
|
162
|
+
* @param {SegmentCoverage | null} [params.segmentCoverage]
|
|
151
163
|
*/
|
|
152
164
|
constructor({
|
|
153
165
|
maxRunsFor,
|
|
@@ -156,7 +168,7 @@ export class EncodeOrchestrator {
|
|
|
156
168
|
contentionPenalties = null,
|
|
157
169
|
refetchSecPerFilmSecond = () => 0,
|
|
158
170
|
startingSpeedFor = () => 0,
|
|
159
|
-
|
|
171
|
+
segmentCoverage = null,
|
|
160
172
|
planSoon = null,
|
|
161
173
|
describeWaits = null,
|
|
162
174
|
admission = null,
|
|
@@ -169,11 +181,9 @@ export class EncodeOrchestrator {
|
|
|
169
181
|
// is about one output, and then only the per-output limit binds.
|
|
170
182
|
this.admission = admission && typeof admission.placesFor === "function" ? admission : null;
|
|
171
183
|
this.#describeWaits = typeof describeWaits === "function" ? describeWaits : () => "";
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
|
|
175
|
-
// under a name that promises a whole segment.
|
|
176
|
-
this.segmentStore = segmentStore;
|
|
184
|
+
// Only the segment facts and operations this planner needs cross the
|
|
185
|
+
// storage boundary; the concrete storage remains in the server assembly.
|
|
186
|
+
this.#segmentCoverage = segmentCoverage;
|
|
177
187
|
this.maxRunsFor = maxRunsFor;
|
|
178
188
|
// Seconds of swarm time per second of film: what re-encoding material that
|
|
179
189
|
// already exists costs the download, over and above the encoder's own time.
|
|
@@ -238,8 +248,8 @@ export class EncodeOrchestrator {
|
|
|
238
248
|
*/
|
|
239
249
|
#upToDateCoverage(address) {
|
|
240
250
|
const coverage = this.coverageOf(address);
|
|
241
|
-
if (this
|
|
242
|
-
coverage.setReady(this.
|
|
251
|
+
if (this.#segmentCoverage) {
|
|
252
|
+
coverage.setReady(this.#segmentCoverage.provenNumbers(address));
|
|
243
253
|
}
|
|
244
254
|
return coverage;
|
|
245
255
|
}
|
|
@@ -549,9 +559,9 @@ export class EncodeOrchestrator {
|
|
|
549
559
|
// Nothing is told to the store: by the time this is called the piece is
|
|
550
560
|
// ALREADY under its served name, because the rename is what closing it
|
|
551
561
|
// means. A statement kept beside the disk would be a second owner of one
|
|
552
|
-
// fact, which
|
|
562
|
+
// fact, which removed the stale second owner from the coverage map.
|
|
553
563
|
this.coverageOf(address).markReady(index);
|
|
554
|
-
this
|
|
564
|
+
this.#segmentCoverage?.announce?.(address, index);
|
|
555
565
|
for (const run of this.runsOn(address)) {
|
|
556
566
|
run.noteProduced(index);
|
|
557
567
|
}
|
|
@@ -805,7 +815,7 @@ export class EncodeOrchestrator {
|
|
|
805
815
|
this.#lastUnmet.delete(address);
|
|
806
816
|
} else if (this.#lastUnmet.get(address) !== wanting) {
|
|
807
817
|
this.#lastUnmet.set(address, wanting);
|
|
808
|
-
const held = this
|
|
818
|
+
const held = this.#segmentCoverage ? this.#segmentCoverage.filesHeld(address) : -1;
|
|
809
819
|
this.logger.warn(
|
|
810
820
|
`encode: #${wanting} of ${address} is wanted and NO ENCODER IS MAKING IT — ` +
|
|
811
821
|
`ready=${coverage.stats().ready} of ${coverage.segmentCount} ` +
|
|
@@ -1009,7 +1019,7 @@ export class EncodeOrchestrator {
|
|
|
1009
1019
|
// is a name match: no stretch to search and no bytes to judge. Done for
|
|
1010
1020
|
// every ending, the normal one included, since a run that finished cleanly
|
|
1011
1021
|
// has nothing under a working name and the sweep then removes nothing.
|
|
1012
|
-
this
|
|
1022
|
+
this.#segmentCoverage?.clearUpAfter(ended.address, ended.from);
|
|
1013
1023
|
this.coverageOf(ended.address).release(ended.run);
|
|
1014
1024
|
const remaining = this.runsOn(ended.address).filter((run) => run !== ended.run);
|
|
1015
1025
|
if (remaining.length === 0) {
|
|
@@ -1182,7 +1192,7 @@ export class EncodeOrchestrator {
|
|
|
1182
1192
|
// different statements: files with nothing proving them closed reads as a
|
|
1183
1193
|
// reporting fault, no files at all reads as an output yet to be made, and
|
|
1184
1194
|
// the difference decides where to look.
|
|
1185
|
-
const held = this
|
|
1195
|
+
const held = this.#segmentCoverage ? this.#segmentCoverage.filesHeld(address) : -1;
|
|
1186
1196
|
// AND WHETHER THE MAP IS BEING SERVED IN ITS OWN ORDER. The zones say
|
|
1187
1197
|
// what matters most; this says what the viewer actually waited for, by
|
|
1188
1198
|
// band. Waits at `now` are the ones that cost a spinner, and until this
|
|
@@ -15,6 +15,14 @@ import { ENCODE_EXIT } from "./encode-exit.js";
|
|
|
15
15
|
import { EncodeRun } from "./EncodeRun.js";
|
|
16
16
|
import { computeOutputDimensions } from "./args.js";
|
|
17
17
|
import { buildRunCommand, trueStartOf } from "./run-command.js";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @typedef {Object} SegmentFiles
|
|
21
|
+
* @property {(address: string) => string} pathFor
|
|
22
|
+
* @property {(address: string) => Buffer | null} initOf
|
|
23
|
+
* @property {(address: string) => string} directoryFor
|
|
24
|
+
* @property {(address: string, makingName: string, format: object) => string | null} publish
|
|
25
|
+
*/
|
|
18
26
|
// How far the accounting of a backward restart looks for work about to be done
|
|
19
27
|
// twice. It runs on the restart path and a session an hour in has thousands of
|
|
20
28
|
// segments; the figure is for a comparison, not an inventory.
|
|
@@ -147,7 +155,9 @@ export class EncodeRuns {
|
|
|
147
155
|
#planScheduled = false;
|
|
148
156
|
|
|
149
157
|
/**
|
|
150
|
-
* @param {object} host - `logger`, `viewerSecondsOn`, `noteRunStarting`, `inputOf`, `producedNumbers`, `servesAudioSeparately`, `disposeSession`, `contentionPenalties`, `encodeCost`, `encodeOrchestrator`, `encoders`, `ffmpegBin`, `outputTimes`, `outputs`, `priority`, `segmentDurationSec`, `
|
|
158
|
+
* @param {object} host - `logger`, `viewerSecondsOn`, `noteRunStarting`, `inputOf`, `producedNumbers`, `servesAudioSeparately`, `disposeSession`, `contentionPenalties`, `encodeCost`, `encodeOrchestrator`, `encoders`, `ffmpegBin`, `outputTimes`, `outputs`, `priority`, `segmentDurationSec`, `segmentFiles`, `videoEncoder`
|
|
159
|
+
* @param {SegmentFiles} host.segmentFiles - The storage operations used to
|
|
160
|
+
* write and inspect output files.
|
|
151
161
|
*/
|
|
152
162
|
constructor(host) {
|
|
153
163
|
this.#host = host;
|
|
@@ -542,7 +552,7 @@ export class EncodeRuns {
|
|
|
542
552
|
* @returns {void}
|
|
543
553
|
*/
|
|
544
554
|
noteRunProducedSegment(session, filePath) {
|
|
545
|
-
if (typeof filePath !== "string" || path.dirname(filePath) !== this.#host.
|
|
555
|
+
if (typeof filePath !== "string" || path.dirname(filePath) !== this.#host.segmentFiles.pathFor(session.outputKey ?? "")) {
|
|
546
556
|
return;
|
|
547
557
|
}
|
|
548
558
|
const index = session.segmentFormat.segmentIndexFromName(path.basename(filePath));
|
|
@@ -580,7 +590,7 @@ export class EncodeRuns {
|
|
|
580
590
|
* @returns {void}
|
|
581
591
|
*/
|
|
582
592
|
#warnIfRunLeavesTheInitBehind(session) {
|
|
583
|
-
const initBytes = this.#host.
|
|
593
|
+
const initBytes = this.#host.segmentFiles.initOf(session.outputKey ?? "");
|
|
584
594
|
if (!session.spec.transcodesVideo || !initBytes || initBytes.length === 0) {
|
|
585
595
|
return; // nothing served yet, or nothing being encoded
|
|
586
596
|
}
|
|
@@ -804,7 +814,7 @@ export class EncodeRuns {
|
|
|
804
814
|
totalSeconds: Number(session.file.durationSeconds) || null,
|
|
805
815
|
spawn: (spawnArgs) =>
|
|
806
816
|
spawn(this.#host.ffmpegBin, spawnArgs, {
|
|
807
|
-
cwd: this.#host.
|
|
817
|
+
cwd: this.#host.segmentFiles.directoryFor(session.outputKey ?? ""),
|
|
808
818
|
// A fourth channel: the encoder names every piece it has CLOSED on it,
|
|
809
819
|
// which is the only proof a piece is whole.
|
|
810
820
|
stdio: ["ignore", "pipe", "pipe", "pipe"]
|
|
@@ -821,7 +831,7 @@ export class EncodeRuns {
|
|
|
821
831
|
// Why this encoder exists, recorded with its argument list. It used to be
|
|
822
832
|
// handed to a separate `start` call; there is no separate call now.
|
|
823
833
|
because,
|
|
824
|
-
onClosed: (name) => this.#host.
|
|
834
|
+
onClosed: (name) => this.#host.segmentFiles.publish(session.outputKey ?? "", name, session.segmentFormat),
|
|
825
835
|
onEnded: (ended) => this.noteRunEnded(session, run, ended)
|
|
826
836
|
});
|
|
827
837
|
// THE ONE FAULT THAT IS OTHERWISE SILENT, asked before this run produces a
|
|
@@ -28,6 +28,14 @@ import { rungForVisiblePicture } from "./quality/visible-rung.js";
|
|
|
28
28
|
import { audioLoadOf, linkAnswerFigures as linkFiguresOf } from "./quality/link-budget.js";
|
|
29
29
|
import { AUDIO_TRANSCODE_KBPS } from "./args.js";
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* @typedef {Object} SegmentOutputFiles
|
|
33
|
+
* @property {() => string[]} addresses
|
|
34
|
+
* @property {(address: string, index: number) => boolean} isClosed
|
|
35
|
+
* @property {(address: string) => number | null} lastReadAt
|
|
36
|
+
* @property {(address: string) => string} directoryFor
|
|
37
|
+
* @property {(address: string, format: object) => void} useFormat
|
|
38
|
+
*/
|
|
31
39
|
|
|
32
40
|
/** Own package version, stamped onto output-start log lines. */
|
|
33
41
|
const PROXY_VERSION = createRequire(import.meta.url)("../../package.json").version;
|
|
@@ -37,7 +45,9 @@ export class OutputOpening {
|
|
|
37
45
|
#host;
|
|
38
46
|
|
|
39
47
|
/**
|
|
40
|
-
* @param {object} host - `logger`, `probeMediaInfo`, `probeKeyframeTimes`, `enabled`, `segmentFormat`, `renditions`, `sourceFiles`, `getCachedMediaInfo`, `localBaseUrl`, `tonemapSupported`, `videoEncoder`, `hostLoad`, `keyframeTables`, `timelines`, `segmentDurationSec`, `softwarePresetBenchmark`, `decodeCostModel`, `encodeCost`, `outputs`, `
|
|
48
|
+
* @param {object} host - `logger`, `probeMediaInfo`, `probeKeyframeTimes`, `enabled`, `segmentFormat`, `renditions`, `sourceFiles`, `getCachedMediaInfo`, `localBaseUrl`, `tonemapSupported`, `videoEncoder`, `hostLoad`, `keyframeTables`, `timelines`, `segmentDurationSec`, `softwarePresetBenchmark`, `decodeCostModel`, `encodeCost`, `outputs`, `segmentOutputFiles`, `hostTimings`, `returns`, `encodeRuns`
|
|
49
|
+
* @param {SegmentOutputFiles} host.segmentOutputFiles - The storage
|
|
50
|
+
* operations needed while selecting or opening an output.
|
|
41
51
|
*/
|
|
42
52
|
constructor(host) {
|
|
43
53
|
this.#host = host;
|
|
@@ -586,8 +596,8 @@ export class OutputOpening {
|
|
|
586
596
|
serving: {
|
|
587
597
|
mode: servingMode ?? (forceExactSize ? "manual" : "auto"),
|
|
588
598
|
linkMbps: viewerLinkMbps,
|
|
589
|
-
keys: [...this.#host.outputs.values()].map((other) => other.outputKey).concat(this.#host.
|
|
590
|
-
readyAt: (key) => this.#host.
|
|
599
|
+
keys: [...this.#host.outputs.values()].map((other) => other.outputKey).concat(this.#host.segmentOutputFiles.addresses()),
|
|
600
|
+
readyAt: (key) => this.#host.segmentOutputFiles.isClosed(key, timeline.indexForTime(Math.max(0, startPositionSeconds))),
|
|
591
601
|
observedPeakMbps: (candidate) => this.#host.observedPeakMbps?.(candidate) ?? null
|
|
592
602
|
}
|
|
593
603
|
});
|
|
@@ -679,9 +689,9 @@ export class OutputOpening {
|
|
|
679
689
|
// A RETURN, if this output was held before — and its age, which is the one
|
|
680
690
|
// term of the keeping period that nothing measures. Read BEFORE the
|
|
681
691
|
// directory is claimed, since claiming it is what marks it read.
|
|
682
|
-
this.#host.returns.note({ lastReadAt: this.#host.
|
|
683
|
-
this.#host.
|
|
684
|
-
this.#host.
|
|
692
|
+
this.#host.returns.note({ lastReadAt: this.#host.segmentOutputFiles.lastReadAt(spec.toKey()), now: Date.now() });
|
|
693
|
+
this.#host.segmentOutputFiles.directoryFor(spec.toKey());
|
|
694
|
+
this.#host.segmentOutputFiles.useFormat(spec.toKey(), segmentFormat);
|
|
685
695
|
this.#host.hostTimings.noteOutputCreated(session, createEntryMs);
|
|
686
696
|
this.#host.encodeRuns.setReadWindow(session, readWindowBytes);
|
|
687
697
|
this.#host.encodeCost.notePredictionFor(session, output.encodeHeight);
|
|
@@ -59,6 +59,8 @@ const BUDGET_DOWNLOAD_OK_FACTOR = 1.0;
|
|
|
59
59
|
// segments (the newest file on disk may still be written and is excluded).
|
|
60
60
|
const LINK_OBSERVED_SEGMENTS = 5;
|
|
61
61
|
|
|
62
|
+
/** @typedef {{ pathOf: (address: string, index: number) => string | null }} SegmentPathLookup */
|
|
63
|
+
|
|
62
64
|
export class QualityController {
|
|
63
65
|
/** What this reads and asks of the rest of the proxy, and nothing else. @type {object} */
|
|
64
66
|
#host;
|
|
@@ -70,7 +72,9 @@ export class QualityController {
|
|
|
70
72
|
#judging = new Set();
|
|
71
73
|
|
|
72
74
|
/**
|
|
73
|
-
* @param {object} host - `isLive`, `liveConsumers`, `liveRunsOf`, `producedNumbers`, `reportHostLoad`, `runStateOf`, `sampleDownloadRates`, `encodeCost`, `getSourceStats`, `outputs`, `qualityOffer`, `segmentDurationSec`, `
|
|
75
|
+
* @param {object} host - `isLive`, `liveConsumers`, `liveRunsOf`, `producedNumbers`, `reportHostLoad`, `runStateOf`, `sampleDownloadRates`, `encodeCost`, `getSourceStats`, `outputs`, `qualityOffer`, `segmentDurationSec`, `segmentPaths`, `videoEncoder`, `prepareSameHeightSwitch`, `sameHeightSwitchPending`, `sameHeightSwitchDirection`, `cancelSameHeightSwitch`, `heightReadyFor`, `bufferOf`, `visiblePictureOf`, `expectedFirstSegmentMs`
|
|
76
|
+
* @param {SegmentPathLookup} host.segmentPaths - Lookup for completed
|
|
77
|
+
* segment paths needed by quality decisions.
|
|
74
78
|
*/
|
|
75
79
|
constructor(host) {
|
|
76
80
|
this.#host = host;
|
|
@@ -751,7 +755,7 @@ export class QualityController {
|
|
|
751
755
|
let bytes = 0;
|
|
752
756
|
try {
|
|
753
757
|
for (const index of completed) {
|
|
754
|
-
const segmentPath = this.#host.
|
|
758
|
+
const segmentPath = this.#host.segmentPaths.pathOf(session.outputKey ?? "", index);
|
|
755
759
|
if (!segmentPath) {
|
|
756
760
|
break;
|
|
757
761
|
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared HTTP responses used by more than one route.
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* The answer given when no output suits this viewer: nothing this proxy holds
|
|
6
|
+
* or could produce is admitted by their link. Machine-readable, because the
|
|
7
|
+
* page has to explain it to the viewer and offer what they can do; the
|
|
8
|
+
* figures are the ones the decision was made on.
|
|
9
|
+
*
|
|
10
|
+
* 409 and not a 5xx: nothing is wrong with the proxy, and repeating the same
|
|
11
|
+
* request against the same link gives the same answer.
|
|
12
|
+
*
|
|
13
|
+
* @param {import("fastify").FastifyReply} reply
|
|
14
|
+
* @param {{ reason: string, figures: object | null, wantedKey?: string }} details
|
|
15
|
+
* @returns {import("fastify").FastifyReply}
|
|
16
|
+
*/
|
|
17
|
+
export function replyOutputUnavailable(reply, details) {
|
|
18
|
+
return reply.code(409).send({
|
|
19
|
+
error: `No output suits this viewer: ${details?.reason ?? "unknown"}.`,
|
|
20
|
+
outcome: "output-unavailable",
|
|
21
|
+
reason: details?.reason ?? "",
|
|
22
|
+
figures: details?.figures ?? null
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The answer given when this machine cannot take the output asked for: nothing
|
|
28
|
+
* it has shown it can encode fits, or it has no place for one more encoder.
|
|
29
|
+
* The page answers it by asking the rest of the pool before anything plays.
|
|
30
|
+
*
|
|
31
|
+
* @param {import("fastify").FastifyReply} reply
|
|
32
|
+
* @param {{ reason: string, figures?: object | null }} details
|
|
33
|
+
* @returns {import("fastify").FastifyReply}
|
|
34
|
+
*/
|
|
35
|
+
export function replyNoCapacity(reply, details) {
|
|
36
|
+
return reply.code(409).send({
|
|
37
|
+
error: `This proxy cannot take this video now: ${details?.reason ?? "unknown"}.`,
|
|
38
|
+
outcome: "no-capacity",
|
|
39
|
+
reason: details?.reason ?? "",
|
|
40
|
+
figures: details?.figures ?? null
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The answer given when an address this viewer was already given cannot be
|
|
46
|
+
* given again without risking a piece under a header it may not match. The
|
|
47
|
+
* page answers it by starting a new viewing where the picture is — which is
|
|
48
|
+
* why it is not a "retry": the same request would be answered the same way.
|
|
49
|
+
*
|
|
50
|
+
* @param {import("fastify").FastifyReply} reply
|
|
51
|
+
* @param {{ reason: string }} details
|
|
52
|
+
* @returns {import("fastify").FastifyReply}
|
|
53
|
+
*/
|
|
54
|
+
export function replyAssignmentLost(reply, details) {
|
|
55
|
+
return reply.code(409).send({
|
|
56
|
+
error: `This part was answered by an output that has gone: ${details?.reason ?? "unknown"}.`,
|
|
57
|
+
outcome: "assignment-lost",
|
|
58
|
+
reason: details?.reason ?? ""
|
|
59
|
+
});
|
|
60
|
+
}
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
import { logger } from "../../utils/logger.js";
|
|
2
|
+
import { bandOf, waits } from "../viewer/WaitLedger.js";
|
|
3
|
+
|
|
4
|
+
const HOLD_HEADER = "x-hold-ms";
|
|
5
|
+
/**
|
|
6
|
+
* Which viewer is asking, or an empty string where the transport names nobody.
|
|
7
|
+
*
|
|
8
|
+
* @param {import("fastify").FastifyRequest} req
|
|
9
|
+
* @returns {string}
|
|
10
|
+
*/
|
|
11
|
+
export function consumerOf(req) {
|
|
12
|
+
return typeof req.query?.consumer === "string" ? req.query.consumer : "";
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The generation the request says it was made in, or NaN when it says none.
|
|
17
|
+
*
|
|
18
|
+
* STAMPED BY THE PAGE when the request was SENT (`webrtc-hls-loader.js`), not
|
|
19
|
+
* when it arrived: a request made before a seek and delivered after it still
|
|
20
|
+
* says it belongs to the viewing that was left. Only a stated non-negative
|
|
21
|
+
* integer counts; anything else is "nothing stated".
|
|
22
|
+
*
|
|
23
|
+
* @param {import("fastify").FastifyRequest} req
|
|
24
|
+
* @returns {number}
|
|
25
|
+
*/
|
|
26
|
+
export function statedGenerationOf(req) {
|
|
27
|
+
const raw = req.query?.generation;
|
|
28
|
+
if (typeof raw !== "string" || !/^\d+$/.test(raw)) {
|
|
29
|
+
return Number.NaN;
|
|
30
|
+
}
|
|
31
|
+
return Number(raw);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Answer at once, and do nothing else, when the request belongs to a viewing
|
|
36
|
+
* its viewer has left and whose window for new requests has passed.
|
|
37
|
+
*
|
|
38
|
+
* BEFORE ANY OTHER STEP OF THE ROUTE. Resolving a quality step or a soundtrack
|
|
39
|
+
* can create an output and registers the viewer on it; a stale request that got
|
|
40
|
+
* that far would leave both behind and be refused only afterwards. The answer
|
|
41
|
+
* is the same one a request made pointless by a seek already gets — 503 with
|
|
42
|
+
* `Retry-After: 0` — because it is the same situation seen from the other end.
|
|
43
|
+
*
|
|
44
|
+
* @param {import("fastify").FastifyRequest} req
|
|
45
|
+
* @param {import("fastify").FastifyReply} reply
|
|
46
|
+
* @param {{ acceptsRequest: Function }} viewerRequests
|
|
47
|
+
* @param {string} fileName - For the log line.
|
|
48
|
+
* @returns {boolean} Whether the request was answered here.
|
|
49
|
+
*/
|
|
50
|
+
export function refusedAsStale(req, reply, viewerRequests, fileName) {
|
|
51
|
+
const consumerId = consumerOf(req);
|
|
52
|
+
const stated = statedGenerationOf(req);
|
|
53
|
+
if (viewerRequests.acceptsRequest(consumerId, stated)) {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
logger.info(
|
|
57
|
+
`[hold] ${fileName} refused: made in generation ${stated} of ${consumerId}, ` +
|
|
58
|
+
`a viewing they have left and whose window for new requests has passed`
|
|
59
|
+
);
|
|
60
|
+
reply.header("Retry-After", "0");
|
|
61
|
+
reply.code(503).send({ error: "Made for a viewing that has been left." });
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Serve one playlist or segment from a named session.
|
|
67
|
+
*
|
|
68
|
+
* Split from the route above because the variant route
|
|
69
|
+
* (`/transcode/:sessionId/v/:height/:fileName`) serves the same files from
|
|
70
|
+
* another session of the same family, and must hold, log and answer them
|
|
71
|
+
* identically — a switch of quality must not go through a different code path
|
|
72
|
+
* from the stream it switches away from.
|
|
73
|
+
*
|
|
74
|
+
* @param {import("fastify").FastifyRequest} req
|
|
75
|
+
* @param {import("fastify").FastifyReply} reply
|
|
76
|
+
* @param {{ serving: object, viewerRequests: object, sessionId: string, fileName: string }} params
|
|
77
|
+
* @returns {Promise<void>}
|
|
78
|
+
*/
|
|
79
|
+
export async function serveSessionFile(req, reply, { serving, viewerRequests, sessionId, fileName }) {
|
|
80
|
+
// Which viewer is asking. One session serves everyone watching a copied
|
|
81
|
+
// picture, so without it their positions collapse into one and a seek by the
|
|
82
|
+
// viewer in front releases the requests held for the viewer behind. Absent on
|
|
83
|
+
// a plain HTTP transport, where no loader of ours builds the URL, and then
|
|
84
|
+
// the single shared position decides as it always did.
|
|
85
|
+
const consumerId = consumerOf(req);
|
|
86
|
+
// Hold the request only briefly, then answer "retry" instead of waiting for
|
|
87
|
+
// the segment. iOS's native HLS player (AVPlayer) enforces a hard ~3.5 s
|
|
88
|
+
// deadline on RESPONSE HEADERS and raises -12889 ("No response for media
|
|
89
|
+
// file") when it passes — it then cancels in-flight requests, probes
|
|
90
|
+
// neighbouring positions and can restart the stream from the beginning. That
|
|
91
|
+
// is exactly the post-seek "player thrashing" seen in the field, because a
|
|
92
|
+
// seek restarts ffmpeg and the first segment then takes far longer than 3.5 s
|
|
93
|
+
// to appear. Holding the connection for 30 s (as this did) guaranteed the
|
|
94
|
+
// timeout on every seek. A short hold keeps the fast path intact (a ready or
|
|
95
|
+
// nearly-ready segment is still served on the first request) while a slow one
|
|
96
|
+
// gets a prompt retryable answer, which resets the player's own deadline.
|
|
97
|
+
// hls.js is unaffected: it consumes the 503 through its retry policy, whose
|
|
98
|
+
// budget the client widens to match (see hls-player.js fragLoadPolicy).
|
|
99
|
+
// Instrumented wait. `clientAborted` flips when the player drops the
|
|
100
|
+
// connection while we are still holding it — the single most informative
|
|
101
|
+
// signal about its real patience, and observable only from this side.
|
|
102
|
+
const holdStartedAt = Date.now();
|
|
103
|
+
let clientAborted = false;
|
|
104
|
+
let onClientAbort = () => {};
|
|
105
|
+
const clientGone = new Promise((resolve) => {
|
|
106
|
+
onClientAbort = () => {
|
|
107
|
+
clientAborted = true;
|
|
108
|
+
resolve();
|
|
109
|
+
};
|
|
110
|
+
});
|
|
111
|
+
req.raw.on("close", onClientAbort);
|
|
112
|
+
const statedHoldMs = Number(req.headers?.[HOLD_HEADER]);
|
|
113
|
+
|
|
114
|
+
const result = await waitForSessionFile(serving, sessionId, fileName, {
|
|
115
|
+
holdMs: Number.isFinite(statedHoldMs) && statedHoldMs > 0 ? statedHoldMs : Number.POSITIVE_INFINITY,
|
|
116
|
+
until: clientGone,
|
|
117
|
+
consumerId
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
req.raw.off("close", onClientAbort);
|
|
121
|
+
const heldMs = Date.now() - holdStartedAt;
|
|
122
|
+
if (result.isPlaylist !== true) {
|
|
123
|
+
const outcome = clientAborted
|
|
124
|
+
? "client-aborted"
|
|
125
|
+
: result.kind === "ok" ? "served" : result.kind;
|
|
126
|
+
// AGAINST THE RANK THE MAP GAVE IT. A wait is the only thing that says
|
|
127
|
+
// whether the prioritisation is being used well, and this is the one place
|
|
128
|
+
// in the proxy where a viewer is measurably waiting for a named segment.
|
|
129
|
+
// Recorded even when the segment was served at once: a run of short waits
|
|
130
|
+
// at the top rank is what "the urgent zone is being served first" looks
|
|
131
|
+
// like, and without them the table would hold only the failures.
|
|
132
|
+
const ranked = result.ranked ?? null;
|
|
133
|
+
if (ranked) {
|
|
134
|
+
waits.note(ranked.address, heldMs, ranked.rank, ranked.topRank);
|
|
135
|
+
}
|
|
136
|
+
logger.info(
|
|
137
|
+
`[hold] ${fileName} ${outcome} after ${heldMs}ms` +
|
|
138
|
+
(ranked ? ` (the map wants it ${bandOf(ranked.rank, ranked.topRank)}, rank ${ranked.rank} of ${ranked.topRank})` : "")
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (result.kind === "not-found") {
|
|
143
|
+
return reply.code(404).send({ error: "Transcode session file was not found." });
|
|
144
|
+
}
|
|
145
|
+
if (result.kind === "superseded") {
|
|
146
|
+
// The viewer moved while this was being held, and this segment is not the
|
|
147
|
+
// one they moved TO — that case is kept and waited out, see
|
|
148
|
+
// `waitForSessionFile`. Answer at once so the player can ask for where it
|
|
149
|
+
// is now; `Retry-After: 0` because there is nothing to wait for.
|
|
150
|
+
//
|
|
151
|
+
// Named in the log, because a refusal that says only "superseded" is what
|
|
152
|
+
// made the 2026-08-18 freeze take a day to explain: the player was refused
|
|
153
|
+
// the segment at its own seek target and nothing recorded which segment or
|
|
154
|
+
// where the viewer was.
|
|
155
|
+
logger.info(
|
|
156
|
+
`[hold] ${fileName} refused: the viewer is at ` +
|
|
157
|
+
`${viewerRequests.viewerPositionOf(sessionId, consumerId).toFixed(1)}s and this is not the segment there`
|
|
158
|
+
);
|
|
159
|
+
reply.header("Retry-After", "0");
|
|
160
|
+
return reply.code(503).send({ error: "Superseded by a seek." });
|
|
161
|
+
}
|
|
162
|
+
if (result.kind === "warming-up") {
|
|
163
|
+
// The segment is still being produced (e.g. just after a seek-restart).
|
|
164
|
+
// Return a retryable 503 — never 202, which hls.js cannot consume as a
|
|
165
|
+
// media segment — so the player retries the fetch shortly.
|
|
166
|
+
reply.header("Retry-After", "1");
|
|
167
|
+
// `Retry-After` tells the player to re-request THIS segment after a short
|
|
168
|
+
// pause. Without it a bare 503 reads as "nothing here", and the player goes
|
|
169
|
+
// looking elsewhere: because our synthetic VOD playlist lists every segment
|
|
170
|
+
// of the file, it believes they all exist and SCANS them (field log: one
|
|
171
|
+
// user seek produced probes at #617, #717, #732…). That scan is what used
|
|
172
|
+
// to steer the encoder off the real target. Whether iOS's native player
|
|
173
|
+
// honours the hint is not guaranteed — its behaviour is closed — but this
|
|
174
|
+
// is the standard, correct way to say "wait, don't look elsewhere", and
|
|
175
|
+
// hls.js already retries the same fragment regardless.
|
|
176
|
+
reply.header("Retry-After", "1");
|
|
177
|
+
// SAY WHY, to the viewer and not only to the log. The same refusal is
|
|
178
|
+
// written into `proxy.log` with the rank the priority map gave this
|
|
179
|
+
// segment; the page had no access to that and told the viewer it did not
|
|
180
|
+
// know why. Field 2026-09-14: `rank 1 of 100` — nothing was making it —
|
|
181
|
+
// printed sixty seconds before the page gave up saying the opposite.
|
|
182
|
+
return reply.code(503).send({
|
|
183
|
+
error: "Transcode segment is still being produced.",
|
|
184
|
+
reason: warmingReason(result.ranked ?? null)
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
if (result.kind === "failed") {
|
|
188
|
+
return reply.code(500).send({ error: result.message });
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (result.isPlaylist) {
|
|
192
|
+
reply.header("Cache-Control", "no-store");
|
|
193
|
+
} else {
|
|
194
|
+
reply.header("Cache-Control", "public, max-age=60");
|
|
195
|
+
holdWhileSending(reply, result.stream, viewerRequests.holdResponse(sessionId, consumerId));
|
|
196
|
+
}
|
|
197
|
+
reply.header("Content-Type", result.contentType);
|
|
198
|
+
return reply.send(result.stream);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Keep the output this response comes from held until the response is over.
|
|
203
|
+
*
|
|
204
|
+
* Over means any of `finish`, `close` or `error` of the response, or `error`
|
|
205
|
+
* of the stream being sent — whichever comes first, and usually more than one
|
|
206
|
+
* of them comes: a response that finishes also closes. `release` is safe to
|
|
207
|
+
* call any number of times, which is what makes listening to all of them
|
|
208
|
+
* correct rather than a leak or a double release.
|
|
209
|
+
*
|
|
210
|
+
* A response already over before it began (the requester went during the
|
|
211
|
+
* hold) is released at once: none of its events will fire again.
|
|
212
|
+
*
|
|
213
|
+
* @param {import("fastify").FastifyReply} reply
|
|
214
|
+
* @param {import("node:stream").Readable | undefined} stream
|
|
215
|
+
* @param {() => void} release
|
|
216
|
+
* @returns {void}
|
|
217
|
+
*/
|
|
218
|
+
function holdWhileSending(reply, stream, release) {
|
|
219
|
+
const response = reply.raw;
|
|
220
|
+
if (!response || response.destroyed || response.writableEnded) {
|
|
221
|
+
release();
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
response.once("finish", release);
|
|
225
|
+
response.once("close", release);
|
|
226
|
+
response.once("error", release);
|
|
227
|
+
stream?.once?.("error", release);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Ask `serving.getFileStream()` again each time the file may have become
|
|
232
|
+
* available — a segment's publication, or a waited-on invalidation — until it
|
|
233
|
+
* is, the session fails, the requester's stated wait runs out, or the
|
|
234
|
+
* requester goes.
|
|
235
|
+
*
|
|
236
|
+
* @param {object} serving - `services/server/SegmentServing.js`
|
|
237
|
+
* @param {string} sessionId
|
|
238
|
+
* @param {string} fileName
|
|
239
|
+
* @param {object} hold
|
|
240
|
+
* @param {number} hold.holdMs - How long the requester will wait; may be
|
|
241
|
+
* infinite when it stated nothing.
|
|
242
|
+
* @param {Promise<void>} [hold.until] - Settles when the requester has gone.
|
|
243
|
+
* @param {string} [hold.consumerId] - Which viewer is asking, where the
|
|
244
|
+
* transport carries it. Their own position is what decides whether a held
|
|
245
|
+
* request has been made pointless by a seek — a session can have several
|
|
246
|
+
* viewers, and the seek epoch belongs to all of them.
|
|
247
|
+
* @returns {Promise<Awaited<ReturnType<import("./SegmentServing.js").SegmentServing["getFileStream"]>>>}
|
|
248
|
+
*/
|
|
249
|
+
export async function waitForSessionFile(serving, sessionId, fileName, { holdMs, until = null, consumerId = "" }) {
|
|
250
|
+
const startedAt = Date.now();
|
|
251
|
+
let gone = false;
|
|
252
|
+
const requesterGone = until ? until.then(() => { gone = true; }) : new Promise(() => {});
|
|
253
|
+
// The viewer's position when this request was made. A seek makes every held
|
|
254
|
+
// request stale — it asks for a segment nobody is going to watch — and hls.js
|
|
255
|
+
// keeps only ONE fragment load outstanding, so holding on blocks the request
|
|
256
|
+
// the player actually needs now. Measured: 57 s of a 58 s backward seek was
|
|
257
|
+
// this wait, and the segment the viewer wanted took 15 ms once it was asked
|
|
258
|
+
// for.
|
|
259
|
+
let seekEpoch = serving.seekEpoch(sessionId);
|
|
260
|
+
/** @type {{ address: string, rank: number, topRank: number } | null} */
|
|
261
|
+
let lastRanked = null;
|
|
262
|
+
while (!gone && Date.now() - startedAt < holdMs) {
|
|
263
|
+
const result = await serving.getFileStream(sessionId, fileName, { consumerId });
|
|
264
|
+
if (result.kind !== "warming-up") {
|
|
265
|
+
return result;
|
|
266
|
+
}
|
|
267
|
+
if (serving.seekEpoch(sessionId) !== seekEpoch) {
|
|
268
|
+
// A seek moved the epoch. Whether THIS request is stale depends on which
|
|
269
|
+
// segment it asks for: the one the viewer has just landed on races the
|
|
270
|
+
// seek notification and would otherwise be refused at the exact moment it
|
|
271
|
+
// is needed — measured 2026-08-18, two 503s within 80 ms on the segment
|
|
272
|
+
// at the seek target, after which the player never asked for it again.
|
|
273
|
+
if (!serving.requestStillWanted(sessionId, fileName, consumerId)) {
|
|
274
|
+
return { kind: "superseded" };
|
|
275
|
+
}
|
|
276
|
+
logger.info(
|
|
277
|
+
`[hold] ${fileName} kept across a seek: it is the segment the viewer now needs`
|
|
278
|
+
);
|
|
279
|
+
seekEpoch = serving.seekEpoch(sessionId);
|
|
280
|
+
}
|
|
281
|
+
// The rank the map last gave this segment, so a request that runs out of
|
|
282
|
+
// patience is still counted against what it was promised. Kept across the
|
|
283
|
+
// polls because the timeout path has no result of its own.
|
|
284
|
+
lastRanked = result.ranked ?? lastRanked;
|
|
285
|
+
const remaining = Math.max(0, holdMs - (Date.now() - startedAt));
|
|
286
|
+
const waitedForSegment = await serving.waitForSegment(sessionId, fileName, remaining, until);
|
|
287
|
+
if (!waitedForSegment && !gone) {
|
|
288
|
+
// Playlists and init files are not segment publications.
|
|
289
|
+
await Promise.race([delay(Math.min(300, remaining)), requesterGone]);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return { kind: "warming-up", ranked: lastRanked };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Resolve after a given number of milliseconds.
|
|
297
|
+
*
|
|
298
|
+
* @param {number} ms
|
|
299
|
+
* @returns {Promise<void>}
|
|
300
|
+
*/
|
|
301
|
+
function delay(ms) {
|
|
302
|
+
return new Promise((resolve) => {
|
|
303
|
+
setTimeout(resolve, ms);
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* What to tell the VIEWER when a segment is not there yet.
|
|
309
|
+
*
|
|
310
|
+
* In their terms — the film, and what is being done about it — never in ours: a
|
|
311
|
+
* viewer cannot read a log, does not know what a rank is, and is not helped by
|
|
312
|
+
* being told which of our parts is waiting for which other.
|
|
313
|
+
*
|
|
314
|
+
* @param {{ rank: number, topRank: number } | null} ranked - What the priority
|
|
315
|
+
* map thought of this segment while the request was held.
|
|
316
|
+
* @returns {string}
|
|
317
|
+
*/
|
|
318
|
+
function warmingReason(ranked) {
|
|
319
|
+
if (ranked && ranked.rank < ranked.topRank) {
|
|
320
|
+
// The map is working somewhere else in the film. That is the answer the
|
|
321
|
+
// failure of 2026-09-14 needed and nobody was given.
|
|
322
|
+
return "This part of the film is not being prepared yet — the proxy is working further along.";
|
|
323
|
+
}
|
|
324
|
+
return "This part of the film is still being prepared.";
|
|
325
|
+
}
|