@torrent-tv/proxy 2.83.8 → 2.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/CLAUDE.md +5 -3
- package/docs/encode-architecture.md +26 -1
- package/docs/logs.md +97 -51
- package/package.json +1 -1
- package/routes/api/link-probe/get.js +51 -0
- package/routes/api/transcode-sessions/net-report/post.js +84 -64
- package/routes/api/transcode-sessions/post.js +139 -137
- package/routes/transcode/session-file/get.js +268 -240
- package/server.js +3 -0
- package/services/data-channel-handler.js +12 -0
- package/services/delivery-probe.js +842 -828
- package/services/hls-session-manager.js +9759 -10006
- package/services/output/OutputSpec.js +202 -195
- package/services/priority/PriorityMap.js +390 -362
- package/services/priority/PriorityOrchestrator.js +320 -318
- package/services/viewer/Viewer.js +179 -83
- package/services/viewer/positions.js +120 -48
- package/services/viewer/report-intake.js +63 -0
- package/test/held-request-width.test.js +8 -5
- package/test/link-probe.test.js +56 -0
- package/test/orchestrator-wired.test.js +14 -8
- package/test/output-spec.test.js +157 -157
- package/test/plan-is-steady.test.js +225 -0
- package/test/priority-map-emptied.test.js +205 -207
- package/test/priority-map-per-output.test.js +209 -211
- package/test/quality-variants.test.js +1167 -1222
- package/test/seek-target-not-superseded.test.js +14 -12
- package/test/stale-request-after-seek.test.js +2 -3
- package/test/viewer-states.test.js +151 -0
- package/test/viewer.test.js +136 -108
- package/test/viewer-position-on-open.test.js +0 -92
|
@@ -1,240 +1,268 @@
|
|
|
1
|
-
import { logger } from "../../../utils/logger.js";
|
|
2
|
-
import { bandOf, waits } from "../../../services/priority/WaitLedger.js";
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* How long a request for a not-yet-produced file is held before answering with
|
|
6
|
-
* a retryable 503.
|
|
7
|
-
*
|
|
8
|
-
* MEASUREMENT MODE (2026-08-02): deliberately far above any plausible player
|
|
9
|
-
* deadline, so OUR limit never fires first. Whatever ends the wait is then the
|
|
10
|
-
* player's own behaviour — which is exactly what we need to observe. The
|
|
11
|
-
* `[hold]` log line at the call site records, per request, whether the segment
|
|
12
|
-
* arrived, whether we gave up, or whether the CLIENT aborted, and after how
|
|
13
|
-
* long.
|
|
14
|
-
*
|
|
15
|
-
* The previous value (2 s) was chosen to dodge a reported iOS AVPlayer ~3.5 s
|
|
16
|
-
* response-header deadline. That deadline never appears in our own logs (no
|
|
17
|
-
* -12889 across six hours of production logs), and every reference project
|
|
18
|
-
* holds rather than refusing: Jellyfin and hls-vod-too hold unbounded,
|
|
19
|
-
* hls-media-server holds 10 s. The early refusal is what made the player probe
|
|
20
|
-
* scattered positions, which then steered the encoder off target. Choose the
|
|
21
|
-
* final value from what this measurement shows, not from a number read
|
|
22
|
-
* elsewhere.
|
|
23
|
-
*/
|
|
24
|
-
const SEGMENT_WAIT_MS = 60_000;
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Serve HLS playlist and segment files from an active transcode session.
|
|
28
|
-
*
|
|
29
|
-
* Briefly waits for the requested file to appear, then answers with a
|
|
30
|
-
* retryable 503 rather than holding the connection, so clients — in
|
|
31
|
-
* particular iOS's native HLS player — never hit their own response
|
|
32
|
-
* deadline while a segment is still being produced.
|
|
33
|
-
*
|
|
34
|
-
* GET /transcode/:sessionId/:fileName
|
|
35
|
-
*
|
|
36
|
-
* @param {import("fastify").FastifyRequest} req
|
|
37
|
-
* @param {import("fastify").FastifyReply} reply
|
|
38
|
-
* @param {{ hlsSessionManager: import("../../../services/hls-session-manager.js").HlsSessionManager }} deps
|
|
39
|
-
* @returns {Promise<void>}
|
|
40
|
-
*/
|
|
41
|
-
export async function handleTranscodeSessionFileGet(req, reply, { hlsSessionManager }) {
|
|
42
|
-
const sessionId = typeof req.params.sessionId === "string" ? req.params.sessionId : "";
|
|
43
|
-
const fileName = typeof req.params.fileName === "string" ? req.params.fileName : "";
|
|
44
|
-
return serveSessionFile(req, reply, { hlsSessionManager, sessionId, fileName });
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Serve one playlist or segment from a named session.
|
|
49
|
-
*
|
|
50
|
-
* Split from the route above because the variant route
|
|
51
|
-
* (`/transcode/:sessionId/v/:height/:fileName`) serves the same files from
|
|
52
|
-
* another session of the same family, and must hold, log and answer them
|
|
53
|
-
* identically — a switch of quality must not go through a different code path
|
|
54
|
-
* from the stream it switches away from.
|
|
55
|
-
*
|
|
56
|
-
* @param {import("fastify").FastifyRequest} req
|
|
57
|
-
* @param {import("fastify").FastifyReply} reply
|
|
58
|
-
* @param {{ hlsSessionManager: import("../../../services/hls-session-manager.js").HlsSessionManager, sessionId: string, fileName: string }} params
|
|
59
|
-
* @returns {Promise<void>}
|
|
60
|
-
*/
|
|
61
|
-
export async function serveSessionFile(req, reply, { hlsSessionManager, sessionId, fileName }) {
|
|
62
|
-
// Which viewer is asking. One session serves everyone watching a copied
|
|
63
|
-
// picture, so without it their positions collapse into one and a seek by the
|
|
64
|
-
// viewer in front releases the requests held for the viewer behind. Absent on
|
|
65
|
-
// a plain HTTP transport, where no loader of ours builds the URL, and then
|
|
66
|
-
// the single shared position decides as it always did.
|
|
67
|
-
const consumerId = typeof req.query?.consumer === "string" ? req.query.consumer : "";
|
|
68
|
-
// Hold the request only briefly, then answer "retry" instead of waiting for
|
|
69
|
-
// the segment. iOS's native HLS player (AVPlayer) enforces a hard ~3.5 s
|
|
70
|
-
// deadline on RESPONSE HEADERS and raises -12889 ("No response for media
|
|
71
|
-
// file") when it passes — it then cancels in-flight requests, probes
|
|
72
|
-
// neighbouring positions and can restart the stream from the beginning. That
|
|
73
|
-
// is exactly the post-seek "player thrashing" seen in the field, because a
|
|
74
|
-
// seek restarts ffmpeg and the first segment then takes far longer than 3.5 s
|
|
75
|
-
// to appear. Holding the connection for 30 s (as this did) guaranteed the
|
|
76
|
-
// timeout on every seek. A short hold keeps the fast path intact (a ready or
|
|
77
|
-
// nearly-ready segment is still served on the first request) while a slow one
|
|
78
|
-
// gets a prompt retryable answer, which resets the player's own deadline.
|
|
79
|
-
// hls.js is unaffected: it consumes the 503 through its retry policy, whose
|
|
80
|
-
// budget the client widens to match (see hls-player.js fragLoadPolicy).
|
|
81
|
-
// Instrumented wait. `clientAborted` flips when the player drops the
|
|
82
|
-
// connection while we are still holding it — the single most informative
|
|
83
|
-
// signal about its real patience, and observable only from this side.
|
|
84
|
-
const holdStartedAt = Date.now();
|
|
85
|
-
let clientAborted = false;
|
|
86
|
-
const onClientAbort = () => { clientAborted = true; };
|
|
87
|
-
req.raw.on("close", onClientAbort);
|
|
88
|
-
|
|
89
|
-
const result = await waitForSessionFile(
|
|
90
|
-
hlsSessionManager,
|
|
91
|
-
sessionId,
|
|
92
|
-
fileName,
|
|
93
|
-
SEGMENT_WAIT_MS,
|
|
94
|
-
consumerId
|
|
95
|
-
);
|
|
96
|
-
|
|
97
|
-
req.raw.off("close", onClientAbort);
|
|
98
|
-
const heldMs = Date.now() - holdStartedAt;
|
|
99
|
-
if (result.isPlaylist !== true) {
|
|
100
|
-
const outcome = clientAborted
|
|
101
|
-
? "client-aborted"
|
|
102
|
-
: result.kind === "ok" ? "served" : result.kind;
|
|
103
|
-
// AGAINST THE RANK THE MAP GAVE IT. A wait is the only thing that says
|
|
104
|
-
// whether the prioritisation is being used well, and this is the one place
|
|
105
|
-
// in the proxy where a viewer is measurably waiting for a named segment.
|
|
106
|
-
// Recorded even when the segment was served at once: a run of short waits
|
|
107
|
-
// at the top rank is what "the urgent zone is being served first" looks
|
|
108
|
-
// like, and without them the table would hold only the failures.
|
|
109
|
-
const ranked = result.ranked ?? null;
|
|
110
|
-
if (ranked) {
|
|
111
|
-
waits.note(ranked.address, heldMs, ranked.rank, ranked.topRank);
|
|
112
|
-
}
|
|
113
|
-
logger.info(
|
|
114
|
-
`[hold] ${fileName} ${outcome} after ${heldMs}ms` +
|
|
115
|
-
(ranked ? ` (the map wants it ${bandOf(ranked.rank, ranked.topRank)}, rank ${ranked.rank} of ${ranked.topRank})` : "")
|
|
116
|
-
);
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
if (result.kind === "not-found") {
|
|
120
|
-
return reply.code(404).send({ error: "Transcode session file was not found." });
|
|
121
|
-
}
|
|
122
|
-
if (result.kind === "superseded") {
|
|
123
|
-
// The viewer moved while this was being held, and this segment is not the
|
|
124
|
-
// one they moved TO — that case is kept and waited out, see
|
|
125
|
-
// `waitForSessionFile`. Answer at once so the player can ask for where it
|
|
126
|
-
// is now; `Retry-After: 0` because there is nothing to wait for.
|
|
127
|
-
//
|
|
128
|
-
// Named in the log, because a refusal that says only "superseded" is what
|
|
129
|
-
// made the 2026-08-18 freeze take a day to explain: the player was refused
|
|
130
|
-
// the segment at its own seek target and nothing recorded which segment or
|
|
131
|
-
// where the viewer was.
|
|
132
|
-
logger.info(
|
|
133
|
-
`[hold] ${fileName} refused: the viewer is at ` +
|
|
134
|
-
`${hlsSessionManager.viewerPositionOf(sessionId, consumerId).toFixed(1)}s and this is not the segment there`
|
|
135
|
-
);
|
|
136
|
-
reply.header("Retry-After", "0");
|
|
137
|
-
return reply.code(503).send({ error: "Superseded by a seek." });
|
|
138
|
-
}
|
|
139
|
-
if (result.kind === "warming-up") {
|
|
140
|
-
// The segment is still being produced (e.g. just after a seek-restart).
|
|
141
|
-
// Return a retryable 503 — never 202, which hls.js cannot consume as a
|
|
142
|
-
// media segment — so the player retries the fetch shortly.
|
|
143
|
-
reply.header("Retry-After", "1");
|
|
144
|
-
// `Retry-After` tells the player to re-request THIS segment after a short
|
|
145
|
-
// pause. Without it a bare 503 reads as "nothing here", and the player goes
|
|
146
|
-
// looking elsewhere: because our synthetic VOD playlist lists every segment
|
|
147
|
-
// of the file, it believes they all exist and SCANS them (field log: one
|
|
148
|
-
// user seek produced probes at #617, #717, #732…). That scan is what used
|
|
149
|
-
// to steer the encoder off the real target. Whether iOS's native player
|
|
150
|
-
// honours the hint is not guaranteed — its behaviour is closed — but this
|
|
151
|
-
// is the standard, correct way to say "wait, don't look elsewhere", and
|
|
152
|
-
// hls.js already retries the same fragment regardless.
|
|
153
|
-
reply.header("Retry-After", "1");
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
* @
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
//
|
|
194
|
-
//
|
|
195
|
-
// for
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
1
|
+
import { logger } from "../../../utils/logger.js";
|
|
2
|
+
import { bandOf, waits } from "../../../services/priority/WaitLedger.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* How long a request for a not-yet-produced file is held before answering with
|
|
6
|
+
* a retryable 503.
|
|
7
|
+
*
|
|
8
|
+
* MEASUREMENT MODE (2026-08-02): deliberately far above any plausible player
|
|
9
|
+
* deadline, so OUR limit never fires first. Whatever ends the wait is then the
|
|
10
|
+
* player's own behaviour — which is exactly what we need to observe. The
|
|
11
|
+
* `[hold]` log line at the call site records, per request, whether the segment
|
|
12
|
+
* arrived, whether we gave up, or whether the CLIENT aborted, and after how
|
|
13
|
+
* long.
|
|
14
|
+
*
|
|
15
|
+
* The previous value (2 s) was chosen to dodge a reported iOS AVPlayer ~3.5 s
|
|
16
|
+
* response-header deadline. That deadline never appears in our own logs (no
|
|
17
|
+
* -12889 across six hours of production logs), and every reference project
|
|
18
|
+
* holds rather than refusing: Jellyfin and hls-vod-too hold unbounded,
|
|
19
|
+
* hls-media-server holds 10 s. The early refusal is what made the player probe
|
|
20
|
+
* scattered positions, which then steered the encoder off target. Choose the
|
|
21
|
+
* final value from what this measurement shows, not from a number read
|
|
22
|
+
* elsewhere.
|
|
23
|
+
*/
|
|
24
|
+
const SEGMENT_WAIT_MS = 60_000;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Serve HLS playlist and segment files from an active transcode session.
|
|
28
|
+
*
|
|
29
|
+
* Briefly waits for the requested file to appear, then answers with a
|
|
30
|
+
* retryable 503 rather than holding the connection, so clients — in
|
|
31
|
+
* particular iOS's native HLS player — never hit their own response
|
|
32
|
+
* deadline while a segment is still being produced.
|
|
33
|
+
*
|
|
34
|
+
* GET /transcode/:sessionId/:fileName
|
|
35
|
+
*
|
|
36
|
+
* @param {import("fastify").FastifyRequest} req
|
|
37
|
+
* @param {import("fastify").FastifyReply} reply
|
|
38
|
+
* @param {{ hlsSessionManager: import("../../../services/hls-session-manager.js").HlsSessionManager }} deps
|
|
39
|
+
* @returns {Promise<void>}
|
|
40
|
+
*/
|
|
41
|
+
export async function handleTranscodeSessionFileGet(req, reply, { hlsSessionManager }) {
|
|
42
|
+
const sessionId = typeof req.params.sessionId === "string" ? req.params.sessionId : "";
|
|
43
|
+
const fileName = typeof req.params.fileName === "string" ? req.params.fileName : "";
|
|
44
|
+
return serveSessionFile(req, reply, { hlsSessionManager, sessionId, fileName });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Serve one playlist or segment from a named session.
|
|
49
|
+
*
|
|
50
|
+
* Split from the route above because the variant route
|
|
51
|
+
* (`/transcode/:sessionId/v/:height/:fileName`) serves the same files from
|
|
52
|
+
* another session of the same family, and must hold, log and answer them
|
|
53
|
+
* identically — a switch of quality must not go through a different code path
|
|
54
|
+
* from the stream it switches away from.
|
|
55
|
+
*
|
|
56
|
+
* @param {import("fastify").FastifyRequest} req
|
|
57
|
+
* @param {import("fastify").FastifyReply} reply
|
|
58
|
+
* @param {{ hlsSessionManager: import("../../../services/hls-session-manager.js").HlsSessionManager, sessionId: string, fileName: string }} params
|
|
59
|
+
* @returns {Promise<void>}
|
|
60
|
+
*/
|
|
61
|
+
export async function serveSessionFile(req, reply, { hlsSessionManager, sessionId, fileName }) {
|
|
62
|
+
// Which viewer is asking. One session serves everyone watching a copied
|
|
63
|
+
// picture, so without it their positions collapse into one and a seek by the
|
|
64
|
+
// viewer in front releases the requests held for the viewer behind. Absent on
|
|
65
|
+
// a plain HTTP transport, where no loader of ours builds the URL, and then
|
|
66
|
+
// the single shared position decides as it always did.
|
|
67
|
+
const consumerId = typeof req.query?.consumer === "string" ? req.query.consumer : "";
|
|
68
|
+
// Hold the request only briefly, then answer "retry" instead of waiting for
|
|
69
|
+
// the segment. iOS's native HLS player (AVPlayer) enforces a hard ~3.5 s
|
|
70
|
+
// deadline on RESPONSE HEADERS and raises -12889 ("No response for media
|
|
71
|
+
// file") when it passes — it then cancels in-flight requests, probes
|
|
72
|
+
// neighbouring positions and can restart the stream from the beginning. That
|
|
73
|
+
// is exactly the post-seek "player thrashing" seen in the field, because a
|
|
74
|
+
// seek restarts ffmpeg and the first segment then takes far longer than 3.5 s
|
|
75
|
+
// to appear. Holding the connection for 30 s (as this did) guaranteed the
|
|
76
|
+
// timeout on every seek. A short hold keeps the fast path intact (a ready or
|
|
77
|
+
// nearly-ready segment is still served on the first request) while a slow one
|
|
78
|
+
// gets a prompt retryable answer, which resets the player's own deadline.
|
|
79
|
+
// hls.js is unaffected: it consumes the 503 through its retry policy, whose
|
|
80
|
+
// budget the client widens to match (see hls-player.js fragLoadPolicy).
|
|
81
|
+
// Instrumented wait. `clientAborted` flips when the player drops the
|
|
82
|
+
// connection while we are still holding it — the single most informative
|
|
83
|
+
// signal about its real patience, and observable only from this side.
|
|
84
|
+
const holdStartedAt = Date.now();
|
|
85
|
+
let clientAborted = false;
|
|
86
|
+
const onClientAbort = () => { clientAborted = true; };
|
|
87
|
+
req.raw.on("close", onClientAbort);
|
|
88
|
+
|
|
89
|
+
const result = await waitForSessionFile(
|
|
90
|
+
hlsSessionManager,
|
|
91
|
+
sessionId,
|
|
92
|
+
fileName,
|
|
93
|
+
SEGMENT_WAIT_MS,
|
|
94
|
+
consumerId
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
req.raw.off("close", onClientAbort);
|
|
98
|
+
const heldMs = Date.now() - holdStartedAt;
|
|
99
|
+
if (result.isPlaylist !== true) {
|
|
100
|
+
const outcome = clientAborted
|
|
101
|
+
? "client-aborted"
|
|
102
|
+
: result.kind === "ok" ? "served" : result.kind;
|
|
103
|
+
// AGAINST THE RANK THE MAP GAVE IT. A wait is the only thing that says
|
|
104
|
+
// whether the prioritisation is being used well, and this is the one place
|
|
105
|
+
// in the proxy where a viewer is measurably waiting for a named segment.
|
|
106
|
+
// Recorded even when the segment was served at once: a run of short waits
|
|
107
|
+
// at the top rank is what "the urgent zone is being served first" looks
|
|
108
|
+
// like, and without them the table would hold only the failures.
|
|
109
|
+
const ranked = result.ranked ?? null;
|
|
110
|
+
if (ranked) {
|
|
111
|
+
waits.note(ranked.address, heldMs, ranked.rank, ranked.topRank);
|
|
112
|
+
}
|
|
113
|
+
logger.info(
|
|
114
|
+
`[hold] ${fileName} ${outcome} after ${heldMs}ms` +
|
|
115
|
+
(ranked ? ` (the map wants it ${bandOf(ranked.rank, ranked.topRank)}, rank ${ranked.rank} of ${ranked.topRank})` : "")
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (result.kind === "not-found") {
|
|
120
|
+
return reply.code(404).send({ error: "Transcode session file was not found." });
|
|
121
|
+
}
|
|
122
|
+
if (result.kind === "superseded") {
|
|
123
|
+
// The viewer moved while this was being held, and this segment is not the
|
|
124
|
+
// one they moved TO — that case is kept and waited out, see
|
|
125
|
+
// `waitForSessionFile`. Answer at once so the player can ask for where it
|
|
126
|
+
// is now; `Retry-After: 0` because there is nothing to wait for.
|
|
127
|
+
//
|
|
128
|
+
// Named in the log, because a refusal that says only "superseded" is what
|
|
129
|
+
// made the 2026-08-18 freeze take a day to explain: the player was refused
|
|
130
|
+
// the segment at its own seek target and nothing recorded which segment or
|
|
131
|
+
// where the viewer was.
|
|
132
|
+
logger.info(
|
|
133
|
+
`[hold] ${fileName} refused: the viewer is at ` +
|
|
134
|
+
`${hlsSessionManager.viewerPositionOf(sessionId, consumerId).toFixed(1)}s and this is not the segment there`
|
|
135
|
+
);
|
|
136
|
+
reply.header("Retry-After", "0");
|
|
137
|
+
return reply.code(503).send({ error: "Superseded by a seek." });
|
|
138
|
+
}
|
|
139
|
+
if (result.kind === "warming-up") {
|
|
140
|
+
// The segment is still being produced (e.g. just after a seek-restart).
|
|
141
|
+
// Return a retryable 503 — never 202, which hls.js cannot consume as a
|
|
142
|
+
// media segment — so the player retries the fetch shortly.
|
|
143
|
+
reply.header("Retry-After", "1");
|
|
144
|
+
// `Retry-After` tells the player to re-request THIS segment after a short
|
|
145
|
+
// pause. Without it a bare 503 reads as "nothing here", and the player goes
|
|
146
|
+
// looking elsewhere: because our synthetic VOD playlist lists every segment
|
|
147
|
+
// of the file, it believes they all exist and SCANS them (field log: one
|
|
148
|
+
// user seek produced probes at #617, #717, #732…). That scan is what used
|
|
149
|
+
// to steer the encoder off the real target. Whether iOS's native player
|
|
150
|
+
// honours the hint is not guaranteed — its behaviour is closed — but this
|
|
151
|
+
// is the standard, correct way to say "wait, don't look elsewhere", and
|
|
152
|
+
// hls.js already retries the same fragment regardless.
|
|
153
|
+
reply.header("Retry-After", "1");
|
|
154
|
+
// SAY WHY, to the viewer and not only to the log. The same refusal is
|
|
155
|
+
// written into `proxy.log` with the rank the priority map gave this
|
|
156
|
+
// segment; the page had no access to that and told the viewer it did not
|
|
157
|
+
// know why. Field 2026-09-14: `rank 1 of 100` — nothing was making it —
|
|
158
|
+
// printed sixty seconds before the page gave up saying the opposite.
|
|
159
|
+
return reply.code(503).send({
|
|
160
|
+
error: "Transcode segment is still being produced.",
|
|
161
|
+
reason: warmingReason(result.ranked ?? null)
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
if (result.kind === "failed") {
|
|
165
|
+
return reply.code(500).send({ error: result.message });
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
if (result.isPlaylist) {
|
|
169
|
+
reply.header("Cache-Control", "no-store");
|
|
170
|
+
} else {
|
|
171
|
+
reply.header("Cache-Control", "public, max-age=60");
|
|
172
|
+
}
|
|
173
|
+
reply.header("Content-Type", result.contentType);
|
|
174
|
+
return reply.send(result.stream);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Poll `hlsSessionManager.getFileStream()` until the file is available,
|
|
179
|
+
* the session fails, or the timeout elapses.
|
|
180
|
+
*
|
|
181
|
+
* @param {import("../../../services/hls-session-manager.js").HlsSessionManager} hlsSessionManager
|
|
182
|
+
* @param {string} sessionId
|
|
183
|
+
* @param {string} fileName
|
|
184
|
+
* @param {number} timeoutMs
|
|
185
|
+
* @param {string} [consumerId] - Which viewer is asking, where the transport
|
|
186
|
+
* carries it. Their own position is what decides whether a held request has
|
|
187
|
+
* been made pointless by a seek — a session can have several viewers, and
|
|
188
|
+
* the seek epoch belongs to all of them.
|
|
189
|
+
* @returns {Promise<Awaited<ReturnType<import("../../../services/hls-session-manager.js").HlsSessionManager["getFileStream"]>>>}
|
|
190
|
+
*/
|
|
191
|
+
export async function waitForSessionFile(hlsSessionManager, sessionId, fileName, timeoutMs, consumerId = "") {
|
|
192
|
+
const startedAt = Date.now();
|
|
193
|
+
// One sequence number for THIS request, reused by every poll below, so the
|
|
194
|
+
// session can tell a newly-arrived request apart from an old one polling
|
|
195
|
+
// again — see HlsSessionManager#ensureEncodingFor for the encoder ping-pong
|
|
196
|
+
// this prevents when one seek-bar scrub fires several segment requests.
|
|
197
|
+
const requestSeq = hlsSessionManager.nextRequestSeq(sessionId);
|
|
198
|
+
// The viewer's position when this request was made. A seek makes every held
|
|
199
|
+
// request stale — it asks for a segment nobody is going to watch — and hls.js
|
|
200
|
+
// keeps only ONE fragment load outstanding, so holding on blocks the request
|
|
201
|
+
// the player actually needs now. Measured: 57 s of a 58 s backward seek was
|
|
202
|
+
// this wait, and the segment the viewer wanted took 15 ms once it was asked
|
|
203
|
+
// for.
|
|
204
|
+
let seekEpoch = hlsSessionManager.seekEpoch(sessionId);
|
|
205
|
+
/** @type {{ address: string, rank: number, topRank: number } | null} */
|
|
206
|
+
let lastRanked = null;
|
|
207
|
+
while (Date.now() - startedAt < timeoutMs) {
|
|
208
|
+
const result = await hlsSessionManager.getFileStream(sessionId, fileName, {
|
|
209
|
+
requestSeq,
|
|
210
|
+
consumerId
|
|
211
|
+
});
|
|
212
|
+
if (result.kind !== "warming-up") {
|
|
213
|
+
return result;
|
|
214
|
+
}
|
|
215
|
+
if (hlsSessionManager.seekEpoch(sessionId) !== seekEpoch) {
|
|
216
|
+
// A seek moved the epoch. Whether THIS request is stale depends on which
|
|
217
|
+
// segment it asks for: the one the viewer has just landed on races the
|
|
218
|
+
// seek notification and would otherwise be refused at the exact moment it
|
|
219
|
+
// is needed — measured 2026-08-18, two 503s within 80 ms on the segment
|
|
220
|
+
// at the seek target, after which the player never asked for it again.
|
|
221
|
+
if (!hlsSessionManager.requestStillWanted(sessionId, fileName, consumerId)) {
|
|
222
|
+
return { kind: "superseded" };
|
|
223
|
+
}
|
|
224
|
+
logger.info(
|
|
225
|
+
`[hold] ${fileName} kept across a seek: it is the segment the viewer now needs`
|
|
226
|
+
);
|
|
227
|
+
seekEpoch = hlsSessionManager.seekEpoch(sessionId);
|
|
228
|
+
}
|
|
229
|
+
// The rank the map last gave this segment, so a request that runs out of
|
|
230
|
+
// patience is still counted against what it was promised. Kept across the
|
|
231
|
+
// polls because the timeout path has no result of its own.
|
|
232
|
+
lastRanked = result.ranked ?? lastRanked;
|
|
233
|
+
await delay(300);
|
|
234
|
+
}
|
|
235
|
+
return { kind: "warming-up", ranked: lastRanked };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Resolve after a given number of milliseconds.
|
|
240
|
+
*
|
|
241
|
+
* @param {number} ms
|
|
242
|
+
* @returns {Promise<void>}
|
|
243
|
+
*/
|
|
244
|
+
function delay(ms) {
|
|
245
|
+
return new Promise((resolve) => {
|
|
246
|
+
setTimeout(resolve, ms);
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* What to tell the VIEWER when a segment is not there yet.
|
|
252
|
+
*
|
|
253
|
+
* In their terms — the film, and what is being done about it — never in ours: a
|
|
254
|
+
* viewer cannot read a log, does not know what a rank is, and is not helped by
|
|
255
|
+
* being told which of our parts is waiting for which other.
|
|
256
|
+
*
|
|
257
|
+
* @param {{ rank: number, topRank: number } | null} ranked - What the priority
|
|
258
|
+
* map thought of this segment while the request was held.
|
|
259
|
+
* @returns {string}
|
|
260
|
+
*/
|
|
261
|
+
function warmingReason(ranked) {
|
|
262
|
+
if (ranked && ranked.rank < ranked.topRank) {
|
|
263
|
+
// The map is working somewhere else in the film. That is the answer the
|
|
264
|
+
// failure of 2026-09-14 needed and nobody was given.
|
|
265
|
+
return "This part of the film is not being prepared yet — the proxy is working further along.";
|
|
266
|
+
}
|
|
267
|
+
return "This part of the film is still being prepared.";
|
|
268
|
+
}
|
package/server.js
CHANGED
|
@@ -17,6 +17,7 @@ import { fileURLToPath } from "node:url";
|
|
|
17
17
|
import { handleHealthGet } from "./routes/health/get.js";
|
|
18
18
|
import { handleHealthzGet } from "./routes/healthz/get.js";
|
|
19
19
|
import { handleApiDeliverySinkGet } from "./routes/api/delivery-sink/get.js";
|
|
20
|
+
import { handleApiLinkProbeGet } from "./routes/api/link-probe/get.js";
|
|
20
21
|
import { handleApiSourcesPost } from "./routes/api/sources/post.js";
|
|
21
22
|
import { handleApiSourceStatsGet } from "./routes/api/sources/stats/get.js";
|
|
22
23
|
import { handleApiSourceFilesGet } from "./routes/api/sources/files/get.js";
|
|
@@ -340,6 +341,8 @@ export async function startProxyServer({
|
|
|
340
341
|
app.get("/api/delivery-sink", async (req, reply) =>
|
|
341
342
|
handleApiDeliverySinkGet(req, reply, { enabled: deliverySink === true })
|
|
342
343
|
);
|
|
344
|
+
// Bytes a browser can time its link with before any film has been chosen.
|
|
345
|
+
app.get("/api/link-probe", async (req, reply) => handleApiLinkProbeGet(req, reply));
|
|
343
346
|
app.post("/api/sources", async (req, reply) =>
|
|
344
347
|
handleApiSourcesPost(req, reply, { sourceRegistry })
|
|
345
348
|
);
|
|
@@ -362,6 +362,18 @@ function makeSendQueueWatcher({ log, getTransportSnapshot, witness, usrsctpState
|
|
|
362
362
|
const snapshot = getTransportSnapshot?.(sessionId) ?? null;
|
|
363
363
|
connection.unknown = snapshot ? 0 : connection.unknown + 1;
|
|
364
364
|
if (connection.unknown >= TRANSPORT_UNKNOWN_HEARTBEATS) {
|
|
365
|
+
// THE SECOND DETECTOR, and it is the one that has to work: `onClosed`
|
|
366
|
+
// does not always come — a peer connection can die without it — and
|
|
367
|
+
// presence is the connection and nothing else, so nothing else will
|
|
368
|
+
// ever say these people have gone. Until now this ended the watch and
|
|
369
|
+
// left them registered for the life of the process, holding the
|
|
370
|
+
// outputs they were watching.
|
|
371
|
+
// Named for what it is: the watcher is handed this, it does not
|
|
372
|
+
// hold the registry. Written as a bare `connectionGone` it was a
|
|
373
|
+
// reference to a function of the FACTORY, which is not in scope here,
|
|
374
|
+
// so the one detector that has to work threw a `ReferenceError`
|
|
375
|
+
// inside its own timer instead of releasing the viewer.
|
|
376
|
+
onConnectionGone?.(sessionId, "the transport stopped answering");
|
|
365
377
|
stop();
|
|
366
378
|
return;
|
|
367
379
|
}
|