@torrent-tv/proxy 2.80.12 → 2.80.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/docs/download-architecture.md +23 -0
- package/docs/encode-architecture.md +30 -0
- package/package.json +1 -1
- package/routes/stream/get.js +324 -300
- package/routes/transcode/session-file/get.js +22 -2
- package/services/encode/SegmentDemand.js +34 -0
- package/services/hls-session-manager.js +14 -16
- package/services/orchestrators/EncodeOrchestrator.js +735 -726
- package/services/priority/WaitLedger.js +142 -0
- package/services/torrent-pool.js +38 -1
- package/services/torrent-worker/client.js +27 -12
- package/test/wait-ledger.test.js +100 -0
package/routes/stream/get.js
CHANGED
|
@@ -1,300 +1,324 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file Byte-range aware torrent file streaming endpoint.
|
|
3
|
-
*
|
|
4
|
-
* Accepts either a `sourceKey` (registered via POST /api/sources) or a raw
|
|
5
|
-
* `sourceType` + `source` pair. Responds with HTTP 206 for range requests
|
|
6
|
-
* and HTTP 200 for full-file requests.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import { parseRange } from "../../utils/parse-range.js";
|
|
10
|
-
import { logger } from "../../utils/logger.js";
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* Resolve source parameters from the query string.
|
|
14
|
-
* Prefers a registered `sourceKey`; falls back to inline `sourceType`+`source`.
|
|
15
|
-
*
|
|
16
|
-
* @param {import("fastify").FastifyRequest["query"]} query
|
|
17
|
-
* @param {ReturnType<import("../../store/source-registry.js").createSourceRegistry>} sourceRegistry
|
|
18
|
-
* @returns {{ sourceType: string, source: string }}
|
|
19
|
-
*/
|
|
20
|
-
function getSourceParams(query, sourceRegistry) {
|
|
21
|
-
const sourceKey = typeof query.sourceKey === "string" ? query.sourceKey : "";
|
|
22
|
-
const sourceTypeFromQuery = typeof query.sourceType === "string" ? query.sourceType : "";
|
|
23
|
-
const sourceFromQuery = typeof query.source === "string" ? query.source : "";
|
|
24
|
-
|
|
25
|
-
const sourceRecord = sourceKey ? sourceRegistry.get(sourceKey) : null;
|
|
26
|
-
const sourceType = sourceRecord?.sourceType ?? sourceTypeFromQuery;
|
|
27
|
-
const source = sourceRecord?.source ?? sourceFromQuery;
|
|
28
|
-
return { sourceType, source };
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* How long a request may wait for a torrent that is still being added.
|
|
33
|
-
*
|
|
34
|
-
* Adding a magnet takes as long as its metadata does — seconds when peers
|
|
35
|
-
* answer, forever when none do — and `getTorrent` waits for it. Awaiting that
|
|
36
|
-
* with no bound is what made this route answer nothing at all.
|
|
37
|
-
*/
|
|
38
|
-
const TORRENT_READY_TIMEOUT_MS = 10_000;
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* The torrent for a source, or a `TORRENT_NOT_READY` error once the wait has
|
|
42
|
-
* gone on long enough to be worth reporting.
|
|
43
|
-
*
|
|
44
|
-
* The underlying add is NOT cancelled: it keeps running and warms the pool, so
|
|
45
|
-
* the client's next attempt is likely to find it ready.
|
|
46
|
-
*
|
|
47
|
-
* @param {import("../../services/torrent-pool.js").TorrentPool} torrentPool
|
|
48
|
-
* @param {string} sourceType
|
|
49
|
-
* @param {string} source
|
|
50
|
-
* @returns {Promise<import("webtorrent").Torrent>}
|
|
51
|
-
*/
|
|
52
|
-
async function waitForTorrent(torrentPool, sourceType, source) {
|
|
53
|
-
let timer = null;
|
|
54
|
-
const expiry = new Promise((_resolve, reject) => {
|
|
55
|
-
timer = setTimeout(() => {
|
|
56
|
-
const error = new Error("Torrent metadata is not available yet.");
|
|
57
|
-
error.code = "TORRENT_NOT_READY";
|
|
58
|
-
reject(error);
|
|
59
|
-
}, TORRENT_READY_TIMEOUT_MS);
|
|
60
|
-
timer.unref?.();
|
|
61
|
-
});
|
|
62
|
-
try {
|
|
63
|
-
return await Promise.race([torrentPool.getTorrent(sourceType, source), expiry]);
|
|
64
|
-
} finally {
|
|
65
|
-
if (timer) {
|
|
66
|
-
clearTimeout(timer);
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* Stream a torrent file over HTTP with byte-range support.
|
|
73
|
-
*
|
|
74
|
-
* GET /stream
|
|
75
|
-
*
|
|
76
|
-
* @param {import("fastify").FastifyRequest} req
|
|
77
|
-
* @param {import("fastify").FastifyReply} reply
|
|
78
|
-
* @param {{ sourceRegistry: ReturnType<import("../../store/source-registry.js").createSourceRegistry>, torrentPool: import("../../services/torrent-pool.js").TorrentPool }} deps
|
|
79
|
-
* @returns {Promise<void>}
|
|
80
|
-
*/
|
|
81
|
-
export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool, noteInputBytes = null }) {
|
|
82
|
-
const fileIndexRaw = typeof req.query.fileIndex === "string" ? req.query.fileIndex : "";
|
|
83
|
-
const fileIndex = Number(fileIndexRaw);
|
|
84
|
-
const { sourceType, source } = getSourceParams(req.query, sourceRegistry);
|
|
85
|
-
|
|
86
|
-
if (!sourceType || !source || !Number.isInteger(fileIndex) || fileIndex < 0) {
|
|
87
|
-
return reply
|
|
88
|
-
.code(400)
|
|
89
|
-
.send({ error: "sourceKey or sourceType+source with fileIndex are required." });
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
let torrent;
|
|
93
|
-
try {
|
|
94
|
-
torrent = await waitForTorrent(torrentPool, sourceType, source);
|
|
95
|
-
} catch (error) {
|
|
96
|
-
if (error instanceof Error && error.code === "TORRENT_NOT_READY") {
|
|
97
|
-
// Say so, rather than holding the connection until the client gives up.
|
|
98
|
-
// Reproduced 2026-08-04 on a magnet whose metadata never arrived: both a
|
|
99
|
-
// ranged GET and a HEAD returned nothing at all for the full 30 s the
|
|
100
|
-
// probe was willing to wait, and the route had written neither a status
|
|
101
|
-
// nor a header — from the client that is indistinguishable from the proxy
|
|
102
|
-
// having died, and it left no trace in the log either.
|
|
103
|
-
reply.header("Retry-After", "1");
|
|
104
|
-
return reply.code(503).send({ error: "Torrent metadata is not available yet." });
|
|
105
|
-
}
|
|
106
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
107
|
-
return reply.code(500).send({ error: `Failed to load torrent source: ${message}` });
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
const file = torrent.files[fileIndex];
|
|
111
|
-
if (!file) {
|
|
112
|
-
return reply.code(404).send({ error: "File index was not found in torrent." });
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
// HEAD asks what a GET would return, not for the bytes. Fastify serves HEAD
|
|
116
|
-
// from this same handler, which used to mean a HEAD started a read of the
|
|
117
|
-
// WHOLE file: the body was discarded by Node, but the read ran on, the
|
|
118
|
-
// response never finished, and the next request on that keep-alive connection
|
|
119
|
-
// waited behind it. Measured on the field host: the keyframe-index HEAD
|
|
120
|
-
// returned headers in 23 ms and then held the connection until its 15 s
|
|
121
|
-
// timeout, which is where the 73 s transcode-session create went.
|
|
122
|
-
if (req.method === "HEAD") {
|
|
123
|
-
// Written to the raw response on purpose. Answering through `reply.send()`
|
|
124
|
-
// with no payload makes Fastify set `content-length: 0`, which is worse
|
|
125
|
-
// than useless here: the keyframe index asks for the file size with this
|
|
126
|
-
// very request and treats 0 as "no index available", silently falling back
|
|
127
|
-
// to an invented segment grid. Hijacking leaves the response to us, and
|
|
128
|
-
// Node omits the body for HEAD by itself.
|
|
129
|
-
reply.hijack();
|
|
130
|
-
reply.raw.writeHead(200, {
|
|
131
|
-
"Accept-Ranges": "bytes",
|
|
132
|
-
"Content-Type": "application/octet-stream",
|
|
133
|
-
"Content-Length": String(file.length),
|
|
134
|
-
"Content-Disposition": `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`
|
|
135
|
-
});
|
|
136
|
-
reply.raw.end();
|
|
137
|
-
return;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
const releaseFile = torrentPool.acquireFile(torrent, fileIndex);
|
|
141
|
-
|
|
142
|
-
const range = parseRange(req.headers.range, file.length);
|
|
143
|
-
// Prioritize the pieces at this read position so a seek (a request at a new
|
|
144
|
-
// byte offset) downloads first instead of waiting behind the sequential
|
|
145
|
-
// backlog — this is what caused ~15-18 s stalls when seeking into an
|
|
146
|
-
// undownloaded region.
|
|
147
|
-
// Only the encoder's own input read tracks where the viewer is. Everything
|
|
148
|
-
// else that comes through here — the codec probe, the keyframe index, a
|
|
149
|
-
// subtitle fetch — reads the file's edges, and treating those as a viewer
|
|
150
|
-
// position made every session start and every encoder restart look like a
|
|
151
|
-
// burst of seeks (measured: two spurious "the viewer moved" per start).
|
|
152
|
-
torrentPool.prioritizeByteRange(torrent, fileIndex, range ? range.start : 0, undefined, {
|
|
153
|
-
wholeFileRead: range === null,
|
|
154
|
-
isPlaybackRead: req.query.reader === "playback"
|
|
155
|
-
});
|
|
156
|
-
|
|
157
|
-
const start = range ? range.start : 0;
|
|
158
|
-
const end = range ? range.end : file.length - 1;
|
|
159
|
-
const contentLength = end - start + 1;
|
|
160
|
-
|
|
161
|
-
// Written straight out of the torrent's shared memory when that is available:
|
|
162
|
-
// no copy on either thread, at the cost of doing the writing by hand, because
|
|
163
|
-
// only the write callback tells us when a piece may be released. Falls back to
|
|
164
|
-
// the ordinary stream for sources without a shared pool.
|
|
165
|
-
// How far ahead of its own read head this reader should ask the swarm for.
|
|
166
|
-
// Supplied by whoever knows the media's byte rate — the transcode session
|
|
167
|
-
// puts it on the ffmpeg input URL, sized in seconds of playback — because
|
|
168
|
-
// this thread knows only bytes, and 32 MB is half a minute of a 1080p film
|
|
169
|
-
// but four seconds of a disc remux. Absent or unusable, the reader's own
|
|
170
|
-
// default stands.
|
|
171
|
-
const windowBytesRaw = Number(req.query.windowBytes);
|
|
172
|
-
const windowBytes =
|
|
173
|
-
Number.isFinite(windowBytesRaw) && windowBytesRaw > 0 ? windowBytesRaw : undefined;
|
|
174
|
-
|
|
175
|
-
// Which transcode session this read feeds, when it feeds one. Put on the URL
|
|
176
|
-
// by the session that builds it, because this route otherwise knows only a
|
|
177
|
-
// file — and two sessions can read one file, so the file cannot stand in for
|
|
178
|
-
// the session.
|
|
179
|
-
const sessionId = typeof req.query.session === "string" ? req.query.session : "";
|
|
180
|
-
|
|
181
|
-
const fragments = typeof file.createFragmentReader === "function"
|
|
182
|
-
? file.createFragmentReader({ start, end, windowBytes })
|
|
183
|
-
: null;
|
|
184
|
-
|
|
185
|
-
if (fragments) {
|
|
186
|
-
reply.hijack();
|
|
187
|
-
reply.raw.writeHead(range ? 206 : 200, {
|
|
188
|
-
"Accept-Ranges": "bytes",
|
|
189
|
-
"Content-Type": "application/octet-stream",
|
|
190
|
-
"Content-Length": String(contentLength),
|
|
191
|
-
"Content-Disposition": `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`,
|
|
192
|
-
...(range ? { "Content-Range": `bytes ${start}-${end}/${file.length}` } : {})
|
|
193
|
-
});
|
|
194
|
-
|
|
195
|
-
// A client that goes away mid-response must stop the read, or pieces keep
|
|
196
|
-
// being fetched for nobody.
|
|
197
|
-
reply.raw.once("close", () => fragments.cancel());
|
|
198
|
-
|
|
199
|
-
let sent = 0;
|
|
200
|
-
try {
|
|
201
|
-
for await (const fragment of fragments) {
|
|
202
|
-
if (reply.raw.writableEnded || reply.raw.destroyed) {
|
|
203
|
-
fragment.release();
|
|
204
|
-
break;
|
|
205
|
-
}
|
|
206
|
-
await new Promise((resolve, reject) => {
|
|
207
|
-
reply.raw.write(fragment.bytes, (error) => (error ? reject(error) : resolve()));
|
|
208
|
-
});
|
|
209
|
-
sent += fragment.bytes.length;
|
|
210
|
-
// Say so, if this read belongs to a transcode session. It is the only
|
|
211
|
-
// proof that a session which has produced nothing yet is nevertheless
|
|
212
|
-
// being fed: the encoder's own progress cannot move until its first
|
|
213
|
-
// frame is decoded, and a viewer waiting for that first frame was being
|
|
214
|
-
// told the proxy had died while the swarm was delivering to it. Field
|
|
215
|
-
// 2026-09-03: 46.3 s on one piece, `processedSeconds` frozen at the
|
|
216
|
-
// start position throughout, and the browser gave up 0.4 s before the
|
|
217
|
-
// piece landed.
|
|
218
|
-
if (noteInputBytes) {
|
|
219
|
-
noteInputBytes(sessionId, fragment.bytes.length);
|
|
220
|
-
}
|
|
221
|
-
// Only now are these bytes gone: the piece can be unpinned, and the
|
|
222
|
-
// slot it occupies reused. Releasing before this point corrupts the
|
|
223
|
-
// response silently.
|
|
224
|
-
fragment.release();
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
//
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
//
|
|
232
|
-
// and
|
|
233
|
-
//
|
|
234
|
-
//
|
|
235
|
-
//
|
|
236
|
-
//
|
|
237
|
-
//
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
}
|
|
250
|
-
reply.raw.
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
stream.
|
|
296
|
-
stream
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @file Byte-range aware torrent file streaming endpoint.
|
|
3
|
+
*
|
|
4
|
+
* Accepts either a `sourceKey` (registered via POST /api/sources) or a raw
|
|
5
|
+
* `sourceType` + `source` pair. Responds with HTTP 206 for range requests
|
|
6
|
+
* and HTTP 200 for full-file requests.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { parseRange } from "../../utils/parse-range.js";
|
|
10
|
+
import { logger } from "../../utils/logger.js";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Resolve source parameters from the query string.
|
|
14
|
+
* Prefers a registered `sourceKey`; falls back to inline `sourceType`+`source`.
|
|
15
|
+
*
|
|
16
|
+
* @param {import("fastify").FastifyRequest["query"]} query
|
|
17
|
+
* @param {ReturnType<import("../../store/source-registry.js").createSourceRegistry>} sourceRegistry
|
|
18
|
+
* @returns {{ sourceType: string, source: string }}
|
|
19
|
+
*/
|
|
20
|
+
function getSourceParams(query, sourceRegistry) {
|
|
21
|
+
const sourceKey = typeof query.sourceKey === "string" ? query.sourceKey : "";
|
|
22
|
+
const sourceTypeFromQuery = typeof query.sourceType === "string" ? query.sourceType : "";
|
|
23
|
+
const sourceFromQuery = typeof query.source === "string" ? query.source : "";
|
|
24
|
+
|
|
25
|
+
const sourceRecord = sourceKey ? sourceRegistry.get(sourceKey) : null;
|
|
26
|
+
const sourceType = sourceRecord?.sourceType ?? sourceTypeFromQuery;
|
|
27
|
+
const source = sourceRecord?.source ?? sourceFromQuery;
|
|
28
|
+
return { sourceType, source };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* How long a request may wait for a torrent that is still being added.
|
|
33
|
+
*
|
|
34
|
+
* Adding a magnet takes as long as its metadata does — seconds when peers
|
|
35
|
+
* answer, forever when none do — and `getTorrent` waits for it. Awaiting that
|
|
36
|
+
* with no bound is what made this route answer nothing at all.
|
|
37
|
+
*/
|
|
38
|
+
const TORRENT_READY_TIMEOUT_MS = 10_000;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The torrent for a source, or a `TORRENT_NOT_READY` error once the wait has
|
|
42
|
+
* gone on long enough to be worth reporting.
|
|
43
|
+
*
|
|
44
|
+
* The underlying add is NOT cancelled: it keeps running and warms the pool, so
|
|
45
|
+
* the client's next attempt is likely to find it ready.
|
|
46
|
+
*
|
|
47
|
+
* @param {import("../../services/torrent-pool.js").TorrentPool} torrentPool
|
|
48
|
+
* @param {string} sourceType
|
|
49
|
+
* @param {string} source
|
|
50
|
+
* @returns {Promise<import("webtorrent").Torrent>}
|
|
51
|
+
*/
|
|
52
|
+
async function waitForTorrent(torrentPool, sourceType, source) {
|
|
53
|
+
let timer = null;
|
|
54
|
+
const expiry = new Promise((_resolve, reject) => {
|
|
55
|
+
timer = setTimeout(() => {
|
|
56
|
+
const error = new Error("Torrent metadata is not available yet.");
|
|
57
|
+
error.code = "TORRENT_NOT_READY";
|
|
58
|
+
reject(error);
|
|
59
|
+
}, TORRENT_READY_TIMEOUT_MS);
|
|
60
|
+
timer.unref?.();
|
|
61
|
+
});
|
|
62
|
+
try {
|
|
63
|
+
return await Promise.race([torrentPool.getTorrent(sourceType, source), expiry]);
|
|
64
|
+
} finally {
|
|
65
|
+
if (timer) {
|
|
66
|
+
clearTimeout(timer);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Stream a torrent file over HTTP with byte-range support.
|
|
73
|
+
*
|
|
74
|
+
* GET /stream
|
|
75
|
+
*
|
|
76
|
+
* @param {import("fastify").FastifyRequest} req
|
|
77
|
+
* @param {import("fastify").FastifyReply} reply
|
|
78
|
+
* @param {{ sourceRegistry: ReturnType<import("../../store/source-registry.js").createSourceRegistry>, torrentPool: import("../../services/torrent-pool.js").TorrentPool }} deps
|
|
79
|
+
* @returns {Promise<void>}
|
|
80
|
+
*/
|
|
81
|
+
export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool, noteInputBytes = null }) {
|
|
82
|
+
const fileIndexRaw = typeof req.query.fileIndex === "string" ? req.query.fileIndex : "";
|
|
83
|
+
const fileIndex = Number(fileIndexRaw);
|
|
84
|
+
const { sourceType, source } = getSourceParams(req.query, sourceRegistry);
|
|
85
|
+
|
|
86
|
+
if (!sourceType || !source || !Number.isInteger(fileIndex) || fileIndex < 0) {
|
|
87
|
+
return reply
|
|
88
|
+
.code(400)
|
|
89
|
+
.send({ error: "sourceKey or sourceType+source with fileIndex are required." });
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
let torrent;
|
|
93
|
+
try {
|
|
94
|
+
torrent = await waitForTorrent(torrentPool, sourceType, source);
|
|
95
|
+
} catch (error) {
|
|
96
|
+
if (error instanceof Error && error.code === "TORRENT_NOT_READY") {
|
|
97
|
+
// Say so, rather than holding the connection until the client gives up.
|
|
98
|
+
// Reproduced 2026-08-04 on a magnet whose metadata never arrived: both a
|
|
99
|
+
// ranged GET and a HEAD returned nothing at all for the full 30 s the
|
|
100
|
+
// probe was willing to wait, and the route had written neither a status
|
|
101
|
+
// nor a header — from the client that is indistinguishable from the proxy
|
|
102
|
+
// having died, and it left no trace in the log either.
|
|
103
|
+
reply.header("Retry-After", "1");
|
|
104
|
+
return reply.code(503).send({ error: "Torrent metadata is not available yet." });
|
|
105
|
+
}
|
|
106
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
107
|
+
return reply.code(500).send({ error: `Failed to load torrent source: ${message}` });
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const file = torrent.files[fileIndex];
|
|
111
|
+
if (!file) {
|
|
112
|
+
return reply.code(404).send({ error: "File index was not found in torrent." });
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// HEAD asks what a GET would return, not for the bytes. Fastify serves HEAD
|
|
116
|
+
// from this same handler, which used to mean a HEAD started a read of the
|
|
117
|
+
// WHOLE file: the body was discarded by Node, but the read ran on, the
|
|
118
|
+
// response never finished, and the next request on that keep-alive connection
|
|
119
|
+
// waited behind it. Measured on the field host: the keyframe-index HEAD
|
|
120
|
+
// returned headers in 23 ms and then held the connection until its 15 s
|
|
121
|
+
// timeout, which is where the 73 s transcode-session create went.
|
|
122
|
+
if (req.method === "HEAD") {
|
|
123
|
+
// Written to the raw response on purpose. Answering through `reply.send()`
|
|
124
|
+
// with no payload makes Fastify set `content-length: 0`, which is worse
|
|
125
|
+
// than useless here: the keyframe index asks for the file size with this
|
|
126
|
+
// very request and treats 0 as "no index available", silently falling back
|
|
127
|
+
// to an invented segment grid. Hijacking leaves the response to us, and
|
|
128
|
+
// Node omits the body for HEAD by itself.
|
|
129
|
+
reply.hijack();
|
|
130
|
+
reply.raw.writeHead(200, {
|
|
131
|
+
"Accept-Ranges": "bytes",
|
|
132
|
+
"Content-Type": "application/octet-stream",
|
|
133
|
+
"Content-Length": String(file.length),
|
|
134
|
+
"Content-Disposition": `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`
|
|
135
|
+
});
|
|
136
|
+
reply.raw.end();
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const releaseFile = torrentPool.acquireFile(torrent, fileIndex);
|
|
141
|
+
|
|
142
|
+
const range = parseRange(req.headers.range, file.length);
|
|
143
|
+
// Prioritize the pieces at this read position so a seek (a request at a new
|
|
144
|
+
// byte offset) downloads first instead of waiting behind the sequential
|
|
145
|
+
// backlog — this is what caused ~15-18 s stalls when seeking into an
|
|
146
|
+
// undownloaded region.
|
|
147
|
+
// Only the encoder's own input read tracks where the viewer is. Everything
|
|
148
|
+
// else that comes through here — the codec probe, the keyframe index, a
|
|
149
|
+
// subtitle fetch — reads the file's edges, and treating those as a viewer
|
|
150
|
+
// position made every session start and every encoder restart look like a
|
|
151
|
+
// burst of seeks (measured: two spurious "the viewer moved" per start).
|
|
152
|
+
torrentPool.prioritizeByteRange(torrent, fileIndex, range ? range.start : 0, undefined, {
|
|
153
|
+
wholeFileRead: range === null,
|
|
154
|
+
isPlaybackRead: req.query.reader === "playback"
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
const start = range ? range.start : 0;
|
|
158
|
+
const end = range ? range.end : file.length - 1;
|
|
159
|
+
const contentLength = end - start + 1;
|
|
160
|
+
|
|
161
|
+
// Written straight out of the torrent's shared memory when that is available:
|
|
162
|
+
// no copy on either thread, at the cost of doing the writing by hand, because
|
|
163
|
+
// only the write callback tells us when a piece may be released. Falls back to
|
|
164
|
+
// the ordinary stream for sources without a shared pool.
|
|
165
|
+
// How far ahead of its own read head this reader should ask the swarm for.
|
|
166
|
+
// Supplied by whoever knows the media's byte rate — the transcode session
|
|
167
|
+
// puts it on the ffmpeg input URL, sized in seconds of playback — because
|
|
168
|
+
// this thread knows only bytes, and 32 MB is half a minute of a 1080p film
|
|
169
|
+
// but four seconds of a disc remux. Absent or unusable, the reader's own
|
|
170
|
+
// default stands.
|
|
171
|
+
const windowBytesRaw = Number(req.query.windowBytes);
|
|
172
|
+
const windowBytes =
|
|
173
|
+
Number.isFinite(windowBytesRaw) && windowBytesRaw > 0 ? windowBytesRaw : undefined;
|
|
174
|
+
|
|
175
|
+
// Which transcode session this read feeds, when it feeds one. Put on the URL
|
|
176
|
+
// by the session that builds it, because this route otherwise knows only a
|
|
177
|
+
// file — and two sessions can read one file, so the file cannot stand in for
|
|
178
|
+
// the session.
|
|
179
|
+
const sessionId = typeof req.query.session === "string" ? req.query.session : "";
|
|
180
|
+
|
|
181
|
+
const fragments = typeof file.createFragmentReader === "function"
|
|
182
|
+
? file.createFragmentReader({ start, end, windowBytes })
|
|
183
|
+
: null;
|
|
184
|
+
|
|
185
|
+
if (fragments) {
|
|
186
|
+
reply.hijack();
|
|
187
|
+
reply.raw.writeHead(range ? 206 : 200, {
|
|
188
|
+
"Accept-Ranges": "bytes",
|
|
189
|
+
"Content-Type": "application/octet-stream",
|
|
190
|
+
"Content-Length": String(contentLength),
|
|
191
|
+
"Content-Disposition": `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`,
|
|
192
|
+
...(range ? { "Content-Range": `bytes ${start}-${end}/${file.length}` } : {})
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
// A client that goes away mid-response must stop the read, or pieces keep
|
|
196
|
+
// being fetched for nobody.
|
|
197
|
+
reply.raw.once("close", () => fragments.cancel());
|
|
198
|
+
|
|
199
|
+
let sent = 0;
|
|
200
|
+
try {
|
|
201
|
+
for await (const fragment of fragments) {
|
|
202
|
+
if (reply.raw.writableEnded || reply.raw.destroyed) {
|
|
203
|
+
fragment.release();
|
|
204
|
+
break;
|
|
205
|
+
}
|
|
206
|
+
await new Promise((resolve, reject) => {
|
|
207
|
+
reply.raw.write(fragment.bytes, (error) => (error ? reject(error) : resolve()));
|
|
208
|
+
});
|
|
209
|
+
sent += fragment.bytes.length;
|
|
210
|
+
// Say so, if this read belongs to a transcode session. It is the only
|
|
211
|
+
// proof that a session which has produced nothing yet is nevertheless
|
|
212
|
+
// being fed: the encoder's own progress cannot move until its first
|
|
213
|
+
// frame is decoded, and a viewer waiting for that first frame was being
|
|
214
|
+
// told the proxy had died while the swarm was delivering to it. Field
|
|
215
|
+
// 2026-09-03: 46.3 s on one piece, `processedSeconds` frozen at the
|
|
216
|
+
// start position throughout, and the browser gave up 0.4 s before the
|
|
217
|
+
// piece landed.
|
|
218
|
+
if (noteInputBytes) {
|
|
219
|
+
noteInputBytes(sessionId, fragment.bytes.length);
|
|
220
|
+
}
|
|
221
|
+
// Only now are these bytes gone: the piece can be unpinned, and the
|
|
222
|
+
// slot it occupies reused. Releasing before this point corrupts the
|
|
223
|
+
// response silently.
|
|
224
|
+
fragment.release();
|
|
225
|
+
}
|
|
226
|
+
// THE BODY MUST BE AS LONG AS THE HEADER PROMISED, and nothing checked.
|
|
227
|
+
//
|
|
228
|
+
// `Content-Length` is committed before the first byte, and the reader can
|
|
229
|
+
// finish early in silence: its `close()` ends the iteration with no
|
|
230
|
+
// accounting, and only the `fail()` path is logged. A client then has a
|
|
231
|
+
// response shorter than declared, and ffmpeg's mp4 demuxer — which holds
|
|
232
|
+
// the sample table and asks for samples past what arrived — starts
|
|
233
|
+
// parsing at wrong offsets. That is exactly `Invalid NAL unit size` with
|
|
234
|
+
// a negative length and `missing picture in access unit`: 2138 of them in
|
|
235
|
+
// one field session on a COPIED picture, where no encoder touches a frame.
|
|
236
|
+
//
|
|
237
|
+
// A single clean read of the same file through this route produced none,
|
|
238
|
+
// and four concurrent ones produced none; what the field session also had
|
|
239
|
+
// was a piece store whose readers wanted every piece it could hold, and
|
|
240
|
+
// 100 of 1395 evictions took a piece a reader had declared. So this says
|
|
241
|
+
// whether the body was short — which either names the cause or removes
|
|
242
|
+
// the last candidate.
|
|
243
|
+
if (sent !== contentLength) {
|
|
244
|
+
logger.error(
|
|
245
|
+
`stream: read of "${file.name}" bytes ${start}-${end} ENDED SHORT — ` +
|
|
246
|
+
`${sent} of ${contentLength} bytes sent under a Content-Length that promised all of them. ` +
|
|
247
|
+
"Whatever is reading this has a truncated body and no way to know it."
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
reply.raw.end();
|
|
251
|
+
} catch (error) {
|
|
252
|
+
// The body is already committed by its headers, so there is nothing
|
|
253
|
+
// useful to send instead — drop the connection and let the client retry.
|
|
254
|
+
// But say why: swallowing this made the route close connections with no
|
|
255
|
+
// status and no trace, which from the client looks like the proxy died
|
|
256
|
+
// and from the log looks like nothing happened at all.
|
|
257
|
+
// WHOSE end it was. A write cancelled because the consumer went away is
|
|
258
|
+
// the ordinary end of a read: ffmpeg is terminated on every seek and
|
|
259
|
+
// whenever the look-ahead bound suspends it, and its connection closes
|
|
260
|
+
// with it. Reported as a failure, that line fired several times a minute
|
|
261
|
+
// during healthy playback — and on 2026-08-09 it was read as the cause of
|
|
262
|
+
// broken audio, which it was not. A read that ends because the reader
|
|
263
|
+
// left is not a fault and must not be dressed as one; anything else is.
|
|
264
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
265
|
+
const consumerLeft = req.raw.aborted || reply.raw.destroyed || /ECANCELED|EPIPE|ERR_STREAM_DESTROYED/.test(message);
|
|
266
|
+
const line =
|
|
267
|
+
`stream: read of "${file.name}" bytes ${start}-${end} ended after ` +
|
|
268
|
+
`${sent} of ${contentLength} bytes: ${message}`;
|
|
269
|
+
if (consumerLeft) {
|
|
270
|
+
logger.debug(`${line} (the reader disconnected — expected on an encoder restart)`);
|
|
271
|
+
} else {
|
|
272
|
+
logger.warn(line);
|
|
273
|
+
}
|
|
274
|
+
reply.raw.destroy();
|
|
275
|
+
} finally {
|
|
276
|
+
releaseFile();
|
|
277
|
+
}
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
reply.header("Accept-Ranges", "bytes");
|
|
282
|
+
reply.header("Content-Type", "application/octet-stream");
|
|
283
|
+
reply.header("Content-Disposition", `inline; filename*=UTF-8''${encodeURIComponent(file.name)}`);
|
|
284
|
+
|
|
285
|
+
if (!range) {
|
|
286
|
+
reply.header("Content-Length", String(file.length));
|
|
287
|
+
const stream = file.createReadStream();
|
|
288
|
+
bindRelease(stream, reply, releaseFile);
|
|
289
|
+
return reply.send(stream);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
reply.code(206);
|
|
293
|
+
reply.header("Content-Length", String(contentLength));
|
|
294
|
+
reply.header("Content-Range", `bytes ${start}-${end}/${file.length}`);
|
|
295
|
+
const stream = file.createReadStream({ start, end });
|
|
296
|
+
bindRelease(stream, reply, releaseFile);
|
|
297
|
+
return reply.send(stream);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Attach event listeners that release the file reference exactly once when
|
|
302
|
+
* the stream or the underlying HTTP connection closes.
|
|
303
|
+
*
|
|
304
|
+
* @param {import("node:stream").Readable} stream
|
|
305
|
+
* @param {import("fastify").FastifyReply} reply
|
|
306
|
+
* @param {() => void} release
|
|
307
|
+
* @returns {void}
|
|
308
|
+
*/
|
|
309
|
+
function bindRelease(stream, reply, release) {
|
|
310
|
+
let released = false;
|
|
311
|
+
const releaseOnce = () => {
|
|
312
|
+
if (released) {
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
released = true;
|
|
316
|
+
release();
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
stream.on("close", releaseOnce);
|
|
320
|
+
stream.on("end", releaseOnce);
|
|
321
|
+
stream.on("error", releaseOnce);
|
|
322
|
+
reply.raw.once("close", releaseOnce);
|
|
323
|
+
reply.raw.once("finish", releaseOnce);
|
|
324
|
+
}
|