@torrent-tv/proxy 2.9.88 → 2.9.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/package.json +1 -1
- package/routes/transcode/session-file/get.js +17 -0
- package/services/hls-session-manager.js +27 -0
- package/services/torrent-pool.js +31 -123
- package/services/torrent-worker/piece-reader.js +232 -43
- package/test/read-window.test.js +242 -0
- package/test/segment-hold-supersede.test.js +93 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
## 2.9.89
|
|
2
|
+
|
|
3
|
+
- **Fix**: What the torrent downloads is now decided by the readers, and by nobody else. Three places were claiming pieces for the same file and overwriting each other on every request: `acquireFile` selected the whole file, `prioritizeByteRange` selected from the read position to the end, and the reader selected its entire requested range. The reader's claim was the worst of the three — ffmpeg opens its input as `bytes <position>-<EOF>`, so the first read of a session claimed the **whole file** and marked **every piece critical**, and nothing ever gave it back, because that read is abandoned a second later when ffmpeg seeks. No later prioritisation could outrank a permanent whole-file claim, which is why 2.9.88 changed nothing measurable. Each read now holds a moving window ahead of its own head, as a **stream selection** — the kind WebTorrent counts rather than merges, so several parallel readers (the codec probe's head and tail, subtitles, one input per viewer) produce the union of their windows — and releases it on completion, cancellation and abandonment. `critical` marks only the piece being waited for and at most two more, which is the rule WebTorrent's own reader uses and what the flag is supposed to mean. `prioritizeByteRange` keeps only what readers cannot do: the read position for the resume figures, and the jump log line.
|
|
4
|
+
- **Fix**: A seek releases the segment requests it made pointless. hls.js keeps one fragment load outstanding, so a request being held for the old position blocks the one for the new position — measured 2026-08-04: a backward seek into fully downloaded data waited **57 s** for a held request for `#609` to run out the 60 s hold, then fetched the segment it wanted in 15 ms. The same hold trapped 45 requests at once during a forward seek. A viewer seek now ends every wait that started before it with a retryable 503, as `hls-media-server` does (`research/hls-seek-prior-art-2026-08-02.md`, prescribed there and never built).
|
|
5
|
+
|
|
1
6
|
## 2.9.88
|
|
2
7
|
|
|
3
8
|
- **Fix**: A seek no longer makes the swarm walk the file to get there. Two faults, both confirmed by running WebTorrent's own selection code on the numbers of a measured session (588 pieces, download at 38.4%, seek to 89.1%). First: a selection carries an `offset` — how many pieces from its start are already downloaded — and the picker scans from `from + offset`; `deselect` subtracts an interval and copies that offset into what survives, so demoting the pieces behind the playhead left `{523-587, offset 226}`, a selection whose scan begins at piece 749 of 587. The seek target ended up wanted by nobody. The range is now re-selected right after the demotion, which replaces the dead entry with a fresh one starting at the playhead. Second: a request with no byte range was reported as an ordinary read at offset 0, and ffmpeg opens its input with exactly such a request and abandons it as soon as it seeks — as do the keyframe index and the codec probe, four of them around every encoder restart. Each one re-selected the whole file from piece zero, undoing the seek; the picker then skipped what was on disk and downloaded forward from the first hole. Measured cost of the pair: a seek to 89.1% of a 4.7 GB film fetched **2.47 GB over 93 s** where one 8 MiB piece was needed. A range-less read now sets the read position only when nothing else has.
|
package/package.json
CHANGED
|
@@ -75,6 +75,13 @@ export async function handleTranscodeSessionFileGet(req, reply, { hlsSessionMana
|
|
|
75
75
|
if (result.kind === "not-found") {
|
|
76
76
|
return reply.code(404).send({ error: "Transcode session file was not found." });
|
|
77
77
|
}
|
|
78
|
+
if (result.kind === "superseded") {
|
|
79
|
+
// The viewer moved while this was being held. Answer at once so the player
|
|
80
|
+
// can ask for where it is now; `Retry-After: 0` because there is nothing to
|
|
81
|
+
// wait for — this segment is simply no longer the one being watched.
|
|
82
|
+
reply.header("Retry-After", "0");
|
|
83
|
+
return reply.code(503).send({ error: "Superseded by a seek." });
|
|
84
|
+
}
|
|
78
85
|
if (result.kind === "warming-up") {
|
|
79
86
|
// The segment is still being produced (e.g. just after a seek-restart).
|
|
80
87
|
// Return a retryable 503 — never 202, which hls.js cannot consume as a
|
|
@@ -122,11 +129,21 @@ async function waitForSessionFile(hlsSessionManager, sessionId, fileName, timeou
|
|
|
122
129
|
// again — see HlsSessionManager#ensureEncodingFor for the encoder ping-pong
|
|
123
130
|
// this prevents when one seek-bar scrub fires several segment requests.
|
|
124
131
|
const requestSeq = hlsSessionManager.nextRequestSeq(sessionId);
|
|
132
|
+
// The viewer's position when this request was made. A seek makes every held
|
|
133
|
+
// request stale — it asks for a segment nobody is going to watch — and hls.js
|
|
134
|
+
// keeps only ONE fragment load outstanding, so holding on blocks the request
|
|
135
|
+
// the player actually needs now. Measured: 57 s of a 58 s backward seek was
|
|
136
|
+
// this wait, and the segment the viewer wanted took 15 ms once it was asked
|
|
137
|
+
// for.
|
|
138
|
+
const seekEpoch = hlsSessionManager.seekEpoch(sessionId);
|
|
125
139
|
while (Date.now() - startedAt < timeoutMs) {
|
|
126
140
|
const result = await hlsSessionManager.getFileStream(sessionId, fileName, { requestSeq });
|
|
127
141
|
if (result.kind !== "warming-up") {
|
|
128
142
|
return result;
|
|
129
143
|
}
|
|
144
|
+
if (hlsSessionManager.seekEpoch(sessionId) !== seekEpoch) {
|
|
145
|
+
return { kind: "superseded" };
|
|
146
|
+
}
|
|
130
147
|
await delay(300);
|
|
131
148
|
}
|
|
132
149
|
return { kind: "warming-up" };
|
|
@@ -1134,6 +1134,9 @@ export class HlsSessionManager {
|
|
|
1134
1134
|
// from one scrub cannot take turns steering the encoder.
|
|
1135
1135
|
requestSeqCounter: 0,
|
|
1136
1136
|
latestRequestSeq: 0,
|
|
1137
|
+
// Bumped by every viewer seek; a held segment request that started under
|
|
1138
|
+
// an older value gives up at once. See requestSeek.
|
|
1139
|
+
waitEpoch: 0,
|
|
1137
1140
|
seekFirstFarAt: 0,
|
|
1138
1141
|
// Circuit breaker: consecutive FAST failures (see SEEK_FAST_FAIL_MS) at
|
|
1139
1142
|
// seekFailureTarget. Reset whenever a run starts at a DIFFERENT target or
|
|
@@ -2233,6 +2236,16 @@ export class HlsSessionManager {
|
|
|
2233
2236
|
if (!session || session.state === "disposed") {
|
|
2234
2237
|
return false;
|
|
2235
2238
|
}
|
|
2239
|
+
// Every segment request being held right now was made for the position the
|
|
2240
|
+
// viewer has just left. Release them: hls.js keeps ONE fragment load
|
|
2241
|
+
// outstanding, so until the one in flight answers, the player cannot ask
|
|
2242
|
+
// for the segment it now needs — measured 2026-08-04, a backward seek into
|
|
2243
|
+
// fully-downloaded data waited 57 s for a held request for #609 to time
|
|
2244
|
+
// out, then fetched the segment it wanted in 15 ms. Bumping the epoch makes
|
|
2245
|
+
// those waits answer "retry" on their next poll instead of running out the
|
|
2246
|
+
// 60 s hold. Prescribed by `hls-media-server` (one outstanding wait per
|
|
2247
|
+
// session) in research/hls-seek-prior-art-2026-08-02.md.
|
|
2248
|
+
session.waitEpoch = (session.waitEpoch ?? 0) + 1;
|
|
2236
2249
|
const index = this.#segmentIndexForTime(session, positionSeconds);
|
|
2237
2250
|
const head = session.encodeStartIndex;
|
|
2238
2251
|
const processed = Number.isFinite(session.progress?.processedSeconds)
|
|
@@ -2435,6 +2448,20 @@ export class HlsSessionManager {
|
|
|
2435
2448
|
return session.requestSeqCounter;
|
|
2436
2449
|
}
|
|
2437
2450
|
|
|
2451
|
+
/**
|
|
2452
|
+
* How many times the viewer has moved since this session started.
|
|
2453
|
+
*
|
|
2454
|
+
* A request being held for a segment answers "retry" as soon as this changes,
|
|
2455
|
+
* because it was made for a position the viewer has left — see `requestSeek`.
|
|
2456
|
+
*
|
|
2457
|
+
* @param {string} sessionId
|
|
2458
|
+
* @returns {number}
|
|
2459
|
+
*/
|
|
2460
|
+
seekEpoch(sessionId) {
|
|
2461
|
+
const session = isSafeSessionId(sessionId) ? this.sessionsById.get(sessionId) : null;
|
|
2462
|
+
return session?.waitEpoch ?? 0;
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2438
2465
|
/**
|
|
2439
2466
|
* Open a read stream for an HLS segment or playlist file from a session.
|
|
2440
2467
|
*
|
package/services/torrent-pool.js
CHANGED
|
@@ -306,17 +306,6 @@ export class TorrentPool {
|
|
|
306
306
|
*/
|
|
307
307
|
#readPositionByTorrent = new Map();
|
|
308
308
|
|
|
309
|
-
/**
|
|
310
|
-
* Lowest piece currently selected for download, per torrent and fileIndex.
|
|
311
|
-
*
|
|
312
|
-
* Needed because selection is not readable back from WebTorrent, and a seek
|
|
313
|
-
* backward has to know whether the pieces it wants were deselected by an
|
|
314
|
-
* earlier seek forward.
|
|
315
|
-
*
|
|
316
|
-
* @type {Map<import("webtorrent").Torrent, Map<number, number>>}
|
|
317
|
-
*/
|
|
318
|
-
#selectedFromPiece = new Map();
|
|
319
|
-
|
|
320
309
|
/** Global disk cap in bytes (0 = disabled). */
|
|
321
310
|
#maxDiskBytes = 0;
|
|
322
311
|
|
|
@@ -982,8 +971,20 @@ export class TorrentPool {
|
|
|
982
971
|
}
|
|
983
972
|
|
|
984
973
|
/**
|
|
985
|
-
*
|
|
986
|
-
*
|
|
974
|
+
* Drop the pieces of files nobody is reading from the download set.
|
|
975
|
+
*
|
|
976
|
+
* It does NOT select the files that ARE in use, and that is the point. What a
|
|
977
|
+
* file needs is decided by the readers walking it: each one claims a moving
|
|
978
|
+
* window around its own read head and gives it back when it ends (see
|
|
979
|
+
* `torrent-worker/piece-reader.js`). Selecting the whole file here as well
|
|
980
|
+
* put a second, contradictory claim on the same pieces — one that covered
|
|
981
|
+
* everything and therefore always outranked the window — and it was re-made
|
|
982
|
+
* on every single `/stream` request, so a seek's prioritisation survived at
|
|
983
|
+
* most until the next one. Measured consequence: a seek to 89.1% of a 4.7 GB
|
|
984
|
+
* film waited 93 s while the swarm fetched 2.47 GB in file order.
|
|
985
|
+
*
|
|
986
|
+
* A file with no reader is deselected outright, which is what stops a torrent
|
|
987
|
+
* downloading files the viewer never opened.
|
|
987
988
|
*
|
|
988
989
|
* @param {import("webtorrent").Torrent} torrent
|
|
989
990
|
* @param {Map<number, number>} usage - fileIndex → refCount.
|
|
@@ -995,14 +996,7 @@ export class TorrentPool {
|
|
|
995
996
|
}
|
|
996
997
|
for (let index = 0; index < torrent.files.length; index += 1) {
|
|
997
998
|
const file = torrent.files[index];
|
|
998
|
-
if (!file) {
|
|
999
|
-
continue;
|
|
1000
|
-
}
|
|
1001
|
-
const shouldSelect = (usage.get(index) ?? 0) > 0;
|
|
1002
|
-
if (shouldSelect) {
|
|
1003
|
-
if (typeof file.select === "function") {
|
|
1004
|
-
file.select();
|
|
1005
|
-
}
|
|
999
|
+
if (!file || (usage.get(index) ?? 0) > 0) {
|
|
1006
1000
|
continue;
|
|
1007
1001
|
}
|
|
1008
1002
|
if (typeof file.deselect === "function") {
|
|
@@ -1012,42 +1006,27 @@ export class TorrentPool {
|
|
|
1012
1006
|
}
|
|
1013
1007
|
|
|
1014
1008
|
/**
|
|
1015
|
-
*
|
|
1016
|
-
* downloads the seek target first instead of waiting behind the sequential
|
|
1017
|
-
* backlog (which caused ~15-18 s stalls when seeking into an undownloaded
|
|
1018
|
-
* region). Called on every range request.
|
|
1009
|
+
* Record where a file is being read from.
|
|
1019
1010
|
*
|
|
1020
|
-
*
|
|
1011
|
+
* This used to also decide what the torrent should download, and that was the
|
|
1012
|
+
* mistake: it was one of THREE places claiming pieces for the same file — the
|
|
1013
|
+
* whole-file `file.select()` in `#syncSelections`, this method, and the reader
|
|
1014
|
+
* itself — and they overwrote each other on every request. The claim now
|
|
1015
|
+
* belongs to the reader alone, which holds a moving window around its own read
|
|
1016
|
+
* head and gives it back when it ends
|
|
1017
|
+
* (`torrent-worker/piece-reader.js`); several readers on one file therefore
|
|
1018
|
+
* produce the union of their windows instead of the last caller's opinion.
|
|
1021
1019
|
*
|
|
1022
|
-
*
|
|
1023
|
-
*
|
|
1024
|
-
*
|
|
1025
|
-
*
|
|
1026
|
-
*
|
|
1027
|
-
* capacity goes to the pieces the player needs next. This only STOPS
|
|
1028
|
-
* fetching the gap; already-downloaded pieces stay on disk (deleting them
|
|
1029
|
-
* is Disk hygiene Level 2), and a later backward seek re-selects the region
|
|
1030
|
-
* via this same call. The whole file is re-selected by `file.select()` on
|
|
1031
|
-
* the next `acquireFile`, so nothing is permanently dropped.
|
|
1032
|
-
*
|
|
1033
|
-
* 2. **Critical read-ahead window** — `critical(playhead, playhead+window)`.
|
|
1034
|
-
* `critical` does not reorder the scan; it enables HOTSWAP (re-request a
|
|
1035
|
-
* block from a faster peer when a slow one reserved it) over the near
|
|
1036
|
-
* window. Reset first so criticality stays a moving window rather than
|
|
1037
|
-
* accumulating over the whole file across seeks.
|
|
1038
|
-
*
|
|
1039
|
-
* Scope: single active reader per file (≈100% today). The multi-viewer union
|
|
1040
|
-
* window — demote only where behind for ALL sessions — is deferred (roadmap
|
|
1041
|
-
* item 23); here the latest read position wins.
|
|
1042
|
-
*
|
|
1043
|
-
* The pinned head/tail (prefetchFileEdges, codec probe) is downloaded up front
|
|
1044
|
-
* and lives forward of the playhead (tail) or is already on disk (head), so
|
|
1045
|
-
* demotion never costs the probe its data.
|
|
1020
|
+
* What is left here is bookkeeping the readers cannot do: `getFileStats`
|
|
1021
|
+
* reports how much of the window ahead of the read head is still missing, so
|
|
1022
|
+
* the viewer can be shown how long a resume will take, and a jump in the read
|
|
1023
|
+
* position is logged because a seek that never reaches the torrent is
|
|
1024
|
+
* invisible otherwise.
|
|
1046
1025
|
*
|
|
1047
1026
|
* @param {import("webtorrent").Torrent} torrent
|
|
1048
1027
|
* @param {number} fileIndex
|
|
1049
1028
|
* @param {number} byteStart - Start offset within the file.
|
|
1050
|
-
* @param {number} [windowBytes] -
|
|
1029
|
+
* @param {number} [windowBytes] - Unused; kept so callers need not change.
|
|
1051
1030
|
* @param {{ wholeFileRead?: boolean }} [options] - `wholeFileRead` marks a
|
|
1052
1031
|
* request that carried no byte range, i.e. one that merely opens the file at
|
|
1053
1032
|
* 0 rather than asking to read from there. See the guard below.
|
|
@@ -1060,24 +1039,17 @@ export class TorrentPool {
|
|
|
1060
1039
|
windowBytes = PRIORITY_WINDOW_BYTES,
|
|
1061
1040
|
options = {}
|
|
1062
1041
|
) {
|
|
1063
|
-
if (!torrent ||
|
|
1064
|
-
return;
|
|
1065
|
-
}
|
|
1066
|
-
const pieceLength = Number(torrent.pieceLength);
|
|
1067
|
-
if (!Number.isFinite(pieceLength) || pieceLength <= 0) {
|
|
1042
|
+
if (!torrent || !Array.isArray(torrent.files)) {
|
|
1068
1043
|
return;
|
|
1069
1044
|
}
|
|
1070
1045
|
const file = torrent.files[fileIndex];
|
|
1071
1046
|
if (!file) {
|
|
1072
1047
|
return;
|
|
1073
1048
|
}
|
|
1074
|
-
const fileOffset = Number.isFinite(file.offset) ? file.offset : 0;
|
|
1075
1049
|
const fileLength = Number(file.length);
|
|
1076
1050
|
if (!Number.isFinite(fileLength) || fileLength <= 0) {
|
|
1077
1051
|
return;
|
|
1078
1052
|
}
|
|
1079
|
-
const fileStartPiece = Math.floor(fileOffset / pieceLength);
|
|
1080
|
-
const fileEndPiece = Math.floor((fileOffset + fileLength - 1) / pieceLength);
|
|
1081
1053
|
|
|
1082
1054
|
const safeStart = Math.max(0, Number(byteStart) || 0);
|
|
1083
1055
|
|
|
@@ -1120,70 +1092,6 @@ export class TorrentPool {
|
|
|
1120
1092
|
);
|
|
1121
1093
|
}
|
|
1122
1094
|
|
|
1123
|
-
const absStart = fileOffset + safeStart;
|
|
1124
|
-
const playheadPiece = Math.floor(absStart / pieceLength);
|
|
1125
|
-
const absWindowEnd = Math.min(
|
|
1126
|
-
fileOffset + fileLength - 1,
|
|
1127
|
-
absStart + Math.max(1, windowBytes) - 1
|
|
1128
|
-
);
|
|
1129
|
-
const windowEndPiece = Math.floor(absWindowEnd / pieceLength);
|
|
1130
|
-
|
|
1131
|
-
// (1) Re-select from the playhead when it moved BACK behind what an earlier
|
|
1132
|
-
// seek deselected. `deselect` removes pieces from the download set, and
|
|
1133
|
-
// `critical` does NOT put them back — it only flags pieces already
|
|
1134
|
-
// selected. So without this, a seek forward followed by a seek backward
|
|
1135
|
-
// leaves the target pieces wanted by nobody: the encoder waits on data
|
|
1136
|
-
// the torrent was told to stop fetching, and waits forever.
|
|
1137
|
-
// Only on a backward move, so repeat calls do not pile up selections.
|
|
1138
|
-
let selectedFrom = this.#selectedFromPiece.get(torrent)?.get(fileIndex);
|
|
1139
|
-
if (selectedFrom === undefined || playheadPiece < selectedFrom) {
|
|
1140
|
-
try {
|
|
1141
|
-
torrent.select(playheadPiece, fileEndPiece, 1);
|
|
1142
|
-
} catch {
|
|
1143
|
-
// Best effort — never break streaming because selection failed.
|
|
1144
|
-
}
|
|
1145
|
-
let perFile = this.#selectedFromPiece.get(torrent);
|
|
1146
|
-
if (!perFile) {
|
|
1147
|
-
perFile = new Map();
|
|
1148
|
-
this.#selectedFromPiece.set(torrent, perFile);
|
|
1149
|
-
}
|
|
1150
|
-
perFile.set(fileIndex, playheadPiece);
|
|
1151
|
-
selectedFrom = playheadPiece;
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
// (2) Demote the gap behind the playhead so the picker scans forward from
|
|
1155
|
-
// the read position. Only when there IS a gap (not at the file start).
|
|
1156
|
-
if (playheadPiece > fileStartPiece && typeof torrent.deselect === "function") {
|
|
1157
|
-
try {
|
|
1158
|
-
torrent.deselect(fileStartPiece, playheadPiece - 1);
|
|
1159
|
-
// `deselect` subtracts the interval and copies the selection's `offset`
|
|
1160
|
-
// — how many pieces from its start are already downloaded — into what
|
|
1161
|
-
// remains. The picker scans from `from + offset`, so the surviving
|
|
1162
|
-
// selection starts scanning far past its own end and can never yield a
|
|
1163
|
-
// piece: measured with the library's own `Selections`, deselecting
|
|
1164
|
-
// 0-522 from `{0-587, offset 226}` leaves `{523-587, offset 226}`,
|
|
1165
|
-
// i.e. a scan starting at piece 749 of 587. The seek target ends up
|
|
1166
|
-
// wanted by nobody. Re-selecting the same range replaces that dead
|
|
1167
|
-
// entry with a fresh one whose offset is 0.
|
|
1168
|
-
torrent.select(playheadPiece, fileEndPiece, 1);
|
|
1169
|
-
this.#selectedFromPiece.get(torrent)?.set(fileIndex, playheadPiece);
|
|
1170
|
-
} catch {
|
|
1171
|
-
// Best effort — never break streaming because demotion failed.
|
|
1172
|
-
}
|
|
1173
|
-
}
|
|
1174
|
-
|
|
1175
|
-
// (3) Reset criticality to a moving read-ahead window (hotswap over the near
|
|
1176
|
-
// pieces), so it does not accumulate over the whole file across seeks.
|
|
1177
|
-
if (Array.isArray(torrent._critical)) {
|
|
1178
|
-
torrent._critical.length = 0;
|
|
1179
|
-
}
|
|
1180
|
-
if (windowEndPiece >= playheadPiece) {
|
|
1181
|
-
try {
|
|
1182
|
-
torrent.critical(playheadPiece, windowEndPiece);
|
|
1183
|
-
} catch {
|
|
1184
|
-
// Best effort.
|
|
1185
|
-
}
|
|
1186
|
-
}
|
|
1187
1095
|
}
|
|
1188
1096
|
|
|
1189
1097
|
/**
|
|
@@ -22,6 +22,145 @@
|
|
|
22
22
|
|
|
23
23
|
import { findSharedStore } from "../piece-store/shared-piece-store.js";
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* How far ahead of the read head pieces are asked for.
|
|
27
|
+
*
|
|
28
|
+
* A read is open-ended — ffmpeg opens its input as `bytes <position>-<EOF>` and
|
|
29
|
+
* keeps it for the whole film — so taking the requested range literally asks
|
|
30
|
+
* for everything from the seek point to the end of the file at once. That is
|
|
31
|
+
* what a seek used to do: the swarm was told the entire tail was wanted, went
|
|
32
|
+
* at it from its first missing piece, and the one piece the decoder was blocked
|
|
33
|
+
* on arrived only when the sequential scan reached it. Measured on a 4.7 GB
|
|
34
|
+
* film: a seek to 89.1% took 93 s and pulled 2.47 GB.
|
|
35
|
+
*
|
|
36
|
+
* So the reader asks for a window and moves it as it goes. The size is a
|
|
37
|
+
* compromise the caller cannot yet express: the right unit is seconds of
|
|
38
|
+
* playback (duration and size are both known — to the transcode session, not to
|
|
39
|
+
* this thread), and 32 MB is about 34 s of a 1080p film but only a few seconds
|
|
40
|
+
* of a disc remux. Sizing it from the real byte rate is a follow-up; what
|
|
41
|
+
* matters here is that it is bounded and moving rather than "to the end".
|
|
42
|
+
*/
|
|
43
|
+
const READ_WINDOW_BYTES = 32 * 1024 * 1024;
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* How many pieces past the one being waited for are marked critical.
|
|
47
|
+
*
|
|
48
|
+
* `critical` means "a reader is blocked on this now" — it is what lets a piece
|
|
49
|
+
* jump the sequential scan. Marking a whole range critical, as this did, says
|
|
50
|
+
* it about hundreds of pieces at once and the signal stops meaning anything.
|
|
51
|
+
* WebTorrent's own reader marks `min(1 MB / pieceLength, 2)` pieces, i.e. the
|
|
52
|
+
* one under the head and at most two more; the same rule is used here.
|
|
53
|
+
*
|
|
54
|
+
* @param {number} pieceLength
|
|
55
|
+
* @returns {number}
|
|
56
|
+
*/
|
|
57
|
+
function criticalRunLength(pieceLength) {
|
|
58
|
+
return Math.min(Math.floor((1024 * 1024) / Math.max(1, pieceLength)), 2);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The pieces a reader at `pieceIndex` wants next, clamped to its own range.
|
|
63
|
+
*
|
|
64
|
+
* @param {{ pieceIndex: number, lastPiece: number, windowPieces: number }} params
|
|
65
|
+
* @returns {{ from: number, to: number }}
|
|
66
|
+
*/
|
|
67
|
+
export function readWindowFor({ pieceIndex, lastPiece, windowPieces }) {
|
|
68
|
+
const span = Math.max(1, windowPieces);
|
|
69
|
+
return { from: pieceIndex, to: Math.min(lastPiece, pieceIndex + span - 1) };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Add this reader's window to the download set as a stream selection.
|
|
74
|
+
*
|
|
75
|
+
* `_select`/`_deselect` with the stream flag are what WebTorrent's own
|
|
76
|
+
* `FileIterator` uses; there is no public call for it, because the public
|
|
77
|
+
* `select` produces the merging, interval-subtracted kind whose bookkeeping
|
|
78
|
+
* cannot express "one of several readers wants this". Falls back to the public
|
|
79
|
+
* call if a future version drops the private one.
|
|
80
|
+
*
|
|
81
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
82
|
+
* @param {{ from: number, to: number }} window
|
|
83
|
+
* @returns {void}
|
|
84
|
+
*/
|
|
85
|
+
function claimWindow(torrent, { from, to }) {
|
|
86
|
+
try {
|
|
87
|
+
if (typeof torrent._select === "function") {
|
|
88
|
+
torrent._select(from, to, 1, null, true);
|
|
89
|
+
} else if (typeof torrent.select === "function") {
|
|
90
|
+
torrent.select(from, to, 1);
|
|
91
|
+
}
|
|
92
|
+
} catch {
|
|
93
|
+
// Best effort — never fail a read because selection bookkeeping refused.
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Take this reader's window back out of the download set.
|
|
99
|
+
*
|
|
100
|
+
* The bounds must match the ones given to {@link claimWindow} exactly: a stream
|
|
101
|
+
* selection is removed by equality, not by overlap.
|
|
102
|
+
*
|
|
103
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
104
|
+
* @param {{ from: number, to: number }} window
|
|
105
|
+
* @returns {void}
|
|
106
|
+
*/
|
|
107
|
+
function releaseWindow(torrent, { from, to }) {
|
|
108
|
+
try {
|
|
109
|
+
if (typeof torrent._deselect === "function") {
|
|
110
|
+
torrent._deselect(from, to, true);
|
|
111
|
+
} else if (typeof torrent.deselect === "function") {
|
|
112
|
+
torrent.deselect(from, to);
|
|
113
|
+
}
|
|
114
|
+
} catch {
|
|
115
|
+
// Best effort.
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Mark the piece a reader is blocked on, clearing the mark it set before.
|
|
121
|
+
*
|
|
122
|
+
* Criticality is never cleared by WebTorrent itself, so a reader that walked a
|
|
123
|
+
* film would leave every piece of it marked. Only the indices this reader set
|
|
124
|
+
* are cleared, so a second reader's mark on the same piece is not stolen — and
|
|
125
|
+
* the flag is advisory anyway.
|
|
126
|
+
*
|
|
127
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
128
|
+
* @param {number} from
|
|
129
|
+
* @param {number} to
|
|
130
|
+
* @param {{ from: number, to: number } | null} previous
|
|
131
|
+
* @returns {{ from: number, to: number } | null}
|
|
132
|
+
*/
|
|
133
|
+
function markCritical(torrent, from, to, previous) {
|
|
134
|
+
if (previous && previous.from === from && previous.to === to) {
|
|
135
|
+
return previous;
|
|
136
|
+
}
|
|
137
|
+
if (previous) {
|
|
138
|
+
clearCritical(torrent, previous);
|
|
139
|
+
}
|
|
140
|
+
try {
|
|
141
|
+
torrent.critical?.(from, to);
|
|
142
|
+
} catch {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
return { from, to };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Drop critical marks this reader set.
|
|
150
|
+
*
|
|
151
|
+
* @param {import("webtorrent").Torrent} torrent
|
|
152
|
+
* @param {{ from: number, to: number }} mark
|
|
153
|
+
* @returns {void}
|
|
154
|
+
*/
|
|
155
|
+
function clearCritical(torrent, { from, to }) {
|
|
156
|
+
if (!Array.isArray(torrent._critical)) {
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
for (let index = from; index <= to; index += 1) {
|
|
160
|
+
torrent._critical[index] = false;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
25
164
|
/**
|
|
26
165
|
* Wait until a piece has been downloaded and verified.
|
|
27
166
|
*
|
|
@@ -100,9 +239,19 @@ function whenPieceReady(torrent, index, cancellation) {
|
|
|
100
239
|
* @param {number} params.start - Inclusive, relative to the file.
|
|
101
240
|
* @param {number} params.end - Inclusive, relative to the file.
|
|
102
241
|
* @param {{ isCancelled: () => boolean }} params.cancellation
|
|
242
|
+
* @param {number} [params.windowBytes] - How far ahead of the read head to ask
|
|
243
|
+
* the swarm for. Defaults to {@link READ_WINDOW_BYTES}; a caller that knows
|
|
244
|
+
* the media's byte rate should size it in seconds of playback instead.
|
|
103
245
|
* @returns {AsyncGenerator<PieceFragment>}
|
|
104
246
|
*/
|
|
105
|
-
export async function* readFragments({
|
|
247
|
+
export async function* readFragments({
|
|
248
|
+
torrent,
|
|
249
|
+
fileIndex,
|
|
250
|
+
start,
|
|
251
|
+
end,
|
|
252
|
+
cancellation,
|
|
253
|
+
windowBytes = READ_WINDOW_BYTES
|
|
254
|
+
}) {
|
|
106
255
|
const store = findSharedStore(torrent);
|
|
107
256
|
if (!store) {
|
|
108
257
|
throw new Error("This torrent is not backed by a shared piece store.");
|
|
@@ -121,55 +270,95 @@ export async function* readFragments({ torrent, fileIndex, start, end, cancellat
|
|
|
121
270
|
const firstPiece = Math.floor(absoluteStart / pieceLength);
|
|
122
271
|
const lastPiece = Math.floor(absoluteEnd / pieceLength);
|
|
123
272
|
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
273
|
+
// This reader owns what it asks for, and gives it back when it is done. The
|
|
274
|
+
// window is a STREAM selection: those are removed by exact bounds and several
|
|
275
|
+
// identical ones coexist — WebTorrent's own source calls that "in a way a
|
|
276
|
+
// count" — so N readers on one torrent produce the union of their windows,
|
|
277
|
+
// and each one leaving takes away only its own. That is what makes several
|
|
278
|
+
// parallel readers (the codec probe's head and tail, subtitles, one input per
|
|
279
|
+
// viewer) cooperate instead of overwrite each other.
|
|
280
|
+
//
|
|
281
|
+
// The previous code selected the whole requested range, marked all of it
|
|
282
|
+
// critical, and never deselected anything — so ffmpeg's opening
|
|
283
|
+
// `bytes 0-<EOF>` left a permanent selection over the entire file, and no
|
|
284
|
+
// later prioritisation could outrank it.
|
|
285
|
+
const windowPieces = Math.max(1, Math.ceil(Math.max(1, windowBytes) / pieceLength));
|
|
286
|
+
const criticalRun = criticalRunLength(pieceLength);
|
|
287
|
+
/** @type {{ from: number, to: number } | null} */
|
|
288
|
+
let window = null;
|
|
289
|
+
/** @type {{ from: number, to: number } | null} */
|
|
290
|
+
let criticalMark = null;
|
|
133
291
|
|
|
134
|
-
|
|
135
|
-
|
|
292
|
+
const moveWindowTo = (pieceIndex) => {
|
|
293
|
+
const next = readWindowFor({ pieceIndex, lastPiece, windowPieces });
|
|
294
|
+
if (window && window.from === next.from && window.to === next.to) {
|
|
136
295
|
return;
|
|
137
296
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
const fromWithinPiece = Math.max(absoluteStart, pieceStart) - pieceStart;
|
|
141
|
-
const toWithinPiece = Math.min(absoluteEnd, pieceStart + pieceLength - 1) - pieceStart;
|
|
142
|
-
|
|
143
|
-
await whenPieceReady(torrent, pieceIndex, cancellation);
|
|
144
|
-
|
|
145
|
-
// Pinned BEFORE it is located, and before any await that could let an
|
|
146
|
-
// eviction run: the offset is only meaningful while the piece is held.
|
|
147
|
-
store.pin(pieceIndex);
|
|
148
|
-
let located = null;
|
|
149
|
-
try {
|
|
150
|
-
located = await store.reside(pieceIndex);
|
|
151
|
-
} catch (error) {
|
|
152
|
-
store.unpin(pieceIndex);
|
|
153
|
-
throw error;
|
|
297
|
+
if (window) {
|
|
298
|
+
releaseWindow(torrent, window);
|
|
154
299
|
}
|
|
300
|
+
claimWindow(torrent, next);
|
|
301
|
+
window = next;
|
|
302
|
+
};
|
|
155
303
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
304
|
+
try {
|
|
305
|
+
for (let pieceIndex = firstPiece; pieceIndex <= lastPiece; pieceIndex += 1) {
|
|
306
|
+
if (cancellation.isCancelled()) {
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
160
309
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
310
|
+
const pieceStart = pieceIndex * pieceLength;
|
|
311
|
+
const fromWithinPiece = Math.max(absoluteStart, pieceStart) - pieceStart;
|
|
312
|
+
const toWithinPiece = Math.min(absoluteEnd, pieceStart + pieceLength - 1) - pieceStart;
|
|
313
|
+
|
|
314
|
+
moveWindowTo(pieceIndex);
|
|
315
|
+
|
|
316
|
+
if (!torrent.bitfield?.get(pieceIndex)) {
|
|
317
|
+
// Blocked here and now — this is the one case `critical` is meant for.
|
|
318
|
+
criticalMark = markCritical(torrent, pieceIndex, Math.min(lastPiece, pieceIndex + criticalRun), criticalMark);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
await whenPieceReady(torrent, pieceIndex, cancellation);
|
|
322
|
+
|
|
323
|
+
// Pinned BEFORE it is located, and before any await that could let an
|
|
324
|
+
// eviction run: the offset is only meaningful while the piece is held.
|
|
325
|
+
store.pin(pieceIndex);
|
|
326
|
+
let located = null;
|
|
327
|
+
try {
|
|
328
|
+
located = await store.reside(pieceIndex);
|
|
329
|
+
} catch (error) {
|
|
171
330
|
store.unpin(pieceIndex);
|
|
331
|
+
throw error;
|
|
172
332
|
}
|
|
173
|
-
|
|
333
|
+
|
|
334
|
+
if (!located) {
|
|
335
|
+
store.unpin(pieceIndex);
|
|
336
|
+
throw new Error(`Piece ${pieceIndex} is verified but absent from the store.`);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
let releasedThisPiece = false;
|
|
340
|
+
yield {
|
|
341
|
+
pieceIndex,
|
|
342
|
+
offset: located.offset + fromWithinPiece,
|
|
343
|
+
length: toWithinPiece - fromWithinPiece + 1,
|
|
344
|
+
release() {
|
|
345
|
+
if (releasedThisPiece) {
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
releasedThisPiece = true;
|
|
349
|
+
store.unpin(pieceIndex);
|
|
350
|
+
}
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
} finally {
|
|
354
|
+
// Reached on completion, on cancellation, on a throw, and when the consumer
|
|
355
|
+
// stops iterating — a window left behind would keep the swarm fetching for
|
|
356
|
+
// a reader that no longer exists.
|
|
357
|
+
if (window) {
|
|
358
|
+
releaseWindow(torrent, window);
|
|
359
|
+
}
|
|
360
|
+
if (criticalMark) {
|
|
361
|
+
clearCritical(torrent, criticalMark);
|
|
362
|
+
}
|
|
174
363
|
}
|
|
175
364
|
}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file What a read asks the torrent to download, and what it gives back.
|
|
3
|
+
*
|
|
4
|
+
* A read used to select its whole requested range and never deselect it.
|
|
5
|
+
* ffmpeg opens its input as `bytes <position>-<EOF>`, so the first read of a
|
|
6
|
+
* session claimed the entire file and marked every piece of it critical — and
|
|
7
|
+
* the claim outlived the read, which is abandoned a second later when ffmpeg
|
|
8
|
+
* seeks. Nothing after that could outrank it: measured on a 4.7 GB film, a seek
|
|
9
|
+
* to 89.1% waited 93 s while the swarm fetched 2.47 GB in file order.
|
|
10
|
+
*
|
|
11
|
+
* Now a read holds a moving window and returns it when it ends. These tests pin
|
|
12
|
+
* the three properties that matter: the claim is bounded, it is given back, and
|
|
13
|
+
* several readers add up instead of overwriting each other.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import test from "node:test";
|
|
17
|
+
import assert from "node:assert/strict";
|
|
18
|
+
import { EventEmitter } from "node:events";
|
|
19
|
+
import os from "node:os";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import fs from "node:fs/promises";
|
|
22
|
+
import { readFragments, readWindowFor } from "../services/torrent-worker/piece-reader.js";
|
|
23
|
+
import { SharedPieceStore } from "../services/piece-store/shared-piece-store.js";
|
|
24
|
+
|
|
25
|
+
const PIECE = 1024;
|
|
26
|
+
// The production window is 32 MB against 8 MiB pieces — four of them. Sized
|
|
27
|
+
// here in pieces so the test does not depend on either constant.
|
|
28
|
+
const WINDOW_PIECES = 4;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* A torrent that records every selection call instead of downloading anything.
|
|
32
|
+
*
|
|
33
|
+
* @param {{ pieceCount: number, present?: (index: number) => boolean }} shape
|
|
34
|
+
*/
|
|
35
|
+
async function recordingTorrent({ pieceCount, present = () => true }) {
|
|
36
|
+
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "read-window-test-"));
|
|
37
|
+
const totalLength = pieceCount * PIECE;
|
|
38
|
+
const store = new SharedPieceStore(PIECE, {
|
|
39
|
+
length: totalLength,
|
|
40
|
+
memoryBytes: 64 * PIECE,
|
|
41
|
+
path: directory,
|
|
42
|
+
name: "test"
|
|
43
|
+
});
|
|
44
|
+
for (let index = 0; index < pieceCount; index += 1) {
|
|
45
|
+
await new Promise((resolve, reject) => {
|
|
46
|
+
store.put(index, Buffer.alloc(PIECE, index % 251), (error) => (error ? reject(error) : resolve()));
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** @type {Array<{ call: string, from: number, to: number, stream?: boolean }>} */
|
|
51
|
+
const calls = [];
|
|
52
|
+
/** Live stream selections, as WebTorrent counts them: exact bounds, duplicates allowed. */
|
|
53
|
+
const held = [];
|
|
54
|
+
|
|
55
|
+
const torrent = Object.assign(new EventEmitter(), {
|
|
56
|
+
pieceLength: PIECE,
|
|
57
|
+
store,
|
|
58
|
+
bitfield: { get: (index) => present(index) },
|
|
59
|
+
files: [{ offset: 0, length: totalLength, name: "file.bin" }],
|
|
60
|
+
_critical: [],
|
|
61
|
+
calls,
|
|
62
|
+
held,
|
|
63
|
+
_select(from, to, _priority, _notify, isStreamSelection) {
|
|
64
|
+
calls.push({ call: "select", from, to, stream: isStreamSelection === true });
|
|
65
|
+
held.push(`${from}-${to}`);
|
|
66
|
+
},
|
|
67
|
+
_deselect(from, to, isStreamSelection) {
|
|
68
|
+
calls.push({ call: "deselect", from, to, stream: isStreamSelection === true });
|
|
69
|
+
const at = held.indexOf(`${from}-${to}`);
|
|
70
|
+
if (at >= 0) {
|
|
71
|
+
held.splice(at, 1);
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
critical(from, to) {
|
|
75
|
+
calls.push({ call: "critical", from, to });
|
|
76
|
+
for (let index = from; index <= to; index += 1) {
|
|
77
|
+
this._critical[index] = true;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
return { torrent, store, directory };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Read a range to the end, releasing every fragment. */
|
|
86
|
+
async function drain(torrent, start, end) {
|
|
87
|
+
for await (const fragment of readFragments({
|
|
88
|
+
torrent,
|
|
89
|
+
fileIndex: 0,
|
|
90
|
+
start,
|
|
91
|
+
end,
|
|
92
|
+
cancellation: { isCancelled: () => false }
|
|
93
|
+
})) {
|
|
94
|
+
fragment.release();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
test("the window is bounded and clamped to the end of the read", () => {
|
|
99
|
+
assert.deepEqual(readWindowFor({ pieceIndex: 10, lastPiece: 999, windowPieces: 4 }), { from: 10, to: 13 });
|
|
100
|
+
assert.deepEqual(
|
|
101
|
+
readWindowFor({ pieceIndex: 997, lastPiece: 999, windowPieces: 4 }),
|
|
102
|
+
{ from: 997, to: 999 },
|
|
103
|
+
"the window never reaches past the range the reader was given"
|
|
104
|
+
);
|
|
105
|
+
assert.deepEqual(
|
|
106
|
+
readWindowFor({ pieceIndex: 5, lastPiece: 999, windowPieces: 0 }),
|
|
107
|
+
{ from: 5, to: 5 },
|
|
108
|
+
"a degenerate size still asks for the piece under the head"
|
|
109
|
+
);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("an open-ended read does not claim the whole file at once", async () => {
|
|
113
|
+
// 8000 pieces of 1 KB — far more than the 32 MB window, so a read to the end
|
|
114
|
+
// of the file is exactly the ffmpeg case.
|
|
115
|
+
const { torrent, store, directory } = await recordingTorrent({ pieceCount: 8000 });
|
|
116
|
+
try {
|
|
117
|
+
// Read only the first two pieces, but ask as ffmpeg does: to the last byte.
|
|
118
|
+
const iterator = readFragments({
|
|
119
|
+
torrent,
|
|
120
|
+
fileIndex: 0,
|
|
121
|
+
start: 0,
|
|
122
|
+
end: 8000 * PIECE - 1,
|
|
123
|
+
cancellation: { isCancelled: () => false },
|
|
124
|
+
windowBytes: WINDOW_PIECES * PIECE
|
|
125
|
+
});
|
|
126
|
+
const first = await iterator.next();
|
|
127
|
+
first.value.release();
|
|
128
|
+
|
|
129
|
+
const selects = torrent.calls.filter((entry) => entry.call === "select");
|
|
130
|
+
assert.ok(selects.length >= 1, "the reader claimed nothing");
|
|
131
|
+
const claimed = selects[0].to - selects[0].from + 1;
|
|
132
|
+
assert.equal(
|
|
133
|
+
claimed,
|
|
134
|
+
WINDOW_PIECES,
|
|
135
|
+
`the reader claimed ${claimed} pieces of the file instead of its window`
|
|
136
|
+
);
|
|
137
|
+
assert.equal(selects[0].stream, true, "the claim must be a stream selection, so it can be counted");
|
|
138
|
+
|
|
139
|
+
await iterator.return();
|
|
140
|
+
} finally {
|
|
141
|
+
store.destroy(() => undefined);
|
|
142
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("a finished read leaves nothing selected", async () => {
|
|
147
|
+
const { torrent, store, directory } = await recordingTorrent({ pieceCount: 40 });
|
|
148
|
+
try {
|
|
149
|
+
await drain(torrent, 0, 40 * PIECE - 1);
|
|
150
|
+
assert.deepEqual(torrent.held, [], "the read kept its claim after finishing");
|
|
151
|
+
} finally {
|
|
152
|
+
store.destroy(() => undefined);
|
|
153
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test("an abandoned read leaves nothing selected", async () => {
|
|
158
|
+
const { torrent, store, directory } = await recordingTorrent({ pieceCount: 8000 });
|
|
159
|
+
try {
|
|
160
|
+
const iterator = readFragments({
|
|
161
|
+
torrent,
|
|
162
|
+
fileIndex: 0,
|
|
163
|
+
start: 0,
|
|
164
|
+
end: 8000 * PIECE - 1,
|
|
165
|
+
cancellation: { isCancelled: () => false },
|
|
166
|
+
windowBytes: WINDOW_PIECES * PIECE
|
|
167
|
+
});
|
|
168
|
+
const first = await iterator.next();
|
|
169
|
+
first.value.release();
|
|
170
|
+
// What ffmpeg does to its opening read the moment it seeks.
|
|
171
|
+
await iterator.return();
|
|
172
|
+
|
|
173
|
+
assert.deepEqual(torrent.held, [], "an abandoned read kept its claim forever");
|
|
174
|
+
} finally {
|
|
175
|
+
store.destroy(() => undefined);
|
|
176
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
177
|
+
}
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("two readers add up, and one leaving takes only its own window", async () => {
|
|
181
|
+
const { torrent, store, directory } = await recordingTorrent({ pieceCount: 8000 });
|
|
182
|
+
try {
|
|
183
|
+
const head = readFragments({
|
|
184
|
+
torrent, fileIndex: 0, start: 0, end: 8000 * PIECE - 1,
|
|
185
|
+
cancellation: { isCancelled: () => false }
|
|
186
|
+
});
|
|
187
|
+
const tail = readFragments({
|
|
188
|
+
torrent, fileIndex: 0, start: 4000 * PIECE, end: 8000 * PIECE - 1,
|
|
189
|
+
cancellation: { isCancelled: () => false }
|
|
190
|
+
});
|
|
191
|
+
(await head.next()).value.release();
|
|
192
|
+
(await tail.next()).value.release();
|
|
193
|
+
|
|
194
|
+
assert.equal(torrent.held.length, 2, "the two readers did not both hold a window");
|
|
195
|
+
const [headWindow, tailWindow] = torrent.held;
|
|
196
|
+
|
|
197
|
+
await tail.return();
|
|
198
|
+
assert.deepEqual(
|
|
199
|
+
torrent.held,
|
|
200
|
+
[headWindow],
|
|
201
|
+
`leaving reader took the wrong window (expected to remove ${tailWindow})`
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
await head.return();
|
|
205
|
+
assert.deepEqual(torrent.held, []);
|
|
206
|
+
} finally {
|
|
207
|
+
store.destroy(() => undefined);
|
|
208
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
test("criticality marks the piece being waited for, not the whole range", async () => {
|
|
213
|
+
// Nothing is present, so the reader blocks on its first piece and marks it.
|
|
214
|
+
let arrived = false;
|
|
215
|
+
const { torrent, store, directory } = await recordingTorrent({
|
|
216
|
+
pieceCount: 8000,
|
|
217
|
+
present: () => arrived
|
|
218
|
+
});
|
|
219
|
+
try {
|
|
220
|
+
const iterator = readFragments({
|
|
221
|
+
torrent, fileIndex: 0, start: 0, end: 8000 * PIECE - 1,
|
|
222
|
+
cancellation: { isCancelled: () => false }
|
|
223
|
+
});
|
|
224
|
+
const pending = iterator.next();
|
|
225
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
226
|
+
|
|
227
|
+
const criticals = torrent.calls.filter((entry) => entry.call === "critical");
|
|
228
|
+
assert.equal(criticals.length, 1);
|
|
229
|
+
assert.ok(
|
|
230
|
+
criticals[0].to - criticals[0].from + 1 <= 3,
|
|
231
|
+
`marked ${criticals[0].to - criticals[0].from + 1} pieces critical; the signal means "blocked here now"`
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
arrived = true;
|
|
235
|
+
torrent.emit("verified", 0);
|
|
236
|
+
(await pending).value.release();
|
|
237
|
+
await iterator.return();
|
|
238
|
+
} finally {
|
|
239
|
+
store.destroy(() => undefined);
|
|
240
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
241
|
+
}
|
|
242
|
+
});
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file A held segment request must not outlive the position it was made for.
|
|
3
|
+
*
|
|
4
|
+
* hls.js keeps ONE fragment load outstanding. So a request being held for a
|
|
5
|
+
* segment blocks the request for wherever the viewer has just moved to, and our
|
|
6
|
+
* route held each one for 60 s. Measured 2026-08-04: a backward seek into fully
|
|
7
|
+
* downloaded data waited 57 s for a held request for `#609` to run out its
|
|
8
|
+
* timer, and the segment the viewer actually wanted was then served in 15 ms.
|
|
9
|
+
*
|
|
10
|
+
* `research/hls-seek-prior-art-2026-08-02.md` prescribed this guard from
|
|
11
|
+
* `hls-media-server` — one outstanding wait per session — and it was never
|
|
12
|
+
* built.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import test from "node:test";
|
|
16
|
+
import assert from "node:assert/strict";
|
|
17
|
+
import { handleTranscodeSessionFileGet } from "../routes/transcode/session-file/get.js";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* A reply that records what the route answered.
|
|
21
|
+
*
|
|
22
|
+
* @returns {{ reply: object, sent: { code: number, headers: Record<string, string>, body: unknown } }}
|
|
23
|
+
*/
|
|
24
|
+
function recordingReply() {
|
|
25
|
+
const sent = { code: 200, headers: {}, body: undefined };
|
|
26
|
+
const reply = {
|
|
27
|
+
code(value) {
|
|
28
|
+
sent.code = value;
|
|
29
|
+
return reply;
|
|
30
|
+
},
|
|
31
|
+
header(name, value) {
|
|
32
|
+
sent.headers[name.toLowerCase()] = String(value);
|
|
33
|
+
return reply;
|
|
34
|
+
},
|
|
35
|
+
send(body) {
|
|
36
|
+
sent.body = body;
|
|
37
|
+
return reply;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
return { reply, sent };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const request = (fileName) => ({
|
|
44
|
+
params: { sessionId: "11111111-2222-3333-4444-555555555555", fileName },
|
|
45
|
+
raw: { on() {}, off() {} }
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("a seek releases a held segment request instead of running out the hold", async () => {
|
|
49
|
+
let epoch = 0;
|
|
50
|
+
let polls = 0;
|
|
51
|
+
const hlsSessionManager = {
|
|
52
|
+
nextRequestSeq: () => 1,
|
|
53
|
+
seekEpoch: () => epoch,
|
|
54
|
+
async getFileStream() {
|
|
55
|
+
polls += 1;
|
|
56
|
+
// The viewer moves while this request is being held.
|
|
57
|
+
if (polls === 2) {
|
|
58
|
+
epoch += 1;
|
|
59
|
+
}
|
|
60
|
+
return { kind: "warming-up" };
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const { reply, sent } = recordingReply();
|
|
65
|
+
const startedAt = Date.now();
|
|
66
|
+
await handleTranscodeSessionFileGet(request("segment-00609.mp4"), reply, { hlsSessionManager });
|
|
67
|
+
const heldMs = Date.now() - startedAt;
|
|
68
|
+
|
|
69
|
+
assert.equal(sent.code, 503, "the player must get a retryable answer, not a stream");
|
|
70
|
+
assert.equal(sent.headers["retry-after"], "0", "nothing to wait for — this segment is not being watched");
|
|
71
|
+
assert.ok(heldMs < 5_000, `the request was held ${heldMs}ms after the seek`);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test("without a seek the request is still held until the segment appears", async () => {
|
|
75
|
+
let polls = 0;
|
|
76
|
+
const hlsSessionManager = {
|
|
77
|
+
nextRequestSeq: () => 1,
|
|
78
|
+
seekEpoch: () => 7,
|
|
79
|
+
async getFileStream() {
|
|
80
|
+
polls += 1;
|
|
81
|
+
if (polls < 3) {
|
|
82
|
+
return { kind: "warming-up" };
|
|
83
|
+
}
|
|
84
|
+
return { kind: "ok", contentType: "video/mp4", stream: "bytes", isPlaylist: false };
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const { reply, sent } = recordingReply();
|
|
89
|
+
await handleTranscodeSessionFileGet(request("segment-00610.mp4"), reply, { hlsSessionManager });
|
|
90
|
+
|
|
91
|
+
assert.equal(sent.body, "bytes", "a segment that arrives late must still be served");
|
|
92
|
+
assert.equal(sent.headers["content-type"], "video/mp4");
|
|
93
|
+
});
|