@torrent-tv/proxy 2.91.0 → 2.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/bin/cli.js +3 -2
  2. package/package.json +18 -2
  3. package/routes/api/playback-plan/post.js +21 -4
  4. package/routes/api/sources/files/get.js +20 -28
  5. package/routes/api/sources/post.js +4 -2
  6. package/routes/api/sources/stats/get.js +4 -11
  7. package/routes/api/sources/viewer/post.js +16 -0
  8. package/routes/api/sources/warm/post.js +6 -215
  9. package/routes/api/subtitles/get.js +22 -125
  10. package/routes/api/subtitles/post.js +26 -0
  11. package/routes/api/transcode-sessions/post.js +24 -2
  12. package/routes/media/get.js +80 -0
  13. package/routes/stream/get.js +26 -94
  14. package/routes/transcode/audio-warm/get.js +9 -21
  15. package/routes/transcode/variant-warm/get.js +8 -14
  16. package/server.js +410 -46
  17. package/services/encode/AdmittedInput.js +136 -0
  18. package/services/encode/CushionReport.js +1 -76
  19. package/services/encode/EncodeInputs.js +142 -0
  20. package/services/encode/EncodeOrchestrator.js +4 -1
  21. package/services/encode/EncodeRun.js +13 -4
  22. package/services/encode/EncodeRuns.js +132 -52
  23. package/services/encode/InputFailures.js +37 -0
  24. package/services/encode/MatroskaInput.js +158 -0
  25. package/services/encode/OutputOpening.js +16 -64
  26. package/services/encode/Renditions.js +1 -17
  27. package/services/encode/admitted-command.js +60 -0
  28. package/services/encode/annex-b-input.js +37 -0
  29. package/services/encode/bit-payload.js +29 -0
  30. package/services/encode/output/OutputCatalog.js +1 -1
  31. package/services/encode/output/Timeline.js +11 -0
  32. package/services/encode/pcm-normalization.js +30 -0
  33. package/services/encode/piece-completeness.js +54 -1
  34. package/services/encode/quality/HostLoad.js +19 -73
  35. package/services/encode/quality/QualityOffer.js +1 -1
  36. package/services/encode/run-command.js +1 -589
  37. package/services/encode/segment-formats/mp4-boxes.js +3 -1
  38. package/services/encode/segment-formats/presentation-segment.js +114 -0
  39. package/services/media/ContainerOrchestrator.js +118 -48
  40. package/services/media/KeyframeTables.js +16 -115
  41. package/services/media/MediaReadRequests.js +120 -0
  42. package/services/media/ProbeRequests.js +135 -0
  43. package/services/media/SegmentInputs.js +48 -0
  44. package/services/media/SourceFile.js +6 -0
  45. package/services/media/SubtitleOrchestrator.js +190 -4
  46. package/services/media/await-plan.js +47 -0
  47. package/services/media/container/AsfContainer.js +217 -0
  48. package/services/media/container/AviContainer.js +343 -34
  49. package/services/media/container/CadencedPictureOrder.js +45 -0
  50. package/services/media/container/Container.js +7 -3
  51. package/services/media/container/ContainerFactory.js +10 -3
  52. package/services/media/container/FfprobeContainer.js +222 -0
  53. package/services/media/container/MatroskaContainer.js +72 -46
  54. package/services/media/container/Mp4Container.js +472 -139
  55. package/services/media/container/MpegPsContainer.js +281 -0
  56. package/services/media/container/MpegTsContainer.js +277 -0
  57. package/services/media/container/PacketIndex.js +270 -0
  58. package/services/media/container/PacketRecords.js +237 -0
  59. package/services/media/container/RetainedBytes.js +39 -0
  60. package/services/media/container/RetainedReads.js +51 -0
  61. package/services/media/container/ac3-frame.js +46 -0
  62. package/services/media/container/asf-packets.js +153 -0
  63. package/services/media/container/avc-picture-order.js +50 -0
  64. package/services/media/container/avi-open-dml.js +47 -0
  65. package/services/media/container/dts-frame.js +51 -0
  66. package/services/media/container/ebml-stream.js +12 -1
  67. package/services/media/container/ffprobe-record.js +45 -0
  68. package/services/media/container/flv-packet-ranges.js +27 -0
  69. package/services/media/container/following-packet-time.js +37 -0
  70. package/services/media/container/h264-configuration.js +183 -0
  71. package/services/media/container/h264-elementary-index.js +245 -0
  72. package/services/media/container/hevc-configuration.js +165 -0
  73. package/services/media/container/hevc-elementary-index.js +132 -0
  74. package/services/media/container/hevc-picture-order.js +66 -0
  75. package/services/media/container/latm-frame.js +99 -0
  76. package/services/media/container/matroska-packets.js +207 -0
  77. package/services/media/container/memory-unavailable.js +8 -0
  78. package/services/media/container/mp3-packet-dependencies.js +41 -0
  79. package/services/media/container/mp4-fragments.js +148 -0
  80. package/services/media/container/mpeg-audio-frame.js +19 -0
  81. package/services/media/container/mpeg-audio-reservoir.js +27 -0
  82. package/services/media/container/mpeg-elementary-index.js +196 -0
  83. package/services/media/container/mpeg-sequence-reader.js +69 -0
  84. package/services/media/container/mpeg-ts-packets.js +93 -0
  85. package/services/media/container/mpeg-video-configuration.js +50 -0
  86. package/services/media/container/mpeg4-picture-timing.js +108 -0
  87. package/services/media/container/pes-header.js +34 -0
  88. package/services/media/playback-declarations.js +57 -0
  89. package/services/media/playback-planner.js +72 -230
  90. package/services/media/probe-packets.js +61 -0
  91. package/services/media/read-playback-declarations.js +18 -0
  92. package/services/media/tracks/AudioTrack.js +9 -0
  93. package/services/media/tracks/TextSubtitleTrack.js +4 -1
  94. package/services/server/OutputLifecycle.js +2 -0
  95. package/services/server/SegmentServing.js +40 -33
  96. package/services/server/ViewerRequests.js +22 -41
  97. package/services/server/controllers/PlaybackController.js +4 -0
  98. package/services/server/controllers/SubtitleController.js +8 -29
  99. package/services/server/transcode-session-files.js +58 -99
  100. package/services/server/wire-outputs.js +58 -15
  101. package/services/storage/IndexMemory.js +85 -0
  102. package/services/storage/MachineBudget.js +27 -3
  103. package/services/storage/allowance.js +8 -1
  104. package/services/storage/piece-store/piece-disk-store.js +18 -3
  105. package/services/storage/piece-store/piece-lru.js +8 -4
  106. package/services/storage/piece-store/shared-piece-store.js +26 -11
  107. package/services/storage/segment-store/SegmentStore.js +49 -18
  108. package/services/storage/wire.js +7 -0
  109. package/services/torrent/Contents.js +52 -20
  110. package/services/torrent/demand/DemandRegister.js +13 -4
  111. package/services/torrent/demand/Window.js +13 -2
  112. package/services/torrent/demand-progress.js +25 -0
  113. package/services/torrent/download/DeadlineScheduler.js +167 -0
  114. package/services/torrent/download/FutureDownload.js +166 -0
  115. package/services/torrent/download/SwarmSelection.js +30 -59
  116. package/services/torrent/download/registry.js +45 -5
  117. package/services/torrent/download/withdraw-claim.js +2 -0
  118. package/services/torrent/torrent-pool.js +39 -600
  119. package/services/torrent/worker/channel.js +8 -1
  120. package/services/torrent/worker/client.js +74 -72
  121. package/services/torrent/worker/files-in-use.js +9 -0
  122. package/services/torrent/worker/handle-state.js +12 -0
  123. package/services/torrent/worker/held-bytes.js +82 -99
  124. package/services/torrent/worker/held-file-bytes.js +17 -0
  125. package/services/torrent/worker/piece-reader.js +67 -683
  126. package/services/torrent/worker/pool-adapter.js +37 -123
  127. package/services/torrent/worker/protocol.js +6 -17
  128. package/services/torrent/worker/whole-file-stats.js +2 -0
  129. package/services/torrent/worker/worker.js +86 -156
  130. package/services/transport/data-channel-handler.js +56 -13
  131. package/services/viewer/DownloadMaps.js +168 -0
  132. package/services/viewer/NativeSourceMap.js +33 -0
  133. package/services/viewer/PriorityMap.js +49 -40
  134. package/services/viewer/PriorityOrchestrator.js +42 -13
  135. package/services/viewer/SourcePreparation.js +278 -0
  136. package/services/viewer/Viewer.js +11 -0
  137. package/services/viewer/Viewers.js +109 -2
  138. package/services/viewer/playback-readiness.js +50 -63
  139. package/services/viewer/segment-demands.js +37 -0
  140. package/services/viewer/urgent-output-ready.js +26 -0
  141. package/services/encode/run-inputs.js +0 -53
  142. package/services/media/keyframe-probe.js +0 -157
  143. package/services/media/media-info-probe.js +0 -93
  144. package/services/torrent/worker/background-fill.js +0 -213
  145. package/services/torrent/worker/resume-warm.js +0 -107
@@ -21,84 +21,14 @@
21
21
  */
22
22
 
23
23
  import { pieceStoreOf } from "../piece-store-of.js";
24
- import { bytesOf, Urgency, urgencyName } from "../demand/index.js";
24
+ import { urgencyName } from "../demand/index.js";
25
25
  import { demandFor } from "../download/registry.js";
26
26
  import { logger } from "../../../utils/logger.js";
27
- import {
28
- askFastestWiresFor,
29
- canPlaceRequests,
30
- describePieceTail,
31
- duplicateTailFor
32
- } from "./fastest-wires.js";
33
27
  import { minimumBufferFrom, requiredSpeedFrom } from "../supply-margin.js";
34
28
 
35
29
  /** Only waits at least this long are reported; sequential reading stays silent. */
36
30
  const PIECE_WAIT_LOG_MS = 1_000;
37
31
 
38
- /** Distinguishes concurrent readers to the piece store. Never reused. */
39
- let readerSequence = 0;
40
-
41
- /**
42
- * How far ahead of the read head pieces are asked for.
43
- *
44
- * A read is open-ended — ffmpeg opens its input as `bytes <position>-<EOF>` and
45
- * keeps it for the whole film — so taking the requested range literally asks
46
- * for everything from the seek point to the end of the file at once. That is
47
- * what a seek used to do: the swarm was told the entire tail was wanted, went
48
- * at it from its first missing piece, and the one piece the decoder was blocked
49
- * on arrived only when the sequential scan reached it. Measured on a 4.7 GB
50
- * film: a seek to 89.1% took 93 s and pulled 2.47 GB.
51
- *
52
- * So the reader asks for a window and moves it as it goes. The size is a
53
- * compromise the caller cannot yet express: the right unit is seconds of
54
- * playback (duration and size are both known — to the transcode session, not to
55
- * this thread), and 32 MB is about 34 s of a 1080p film but only a few seconds
56
- * of a disc remux. Sizing it from the real byte rate is a follow-up; what
57
- * matters here is that it is bounded and moving rather than "to the end".
58
- */
59
- const READ_WINDOW_BYTES = 32 * 1024 * 1024;
60
-
61
- /**
62
- * The pieces a reader at `pieceIndex` wants next, clamped to its own range.
63
- *
64
- * @param {{ pieceIndex: number, lastPiece: number, windowPieces: number }} params
65
- * @returns {{ from: number, to: number }}
66
- */
67
- export function readWindowFor({ pieceIndex, lastPiece, windowPieces }) {
68
- const span = Math.max(1, windowPieces);
69
- return { from: pieceIndex, to: Math.min(lastPiece, pieceIndex + span - 1) };
70
- }
71
-
72
- /**
73
- * How wide the window should be after a piece that made the reader wait — or
74
- * did not.
75
- *
76
- * The swarm's surplus is what pays for this. Measured 2026-08-17 on the field
77
- * torrent: 5.1-5.9 MB/s delivered against a film consumed at about 1 MB/s, and
78
- * the reader still blocked 47 times in two minutes, median 1.5 s, worst 4.5 s.
79
- * A fivefold surplus never became distance ahead of the head, because the
80
- * window is a fixed number of seconds of playback and everything past it is
81
- * ordinary background fill at no priority.
82
- *
83
- * So the window follows the evidence: every wait that mattered widens it by a
84
- * piece, every piece that was already there narrows it back toward the size the
85
- * caller asked for. Nothing here is chosen — the wait is measured, the
86
- * threshold is the one that already defines "a wait worth recording", and the
87
- * ceiling is this reader's share of the store's memory, so widening can never
88
- * cost more than the store can hold.
89
- *
90
- * @param {{ current: number, base: number, ceiling: number, waitedMs: number, waitThresholdMs: number }} params
91
- * @returns {number}
92
- */
93
- export function nextWindowPieces({ current, base, ceiling, waitedMs, waitThresholdMs }) {
94
- const floor = Math.max(1, Math.floor(base));
95
- const top = Math.max(floor, Math.floor(ceiling));
96
- const now = Math.min(top, Math.max(floor, Math.floor(current)));
97
- if (waitedMs >= waitThresholdMs) {
98
- return Math.min(top, now + 1);
99
- }
100
- return Math.max(floor, now - 1);
101
- }
102
32
 
103
33
  /**
104
34
  * Who is working on the piece a reader is blocked on, right now.
@@ -147,12 +77,14 @@ export function pieceSupply(torrent, pieceIndex) {
147
77
  * @param {{ isCancelled: () => boolean, onCancel?: (listener: () => void) => () => void }} cancellation
148
78
  * @returns {Promise<void>}
149
79
  */
150
- function whenPieceReady(torrent, index, cancellation) {
80
+ function whenPieceReady(torrent, index, cancellation, wanted) {
151
81
  if (torrent.bitfield?.get(index)) {
152
82
  return Promise.resolve();
153
83
  }
154
84
 
155
85
  return new Promise((resolve, reject) => {
86
+ let stopListening = () => {};
87
+ let finished = false;
156
88
  /** @param {number} verifiedIndex */
157
89
  const onVerified = (verifiedIndex) => {
158
90
  if (verifiedIndex === index) {
@@ -160,6 +92,14 @@ function whenPieceReady(torrent, index, cancellation) {
160
92
  resolve();
161
93
  }
162
94
  };
95
+ const onMapChanged = () => {
96
+ if (wanted()) return;
97
+ cleanup();
98
+ const error = new Error(`Piece ${index} is outside the source priority map.`);
99
+ error.code = "SOURCE_RANGE_NOT_WANTED";
100
+ error.canRetry = false;
101
+ reject(error);
102
+ };
163
103
  const onDestroyed = () => {
164
104
  cleanup();
165
105
  reject(new Error(`Torrent went away while waiting for piece ${index}.`));
@@ -172,25 +112,31 @@ function whenPieceReady(torrent, index, cancellation) {
172
112
  reject(new Error(`Read cancelled while waiting for piece ${index}.`));
173
113
  return;
174
114
  }
175
- const stopListening = cancellation.onCancel?.(() => {
176
- cleanup();
177
- reject(new Error(`Read cancelled while waiting for piece ${index}.`));
178
- }) ?? (() => {});
179
-
180
115
  function cleanup() {
116
+ finished = true;
181
117
  stopListening();
182
118
  torrent.removeListener("verified", onVerified);
183
119
  torrent.removeListener("close", onDestroyed);
120
+ torrent.removeListener("priority-map-changed", onMapChanged);
184
121
  }
185
122
 
186
123
  torrent.on("verified", onVerified);
187
124
  torrent.once("close", onDestroyed);
125
+ torrent.on("priority-map-changed", onMapChanged);
126
+ stopListening = cancellation.onCancel?.(() => {
127
+ cleanup();
128
+ reject(new Error(`Read cancelled while waiting for piece ${index}.`));
129
+ }) ?? (() => {});
130
+ if (finished) {
131
+ stopListening();
132
+ return;
133
+ }
188
134
 
189
135
  // The piece may have arrived between the check above and this listener.
190
136
  if (torrent.bitfield?.get(index)) {
191
137
  cleanup();
192
138
  resolve();
193
- }
139
+ } else onMapChanged();
194
140
  });
195
141
  }
196
142
 
@@ -217,9 +163,6 @@ function whenPieceReady(torrent, index, cancellation) {
217
163
  * @param {number} params.start - Inclusive, relative to the file.
218
164
  * @param {number} params.end - Inclusive, relative to the file.
219
165
  * @param {{ isCancelled: () => boolean }} params.cancellation
220
- * @param {number} [params.windowBytes] - How far ahead of the read head to ask
221
- * the swarm for. Defaults to {@link READ_WINDOW_BYTES}; a caller that knows
222
- * the media's byte rate should size it in seconds of playback instead.
223
166
  * @returns {AsyncGenerator<PieceFragment>}
224
167
  */
225
168
  /**
@@ -290,20 +233,6 @@ export function supplyFiguresFor(infoHash, fileName, segmentSeconds) {
290
233
  };
291
234
  }
292
235
 
293
- /**
294
- * Waits split by whether the blocked piece was steered onto another peer.
295
- *
296
- * The steering is visible per wait already — how many peers held the piece, how
297
- * many were asked, what the tail looked like. What was NOT visible is what it
298
- * bought, and that cannot be read off one line: it is the difference between
299
- * the waits where a second peer was asked and the waits where none could be.
300
- * Kept per file, reported with the same summary, so a session says by number
301
- * whether the swap shortens the tail instead of leaving it to impression.
302
- *
303
- * @type {Map<string, { swapped: number[], alone: number[] }>}
304
- */
305
- const waitsBySteering = new Map();
306
-
307
236
  /**
308
237
  * Record one wait against the priority level of the requested piece. The level
309
238
  * identifies which part of the priority map needs more lead time.
@@ -357,41 +286,6 @@ function describeWaitLevels(key) {
357
286
  */
358
287
  const waitsByLevel = new Map();
359
288
 
360
- function noteSteeringOutcome(key, waitedMs, steered) {
361
- let split = waitsBySteering.get(key);
362
- if (!split) {
363
- split = { swapped: [], alone: [] };
364
- waitsBySteering.set(key, split);
365
- }
366
- const into = steered ? split.swapped : split.alone;
367
- into.push(waitedMs);
368
- while (into.length > SUPPLY_WAIT_HISTORY) {
369
- into.shift();
370
- }
371
- }
372
-
373
- /**
374
- * What the split says, or null while one side of it is still empty — a
375
- * comparison needs both.
376
- *
377
- * @param {string} key
378
- * @returns {string | null}
379
- */
380
- function describeSteering(key) {
381
- const split = waitsBySteering.get(key);
382
- if (!split || split.swapped.length === 0 || split.alone.length === 0) {
383
- return null;
384
- }
385
- const middle = (values) => {
386
- const sorted = [...values].sort((left, right) => left - right);
387
- return sorted[Math.floor(sorted.length / 2)];
388
- };
389
- return (
390
- `steered ${split.swapped.length} waits median ${middle(split.swapped)}ms, ` +
391
- `unsteered ${split.alone.length} waits median ${middle(split.alone)}ms`
392
- );
393
- }
394
-
395
289
  /**
396
290
  * How many readers are blocked on a torrent AT THIS MOMENT, by infohash.
397
291
  *
@@ -503,12 +397,7 @@ function noteSupplyWait(key, label, waitedMs) {
503
397
  `one every ${demand.medianIntervalSec.toFixed(2)}s of running, ${demand.samples} stall(s) ` +
504
398
  `from ${demand.waits} wait(s)) — ` +
505
399
  `and the smallest buffer that hides it is ${buffer ? buffer.seconds.toFixed(1) : "?"}s` +
506
- // What steering the blocked piece onto another peer bought, as the
507
- // difference between the waits where it placed something and the waits
508
- // where it could not. Absent until both sides have a sample, because a
509
- // comparison with one side empty is not a comparison.
510
- (describeSteering(key) ? ` — ${describeSteering(key)}` : "") +
511
- // The comparison this release exists to make. Read it first.
400
+ // Attribute source waits to the level declared by the shared map.
512
401
  (describeWaitLevels(key) ? ` — ${describeWaitLevels(key)}` : "")
513
402
  );
514
403
  }
@@ -520,559 +409,54 @@ function noteSupplyWait(key, label, waitedMs) {
520
409
  */
521
410
  const SEGMENT_SECONDS_FOR_BUFFER = 4;
522
411
 
523
- export async function* readFragments({
524
- torrent,
525
- fileIndex,
526
- start,
527
- end,
528
- cancellation,
529
- windowBytes = READ_WINDOW_BYTES
530
- }) {
412
+ export async function* readFragments({ torrent, fileIndex, start, end, cancellation }) {
531
413
  const store = pieceStoreOf(torrent);
532
- if (!store) {
533
- throw new Error("This torrent is not backed by a shared piece store.");
534
- }
535
- // What this reader wants goes here and nowhere else. `SwarmSelection` is the
536
- // only thing that turns any of it into a request to the swarm, so a reader
537
- // can no longer contradict the background fill or the pool.
538
- const { register, selection } = demandFor(torrent);
539
-
540
414
  const file = torrent.files?.[fileIndex];
541
- if (!file) {
542
- throw new Error(`File ${fileIndex} not found.`);
543
- }
544
-
415
+ if (!store || !file) throw new Error("The source file or shared piece store is unavailable.");
545
416
  const pieceLength = torrent.pieceLength;
546
- // Piece numbers are torrent-wide, so a file's own offsets have to be lifted
547
- // into the torrent's address space first.
548
- const absoluteStart = file.offset + start;
549
- const absoluteEnd = file.offset + end;
550
- const firstPiece = Math.floor(absoluteStart / pieceLength);
551
- const lastPiece = Math.floor(absoluteEnd / pieceLength);
552
-
553
- // This reader owns what it asks for, and gives it back when it is done. The
554
- // window is a STREAM selection: those are removed by exact bounds and several
555
- // identical ones coexist — WebTorrent's own source calls that "in a way a
556
- // count" — so N readers on one torrent produce the union of their windows,
557
- // and each one leaving takes away only its own. That is what makes several
558
- // parallel readers (the codec probe's head and tail, subtitles, one input per
559
- // viewer) cooperate instead of overwrite each other.
560
- //
561
- // The previous code selected the whole requested range, marked all of it
562
- // critical, and never deselected anything — so ffmpeg's opening
563
- // `bytes 0-<EOF>` left a permanent selection over the entire file, and no
564
- // later prioritisation could outrank it.
565
- const basePieces = Math.max(1, Math.ceil(Math.max(1, windowBytes) / pieceLength));
566
- // What the window is RIGHT NOW. It starts at what the caller sized in seconds
567
- // of playback and grows while the reader keeps being made to wait — see
568
- // `nextWindowPieces`.
569
- let windowPieces = basePieces;
570
- /**
571
- * The widest this reader may go: its share of what the store can hold in
572
- * memory. Measured rather than chosen — the capacity is the store's own, and
573
- * the number of readers is how many windows are declared on it right now.
574
- *
575
- * @returns {number}
576
- */
577
- const ceilingPieces = () => {
578
- const capacity = Number(store?.capacity);
579
- if (!Number.isFinite(capacity) || capacity <= 0) {
580
- return basePieces;
581
- }
582
- const readers = Math.max(1, store.protectedRanges?.().length ?? 1);
583
- return Math.max(basePieces, Math.floor(capacity / readers));
584
- };
585
- /** @type {{ from: number, to: number } | null} */
586
- let window = null;
587
- /** @type {{ from: number, to: number } | null} */
588
- let blockedStated = false;
589
- /**
590
- * Drops the pin of the fragment currently in the consumer's hands, if it
591
- * still holds one. See where it is assigned.
592
- *
593
- * @type {(() => void) | null}
594
- */
595
- let releaseHeldPin = null;
596
-
597
- // Identity of this read, so the store can tell one reader's window from
598
- // another's. Each read gets its own; `readerSequence` never repeats within a
599
- // process.
600
- const readerId = `read-${(readerSequence += 1)}`;
601
-
602
- /**
603
- * Set when the window JUMPS, cleared by the first wait after it.
604
- *
605
- * The wait that follows a jump is the cost of the jump: the pieces at the new
606
- * position have not been asked for yet, and the encoder is restarting. It is
607
- * not evidence about how well this swarm SUSTAINS a read, which is the only
608
- * thing `requiredSpeed` is about — and letting it in is what collapsed the
609
- * quality offer 131 ms after the seek measured on 2026-08-18, refusing every
610
- * re-encoded rung on the strength of one jump.
611
- *
612
- * @type {boolean}
613
- */
614
- let waitBelongsToJump = false;
615
-
616
- /** What the consumer has taken from this read, in bytes. */
617
- let deliveredBytes = 0;
618
- // How often this read had to stop, and for how long in total. Reported at the
619
- // end whatever the outcome, so a read that never stopped is counted too.
620
- let waitCount = 0;
621
- let waitedTotalMs = 0;
622
- const readStartedAt = Date.now();
623
-
624
- /**
625
- * Where the reader's claim starts: the first piece it does not already have. Everything between the read position and that piece
626
- * is on disk or in memory, so claiming it asks the swarm for what we hold.
627
- *
628
- * @param {number} pieceIndex
629
- * @returns {number}
630
- */
631
- const firstMissingFrom = (pieceIndex) => {
632
- for (let index = pieceIndex; index <= lastPiece; index += 1) {
633
- if (!torrent.bitfield?.get?.(index)) {
634
- return index;
635
- }
636
- }
637
- return pieceIndex;
638
- };
639
-
640
-
641
- /**
642
- * State one band as a need, in bytes.
643
- *
644
- * The urgency is part of the claimant key, so this blocked-read claim can be
645
- * withdrawn without changing the priority map's claims for the same file.
646
- *
647
- * @param {{ from: number, to: number, urgency: number }} band
648
- * @returns {void}
649
- */
650
- const stateBand = (band) => {
651
- const range = bytesOf({
652
- fileOffset: Number(file.offset),
653
- fileLength: Number(file.length),
654
- from: band.from,
655
- to: band.to,
656
- pieceLength
657
- });
658
- if (!range) {
659
- return;
660
- }
661
- register.state({
662
- claimant: `${readerId}:${urgencyName(band.urgency)}`,
663
- fileIndex,
664
- byteStart: range.byteStart,
665
- byteEnd: range.byteEnd,
666
- urgency: band.urgency
667
- });
668
- };
669
-
670
- /**
671
- * Move the window the reader is working through.
672
- *
673
- * It states NOTHING to the swarm. What should be downloaded ahead of a viewer
674
- * is the priority map's answer, stated once for the whole file by the side
675
- * that knows where the viewers are; a read is consumption, not a forecast.
676
- *
677
- * A read used to declare a rolling window of its own — the piece it wanted
678
- * and two bands beyond it — and with fifteen reads on one file that was
679
- * fifteen windows on a piece store holding sixteen pieces. Half of all
680
- * evictions then took a piece a reader had declared, two thirds of reads came
681
- * back from disk, and the bytes handed out stopped being the file's: twenty-
682
- * two source-parse errors, a segment the player refused, and an empty picture
683
- * for six minutes (field 2026-09-05).
684
- *
685
- * What the window is still for: the pieces to mark critical when the reader
686
- * is actually stopped, and the range to bring back from disk after a jump.
687
- *
688
- * @param {number} pieceIndex
689
- * @returns {void}
690
- */
691
- /**
692
- * Where the priority map puts one piece, in the map's own levels.
693
- *
694
- * A stated window is file-relative — `bytesOf` subtracts the file's offset
695
- * within the torrent — so the piece is put in that frame before it is looked
696
- * up, or every answer belongs to some other part of the torrent.
697
- *
698
- * @param {number} pieceIndex
699
- * @returns {number | null}
700
- */
701
- const mapLevelOf = (pieceIndex) =>
702
- register.urgencyAt(fileIndex, pieceIndex * pieceLength - Number(file.offset));
703
-
704
- const moveWindowTo = (pieceIndex) => {
705
- const anchor = firstMissingFrom(pieceIndex);
706
- const next = readWindowFor({ pieceIndex: anchor, lastPiece, windowPieces });
707
- if (window && window.from === next.from && window.to === next.to) {
708
- return;
709
- }
710
- const isJump = !window || next.from > window.to || next.from < window.from;
711
- window = next;
712
- if (isJump) {
713
- waitBelongsToJump = true;
714
- // A jump — a seek, not the window sliding along — can land on pieces that
715
- // are already downloaded but have been spilled to disk. Bring the whole
716
- // window back at once instead of one disk round trip per piece as the
717
- // reader reaches them.
718
- const revived = store.warmRange?.(next.from, next.to) ?? 0;
719
- if (revived > 0) {
720
- logger.info(
721
- `piece-reader: reviving ${revived} spilled piece(s) of ${next.from}-${next.to} ` +
722
- `for a jump to ${(start / 1024 / 1024).toFixed(0)}MB of "${file.name}"`
723
- );
724
- }
725
- }
726
- };
727
-
728
- // How many times THIS piece has been asked for again after the store
729
- // withdrew it. Reset once a piece is in hand, so the allowance is per piece.
730
- let withdrawnRetries = 0;
731
-
417
+ const absoluteStart = file.offset + start, absoluteEnd = file.offset + end;
418
+ const first = Math.floor(absoluteStart / pieceLength), last = Math.floor(absoluteEnd / pieceLength);
419
+ const { register } = demandFor(torrent);
420
+ let release = null;
732
421
  try {
733
- for (let pieceIndex = firstPiece; pieceIndex <= lastPiece; pieceIndex += 1) {
734
- if (cancellation.isCancelled()) {
735
- return;
736
- }
737
-
738
- const pieceStart = pieceIndex * pieceLength;
739
- const fromWithinPiece = Math.max(absoluteStart, pieceStart) - pieceStart;
740
- const toWithinPiece = Math.min(absoluteEnd, pieceStart + pieceLength - 1) - pieceStart;
741
-
742
- moveWindowTo(pieceIndex);
743
-
744
- if (!torrent.bitfield?.get(pieceIndex)) {
745
- // Everything from here to the end of the window is wanted NOW, so all
746
- // of it is marked, not just the piece under the head. `critical`
747
- // enables hotswap: a block reserved by a slow peer is re-requested from
748
- // a faster one instead of holding up the reader. Measured 2026-08-04
749
- // with only the blocked piece marked, the first segment after a seek
750
- // took 7.2 s while its four 4 MB pieces arrived one after another at
751
- // ~2.2 MB/s, with waits of 1.3 s and 2.8 s on single pieces.
752
- //
753
- // This is not the old behaviour returning: that marked the whole
754
- // REQUESTED RANGE, which for ffmpeg's input means every piece to the
755
- // end of the file — hundreds of them, at which point the flag says
756
- // nothing. A window is what a reader genuinely needs next.
757
- // Stated as its own need at the level that is being waited on, which
758
- // is what carries the permission to take a block from a slow peer.
759
- // Nothing here calls the library: `reconcile` reads what is stated.
760
- stateBand({ from: pieceIndex, to: window.to, urgency: Urgency.BLOCKED });
761
- blockedStated = true;
762
- selection.reconcile();
763
- }
764
-
765
- const waitStartedAt = Date.now();
766
- // What the WHOLE torrent received while this one piece was missing. It is
767
- // the reading that separates the two causes a wait can have, and neither
768
- // could be told from the other before: bytes arriving briskly throughout
769
- // mean the swarm had capacity and this piece was stuck behind the wire
770
- // that reserved it — the blocked-piece tail; bytes barely moving mean
771
- // there was nothing to be had, and no reordering of requests would have
772
- // helped. Asked of 2026-08-31, when a session with 4-5 peers of 38 known
773
- // waited 41.32 s at worst and the log could not say which it was.
774
- const downloadedAtWaitStart = Number(torrent?.downloaded) || 0;
775
- // The reader is blocked, so this piece is now the only thing that matters
776
- // on this torrent: hand it to the fastest wires that hold it. A block is
777
- // reserved for exactly one wire, and the read ends when the slowest
778
- // holder delivers — measured 2026-08-17, the swarm had a fivefold surplus
779
- // of bandwidth and the reader still waited 1.0-4.5 s, 47 times in two
780
- // minutes, on pieces five peers already had.
781
- let pushed = { asked: 0, attempted: 0, considered: 0, fastestBytesPerSecond: 0 };
782
- // The tail as it stood at an attempt that placed NOTHING — the state the
783
- // duplication work has to answer, and the only one worth a line. Sampled
784
- // at that instant rather than once up front, because the steering runs
785
- // again every half second and the piece changes under it; the last such
786
- // reading is kept, so the line describes the most recent failure.
787
- let tailWhenNothingPlaced = null;
788
- // What duplicating the tail placed, summed over the wait. Measured
789
- // 2026-08-19 on a real swarm: the blocks a reader waits on are 2-14 of
790
- // 512 and sit on wires the library considers fast, so its own hotswap
791
- // never fires for them — see `duplicateTailFor`.
792
- let duplicated = 0;
793
- const pushToFastest = () => {
794
- try {
795
- const result = askFastestWiresFor(torrent, pieceIndex);
796
- if (result.asked === 0) {
797
- tailWhenNothingPlaced = describePieceTail(torrent, pieceIndex);
798
- }
799
- // Every attempt, not only the ones where nothing else could be placed.
800
- // The ordinary steering asks for whatever blocks are still free; the
801
- // read, meanwhile, ends when the LAST block arrives, and that block is
802
- // reserved to one wire whether or not other blocks could be asked for.
803
- // Field 2026-09-03: 46.3 s on one piece, ordinary requests placed on
804
- // 54 of 87 attempts throughout, and a tail of 3 blocks of 512 held by
805
- // wires at 51-99 KB/s to the end — while duplication, which ran only
806
- // on the 33 attempts that placed nothing, managed 5 blocks in the
807
- // whole wait. `duplicateTailFor` bounds itself by the tail's length,
808
- // so a piece that is merely still arriving is left alone.
809
- duplicated += duplicateTailFor(torrent, pieceIndex).duplicated;
810
- pushed = {
811
- asked: pushed.asked + result.asked,
812
- refusedWhileReserved:
813
- (pushed.refusedWhileReserved ?? 0) + (result.refusedWhileReserved ?? 0),
814
- // Summed like the successes, so the line compares two totals over
815
- // the same attempts instead of a total against a snapshot.
816
- attempted: (pushed.attempted ?? 0) + result.attempted,
817
- considered: result.considered,
818
- fastestBytesPerSecond: result.fastestBytesPerSecond
819
- };
820
- } catch (error) {
821
- // The entry is internal to the library; if a version changes it, this
822
- // lever stops working and that must be visible rather than silent.
823
- logger.warn(`piece-reader: could not steer piece ${pieceIndex} — ${error?.message ?? error}`);
422
+ for (let index = first; index <= last; index++) {
423
+ if (cancellation.isCancelled()) return;
424
+ const pieceStart = index * pieceLength;
425
+ const wanted = () => register.windows().some(window => window.fileIndex === fileIndex &&
426
+ window.byteStart + file.offset < pieceStart + pieceLength && window.byteEnd + file.offset >= pieceStart);
427
+ let retries = 0;
428
+ while (true) {
429
+ const waitAt = Date.now();
430
+ countBlockedReader(torrent.infoHash, 1);
431
+ try { await whenPieceReady(torrent, index, cancellation, wanted); }
432
+ finally { countBlockedReader(torrent.infoHash, -1); }
433
+ const waitedMs = Date.now() - waitAt;
434
+ if (index > first && waitedMs >= PIECE_WAIT_LOG_MS) {
435
+ const key = `${torrent.infoHash ?? "?"}/${file.name ?? "?"}`;
436
+ noteSupplyWait(key, file.name ?? "", waitedMs);
437
+ const level = register.urgencyAt(fileIndex, Math.max(0, pieceStart - file.offset));
438
+ if (level !== null) noteWaitLevel(key, waitedMs, level);
824
439
  }
825
- };
826
- if (canPlaceRequests(torrent)) {
827
- pushToFastest();
828
- } else {
829
- // Nothing can be placed at all on this build, so the tail is the whole
830
- // of the answer.
831
- tailWhenNothingPlaced = describePieceTail(torrent, pieceIndex);
832
- logger.warn(
833
- "piece-reader: this webtorrent build offers no way to place a request; " +
834
- "the blocked piece cannot be steered onto a faster peer"
835
- );
836
- }
837
- // Sampled while waiting rather than after: once the piece lands, nothing
838
- // is outstanding on it any more and every count reads zero.
839
- let supply = null;
840
- const supplyProbe = setInterval(() => {
841
- const sample = pieceSupply(torrent, pieceIndex);
842
- if (!supply || sample.blocks > supply.blocks) {
843
- supply = sample;
844
- }
845
- // Wires come and go, and their speeds change: a holder that was slow a
846
- // moment ago may now be the fastest one available.
847
- pushToFastest();
848
- }, 500);
849
- // Counted for exactly as long as this reader is inside the wait, so that
850
- // background work can stand aside while the viewer's own reading is
851
- // starving. The `finally` is what makes it safe: a cancelled or failed
852
- // read must not leave the torrent looking permanently blocked, which
853
- // would stop that background work for the rest of the session.
854
- countBlockedReader(torrent?.infoHash, 1);
855
- try {
856
- await whenPieceReady(torrent, pieceIndex, cancellation);
857
- } finally {
858
- countBlockedReader(torrent?.infoHash, -1);
859
- clearInterval(supplyProbe);
860
- }
861
- // What a reader spent waiting for data, attributed to the exact piece. A
862
- // seek's cost is dominated by the first segment after the encoder
863
- // restarts (measured 9.2-9.4 s), and without this there is no way to say
864
- // whether that is the swarm, the picker, or ffmpeg. Logged only when the
865
- // wait is long enough to matter, so ordinary sequential reading is silent.
866
- const waitedMs = Date.now() - waitStartedAt;
867
- // Where the map puts this piece, read once and used by both the
868
- // attribution below and the line beside it.
869
- const wantedBy = mapLevelOf(pieceIndex);
870
- // The window answers to what just happened: a wait means the lead was too
871
- // short, an immediate hit means it is longer than it needs to be. Applied
872
- // before the logging below so the line reports the window the next piece
873
- // will actually use.
874
- if (waitBelongsToJump) {
875
- // Recorded nowhere: see `waitBelongsToJump`. Said out loud, because a
876
- // gap in the supply history is otherwise indistinguishable from a swarm
877
- // that never made the reader wait.
878
- logger.info(
879
- `piece-reader: ${waitedMs}ms on the first piece after a jump — the cost of moving, ` +
880
- `not of this swarm's supply, so it is not counted against the quality offer`
881
- );
882
- waitBelongsToJump = false;
883
- } else {
884
- const supplyKey = `${torrent?.infoHash ?? "?"}/${file?.name ?? "?"}`;
885
- noteSteeringOutcome(supplyKey, waitedMs, pushed.asked > 0 || duplicated > 0);
886
- // Which level of the priority map the reader was stopped in. A wait
887
- // belongs to a level, and the level says whether that zone is asked for
888
- // too late — the reader itself no longer has an opinion about it.
889
- noteWaitLevel(
890
- supplyKey,
891
- waitedMs,
892
- wantedBy ?? Urgency.BLOCKED
893
- );
894
- waitCount += 1;
895
- waitedTotalMs += waitedMs;
896
- noteSupplyWait(supplyKey, file?.name ?? "", waitedMs);
897
- }
898
- const widened = nextWindowPieces({
899
- current: windowPieces,
900
- base: basePieces,
901
- ceiling: ceilingPieces(),
902
- waitedMs,
903
- waitThresholdMs: PIECE_WAIT_LOG_MS
904
- });
905
- if (widened !== windowPieces) {
906
- windowPieces = widened;
907
- }
908
- if (waitedMs >= PIECE_WAIT_LOG_MS) {
909
- const rateKbps = Math.round(pieceLength / 1024 / (waitedMs / 1000));
910
- // Two rates, side by side, and no verdict word between them: the swarm's
911
- // own delivery during the wait against what this piece managed. Both are
912
- // measured; which one a reader calls "the cause" follows from the pair
913
- // without a threshold having to be chosen here. Far apart means the
914
- // bytes were flowing and this piece was not among them; close together,
915
- // or both near zero, means there was nothing to deliver.
916
- const swarmBytes = Math.max(0, (Number(torrent?.downloaded) || 0) - downloadedAtWaitStart);
917
- const swarmKbps = Math.round(swarmBytes / 1024 / (waitedMs / 1000));
918
- logger.info(
919
- `piece-reader: waited ${waitedMs}ms for piece ${pieceIndex} ` +
920
- `(${pieceIndex - firstPiece + 1} of ${lastPiece - firstPiece + 1} in a read from ` +
921
- `${(start / 1024 / 1024).toFixed(0)}MB of "${file.name}") ` +
922
- `— ${rateKbps}KB/s on this piece while the swarm delivered ` +
923
- `${swarmKbps}KB/s (${(swarmBytes / 1024 / 1024).toFixed(1)}MB) across the torrent, ` +
924
- `${Number(torrent?.numPeers) || 0} peers connected; ` +
925
- (supply
926
- ? `${supply.holders}/${supply.peers} peers had it, ${supply.askedOf} were asked, ` +
927
- `${supply.blocks} blocks (${Math.round((supply.blocks * 16384) / 1024)}KB) in flight at peak`
928
- : "no sample taken") +
929
- // What WE did about it, so the next session says whether steering
930
- // the piece onto faster holders shortens the tail — by number
931
- // rather than by impression.
932
- `; steered onto ${pushed.asked} of ${pushed.attempted} asks (${pushed.considered} peers held it)` +
933
- // How often a fast peer we picked was refused because every block
934
- // was already spoken for and the library declined to take one from
935
- // a slow holder. Its thresholds are constants, not settings: the
936
- // asker must be above 16 KB/s, the holder below 48 KB/s and twice
937
- // as slow. A number here is what would justify replacing that rule;
938
- // a zero says the thresholds are not what we are short of.
939
- ((pushed.refusedWhileReserved ?? 0) > 0
940
- ? `; ${pushed.refusedWhileReserved} refused with every block reserved`
941
- : "") +
942
- (pushed.fastestBytesPerSecond > 0
943
- ? `, fastest ${Math.round(pushed.fastestBytesPerSecond / 1024)}KB/s`
944
- : "") +
945
- // Only when the steering placed nothing, which is the case that
946
- // decides whether duplicating the tail is worth building: it says
947
- // how much of the piece is still missing and which wires are
948
- // holding it, slowest first.
949
- (tailWhenNothingPlaced
950
- ? `; tail ${tailWhenNothingPlaced.missing}/${tailWhenNothingPlaced.chunks} blocks missing, held by ` +
951
- (tailWhenNothingPlaced.outstanding.length > 0
952
- ? tailWhenNothingPlaced.outstanding
953
- .map((wire) => `${wire.blocks}@${Math.round(wire.bytesPerSecond / 1024)}KB/s` +
954
- (wire.choking ? " (choking)" : ""))
955
- .join(" ")
956
- : "nobody")
957
- : "") +
958
- // What we did about the tail, so the next session says by number
959
- // whether a second copy of those blocks shortens the wait.
960
- (duplicated > 0 ? `; duplicated ${duplicated} blocks` : "") +
961
- // Where the priority map puts this piece. A wait at the level the
962
- // viewer is about to reach is a different fault from a wait on the
963
- // speculative tail, and the reader states nothing of its own that
964
- // could be read instead.
965
- `; the map wants it ${wantedBy === null
966
- ? "nowhere — nobody asked for this piece"
967
- : urgencyName(wantedBy)}`
968
- );
969
- }
970
-
971
- // Pinned BEFORE it is located, and before any await that could let an
972
- // eviction run: the offset is only meaningful while the piece is held.
973
- store.pin(pieceIndex);
974
- let located = null;
975
- try {
976
- located = await store.reside(pieceIndex);
977
- } catch (error) {
978
- store.unpin(pieceIndex);
979
- throw error;
980
- }
981
-
982
- if (!located) {
983
- store.unpin(pieceIndex);
984
- // THE CLAIM HAS JUST BEEN WITHDRAWN, so this is a wait and not a
985
- // failure. The store drops a piece behind every read head, and a read
986
- // that re-opens an input at byte 0 — which is every encoder restart —
987
- // asks for exactly those pieces. Until 2026-09-12 this threw, ffmpeg
988
- // read the empty body as the end of the file, and the encoder died and
989
- // was restarted into the same emptiness: field, `Piece 0 is verified
990
- // but absent from the store` answered every read for 92 minutes while
991
- // the viewer looked at a still picture.
992
- //
993
- // Going back one step re-runs this piece's whole path rather than a
994
- // shortened copy of it: the bitfield now says the piece is missing, so
995
- // the block above declares it, steers it onto the fastest holders and
996
- // waits for it, exactly as it does for a piece that was never here.
997
- // Once per piece — a second emptiness means the bytes are not coming
998
- // and the caller must hear so.
999
- if (withdrawnRetries === 0) {
1000
- withdrawnRetries += 1;
1001
- pieceIndex -= 1;
1002
- continue;
440
+ if (cancellation.isCancelled()) return;
441
+ store.pin(index);
442
+ let located;
443
+ try { located = await store.reside(index); }
444
+ catch (error) { store.unpin(index); throw error; }
445
+ if (!located) {
446
+ store.unpin(index);
447
+ if (retries++ === 0) continue;
448
+ throw new Error(`Piece ${index} was withdrawn from the store and did not come back.`);
1003
449
  }
1004
- throw new Error(
1005
- `Piece ${pieceIndex} was withdrawn from the store and did not come back.`
1006
- );
450
+ let released = false;
451
+ release = () => { if (!released) { released = true; store.unpin(index); } };
452
+ const from = Math.max(absoluteStart, pieceStart) - pieceStart;
453
+ const through = Math.min(absoluteEnd, pieceStart + pieceLength - 1) - pieceStart;
454
+ yield { pieceIndex: index, buffer: located.buffer, offset: located.offset + from,
455
+ length: through - from + 1, release };
456
+ release();
457
+ release = null;
458
+ break;
1007
459
  }
1008
- // Counted per piece: a long read may legitimately meet this more than
1009
- // once, and each piece is entitled to its own second chance.
1010
- withdrawnRetries = 0;
1011
-
1012
- let releasedThisPiece = false;
1013
- // Remembered so the generator can drop it itself. The pin is taken here
1014
- // and the consumer is expected to release it — but a consumer that
1015
- // ABANDONS the iterator never gets the chance, and a seek abandons it
1016
- // every time: the encoder is killed, the response is torn down, and the
1017
- // loop is left between two fragments. Field 2026-08-06: after one seek
1018
- // every slot in the store was pinned, the store answered
1019
- // `Every resident piece is pinned; no slot can be freed` — to the
1020
- // WebTorrent client, which closed the store and destroyed the torrent —
1021
- // and the session died with `File 0 not found`.
1022
- releaseHeldPin = () => {
1023
- if (!releasedThisPiece) {
1024
- releasedThisPiece = true;
1025
- store.unpin(pieceIndex);
1026
- }
1027
- };
1028
- // What the consumer takes, counted as it is handed over: for a viewer this
1029
- // is the film's own byte rate, which is what the band widths are derived
1030
- // against.
1031
- deliveredBytes += toWithinPiece - fromWithinPiece + 1;
1032
- yield {
1033
- pieceIndex,
1034
- buffer: located.buffer,
1035
- offset: located.offset + fromWithinPiece,
1036
- length: toWithinPiece - fromWithinPiece + 1,
1037
- release() {
1038
- if (releasedThisPiece) {
1039
- return;
1040
- }
1041
- releasedThisPiece = true;
1042
- store.unpin(pieceIndex);
1043
- }
1044
- };
1045
- // Handed back, and released by the consumer or not at all — either way
1046
- // this reader no longer owes anything for it.
1047
- releaseHeldPin = null;
1048
- }
1049
- } finally {
1050
- // A fragment handed out and never released is a slot lost for the life of
1051
- // the process. Reached on every exit, including the consumer walking away.
1052
- if (releaseHeldPin) {
1053
- releaseHeldPin();
1054
- releaseHeldPin = null;
1055
460
  }
1056
- // What this read did, said once at its end and under EVERY outcome: how
1057
- // much it delivered, and how much of that time it spent stopped. Said even
1058
- // when it never stopped, because "no wait" is the result worth counting and
1059
- // a line printed only beside a wait cannot report it.
1060
- const readSeconds = (Date.now() - readStartedAt) / 1000;
1061
- if (deliveredBytes > 0) {
1062
- logger.info(
1063
- `read "${String(file?.name ?? "?").slice(0, 40)}" ` +
1064
- `delivered=${(deliveredBytes / 1e6).toFixed(1)}MB in ${readSeconds.toFixed(1)}s ` +
1065
- `waits=${waitCount} waited=${(waitedTotalMs / 1000).toFixed(1)}s`
1066
- );
1067
- }
1068
- // Reached on completion, on cancellation, on a throw, and when the consumer
1069
- // stops iterating — a claim left behind would keep the swarm fetching for a
1070
- // reader that no longer exists.
1071
- if (blockedStated) {
1072
- register.withdraw(`${readerId}:${urgencyName(Urgency.BLOCKED)}`);
1073
- }
1074
- // Once, after everything has been withdrawn — and it releases this reader's
1075
- // hold on memory as well, because both views come from the same statement.
1076
- selection.reconcile();
1077
- }
461
+ } finally { release?.(); }
1078
462
  }