@torrent-tv/proxy 2.83.4 → 2.83.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -33,6 +33,20 @@ import { contentionPenalty } from "../encode/contention.js";
33
33
  import { waits } from "../priority/WaitLedger.js";
34
34
  import { SegmentDemand } from "../encode/SegmentDemand.js";
35
35
 
36
+ /**
37
+ * How long nothing is placed on an output whose input has just gone, and the
38
+ * ceiling that doubling reaches.
39
+ *
40
+ * Both are chosen, and are named here as chosen rather than dressed as
41
+ * measurements: what they bound is not how long the data takes to come back —
42
+ * that is the swarm's business and nobody here can know it — but how often it
43
+ * is worth asking. They replace the same two figures, with the same values,
44
+ * that lived in the session manager and governed only the dead run's own retry
45
+ * while the plan placed fresh runs beside it every half second.
46
+ */
47
+ const INPUT_QUIET_BASE_MS = 2_000;
48
+ const INPUT_QUIET_MAX_MS = 15_000;
49
+
36
50
  export class EncodeOrchestrator {
37
51
  /** Output address to what has been made of it. @type {Map<string, CoverageMap>} */
38
52
  #coverage = new Map();
@@ -47,6 +61,21 @@ export class EncodeOrchestrator {
47
61
  /** How runs have ended, by cause. @type {Map<string, number>} */
48
62
  #endings = new Map();
49
63
 
64
+ /**
65
+ * How many times in a row an output's run has died because its input was not
66
+ * there. Cleared only by a run that PRODUCED something, which is the only
67
+ * proof that the input can be read — see `#noteInputAvailability`.
68
+ *
69
+ * @type {Map<string, number>}
70
+ */
71
+ #inputLostAttempts = new Map();
72
+
73
+ /** Output address to the time before which nothing is placed on it. @type {Map<string, number>} */
74
+ #quietUntil = new Map();
75
+
76
+ /** The wake-up per output, so a quiet output is reconsidered. @type {Map<string, NodeJS.Timeout>} */
77
+ #quietTimers = new Map();
78
+
50
79
  /** The last state said out loud, so an unchanged state is not repeated. */
51
80
  #lastDescribed = "";
52
81
 
@@ -97,6 +126,7 @@ export class EncodeOrchestrator {
97
126
  refetchSecPerFilmSecond = () => 0,
98
127
  startingSpeedFor = () => 0,
99
128
  segmentStore = null,
129
+ planSoon = null,
100
130
  logger,
101
131
  now
102
132
  }) {
@@ -124,6 +154,9 @@ export class EncodeOrchestrator {
124
154
  this.segmentSeconds = segmentSeconds;
125
155
  this.logger = logger;
126
156
  this.now = typeof now === "function" ? now : Date.now;
157
+ // Asked for, not commanded: this class decides, and something else owns the
158
+ // loop that calls it. Absent in a test, where the clock is the test's own.
159
+ this.planSoon = typeof planSoon === "function" ? planSoon : () => undefined;
127
160
  }
128
161
 
129
162
  /**
@@ -275,6 +308,14 @@ export class EncodeOrchestrator {
275
308
  */
276
309
  notePriorityMap(address, zones) {
277
310
  this.demand.state(address, zones);
311
+ // NOBODY IS COMING HERE, so what was remembered about this output's input
312
+ // is about nobody. Kept, those three entries would stay for the life of the
313
+ // process, and the wait they describe would greet whoever opens this output
314
+ // next — a viewer arriving is new information, and one attempt for them is
315
+ // right whatever the last one met.
316
+ if (!Array.isArray(zones) || zones.length === 0) {
317
+ this.#forgetInputState(address);
318
+ }
278
319
  }
279
320
 
280
321
  /**
@@ -366,6 +407,19 @@ export class EncodeOrchestrator {
366
407
  });
367
408
  }
368
409
  }
410
+ // ITS INPUT WAS NOT THERE A MOMENT AGO, so nothing is placed yet. The sweep
411
+ // above still runs — a claim left by a dead run must be released whatever
412
+ // the reason — and only the placing waits. This is the one thing that makes
413
+ // the delay bind: it used to be timed against the dead run, which the plan
414
+ // does not consult, so a fresh run went to the same place as fast as ffmpeg
415
+ // could fail there.
416
+ //
417
+ // AND ONLY WHILE NOTHING IS PRODUCING THERE. A run still alive on this
418
+ // output is proof the input can be read, whatever a run beside it met, so
419
+ // the wait must not silence an output that is working.
420
+ if (this.#isQuiet(address) && this.runsOn(address).every((run) => !run.isAlive)) {
421
+ return;
422
+ }
369
423
  // ONE MAP, NOT ONE WINDOW PER VIEWER PER ZONE.
370
424
  //
371
425
  // Two viewers a few seconds apart state stretches that overlap, and the plan
@@ -696,6 +750,115 @@ export class EncodeOrchestrator {
696
750
  this.#runs.set(ended.address, remaining);
697
751
  }
698
752
  this.#endings.set(ended.ending, (this.#endings.get(ended.ending) ?? 0) + 1);
753
+ this.#noteInputAvailability(ended);
754
+ }
755
+
756
+ /**
757
+ * Remember, per output, that its input was not there — and until when nothing
758
+ * is to be placed on it.
759
+ *
760
+ * **Why the plan has to hold this.** A run whose input has gone is not alive,
761
+ * so the plan sees the stretch it held as free and places another run there at
762
+ * once — and the next one dies the same way, because nothing about the state
763
+ * has changed. There WAS a delay for exactly this, doubling from 2 s to 15 s,
764
+ * and it governed only the dead run's own retry while the plan went on placing
765
+ * fresh ones beside it. The comment beside that timer predicted this in as
766
+ * many words and the code did not prevent it. Field 2026-09-12: 2432 ffmpeg
767
+ * starts in 23 minutes, one every 0.57 s, for 61 minutes, against a delay that
768
+ * had long since reached its 15 s cap.
769
+ *
770
+ * So the delay lives where the decision is taken. It is not a cure for the
771
+ * input being away — that is the store's claim being withdrawn and the piece
772
+ * being fetched again — it is what stops one absent input from costing a
773
+ * thousand processes and a quarter of a million log lines while it is away.
774
+ *
775
+ * @param {{ address: string, ending: string }} ended
776
+ * @returns {void}
777
+ */
778
+ #noteInputAvailability(ended) {
779
+ if (ended.ending !== ENCODE_EXIT.INPUT_LOST) {
780
+ // CLEARED ONLY BY PROOF THAT THE INPUT WAS THERE, which is a segment
781
+ // having come out of it. Any other ending was nearly the rule here and is
782
+ // wrong: at the moment of failure several runs end at once, and one of
783
+ // them ending `gone` or `stopped` without producing a thing says nothing
784
+ // about the input — it would have lifted the wait the one beside it had
785
+ // just set, which is the storm again with an extra step.
786
+ const produced =
787
+ ended.firstOutputMs !== null && ended.firstOutputMs !== undefined
788
+ ? true
789
+ : Number.isFinite(ended.reached) && Number.isFinite(ended.from) && ended.reached >= ended.from;
790
+ if (produced) {
791
+ this.#inputLostAttempts.delete(ended.address);
792
+ this.#quietUntil.delete(ended.address);
793
+ }
794
+ return;
795
+ }
796
+ const attempts = (this.#inputLostAttempts.get(ended.address) ?? 0) + 1;
797
+ this.#inputLostAttempts.set(ended.address, attempts);
798
+ const delayMs = Math.min(
799
+ INPUT_QUIET_MAX_MS,
800
+ INPUT_QUIET_BASE_MS * 2 ** Math.min(attempts - 1, 6)
801
+ );
802
+ this.#quietUntil.set(ended.address, this.now() + delayMs);
803
+ this.logger.info(
804
+ `encode-plan on ${ended.address}: its input was not there (attempt ${attempts}) — ` +
805
+ `placing nothing for ${Math.round(delayMs / 1000)}s`
806
+ );
807
+ // THE WAKE-UP, because a plan that refuses to act needs something to ask it
808
+ // again: nothing about the state changes while the data is away, so no event
809
+ // would arrive to reconsider it.
810
+ const timer = setTimeout(() => {
811
+ this.#quietTimers.delete(ended.address);
812
+ this.planSoon();
813
+ }, delayMs);
814
+ timer.unref?.();
815
+ const previous = this.#quietTimers.get(ended.address);
816
+ if (previous) {
817
+ clearTimeout(previous);
818
+ }
819
+ this.#quietTimers.set(ended.address, timer);
820
+ }
821
+
822
+ /**
823
+ * Whether this output is still waiting before anything may be placed on it.
824
+ *
825
+ * A boolean rather than the milliseconds left: the figure had one reader and
826
+ * that reader only asked whether it was above zero, so it was a number
827
+ * computed and thrown away. How long the wait is was said when it began.
828
+ *
829
+ * @param {string} address
830
+ * @returns {boolean}
831
+ */
832
+ #isQuiet(address) {
833
+ const until = this.#quietUntil.get(address);
834
+ if (!Number.isFinite(until)) {
835
+ return false;
836
+ }
837
+ if (until - this.now() <= 0) {
838
+ this.#quietUntil.delete(address);
839
+ return false;
840
+ }
841
+ return true;
842
+ }
843
+
844
+ /**
845
+ * Forget everything remembered about an output nobody is producing for.
846
+ *
847
+ * Three maps are keyed by address, and an output that goes away leaves an
848
+ * entry in each for the life of the process. Small, and exactly the shape of
849
+ * accumulation this layer was built to remove from the session.
850
+ *
851
+ * @param {string} address
852
+ * @returns {void}
853
+ */
854
+ #forgetInputState(address) {
855
+ this.#inputLostAttempts.delete(address);
856
+ this.#quietUntil.delete(address);
857
+ const timer = this.#quietTimers.get(address);
858
+ if (timer) {
859
+ clearTimeout(timer);
860
+ this.#quietTimers.delete(address);
861
+ }
699
862
  }
700
863
 
701
864
  /**
@@ -83,6 +83,17 @@ export class PieceDiskStore {
83
83
  /** Where the live readers stand, from whoever holds that fact. @type {() => number[]} */
84
84
  #readHeads;
85
85
 
86
+ /**
87
+ * Said whenever this tier stops holding a piece, whatever the reason.
88
+ *
89
+ * It states THIS tier's own fact and nothing more — not whether the bytes can
90
+ * still be had elsewhere, which this class has no way of knowing. Whoever
91
+ * listens decides what the loss means.
92
+ *
93
+ * @type {(index: number) => void}
94
+ */
95
+ #onForgotten;
96
+
86
97
  /**
87
98
  * @param {object} params
88
99
  * @param {string} params.directory - Where this torrent's pieces live.
@@ -95,12 +106,21 @@ export class PieceDiskStore {
95
106
  * means nobody has said yet, and nothing is evicted until somebody does.
96
107
  * @param {() => number} [params.now]
97
108
  */
98
- constructor({ directory, name, chunkLength, allowanceBytes = null, now = Date.now, readHeads = () => [] }) {
109
+ constructor({
110
+ directory,
111
+ name,
112
+ chunkLength,
113
+ allowanceBytes = null,
114
+ now = Date.now,
115
+ readHeads = () => [],
116
+ onForgotten = () => undefined
117
+ }) {
99
118
  this.#directory = path.join(directory, name);
100
119
  this.#chunkLength = chunkLength;
101
120
  this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
102
121
  this.#now = now;
103
122
  this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];
123
+ this.#onForgotten = typeof onForgotten === "function" ? onForgotten : () => undefined;
104
124
  this.#adoptWhatIsAlreadyHere();
105
125
  }
106
126
 
@@ -345,6 +365,9 @@ export class PieceDiskStore {
345
365
  }
346
366
  });
347
367
  this.#removing.set(index, removal);
368
+ // SAID AFTER THE FACT IS TRUE, so a listener that asks this tier whether it
369
+ // holds the piece is told the same thing this method has just made so.
370
+ this.#onForgotten(index);
348
371
  }
349
372
 
350
373
  /**
@@ -321,6 +321,11 @@ export class SharedPieceStore {
321
321
  admittedWithoutSlot: 0,
322
322
  evictedOnRevise: 0,
323
323
  spillFailures: 0,
324
+ // Claims withdrawn: pieces this store stopped being able to produce at all,
325
+ // and said so. It is the figure that says whether the bargain the eviction
326
+ // states — "a seek back re-downloads it" — is being honoured, and before
327
+ // 2026-09-12 it was not honoured once, because nothing was ever said.
328
+ withdrawn: 0,
324
329
  // Whether the store is doing its job or being asked to hold more than it
325
330
  // has room for. An eviction that had to take a piece a reader declared it
326
331
  // wants is the second, and it comes back from disk moments later
@@ -413,6 +418,31 @@ export class SharedPieceStore {
413
418
  */
414
419
  #isElsewhere = null;
415
420
 
421
+ /**
422
+ * Said when this store can no longer produce a piece AT ALL — not resident,
423
+ * not on disk, and not inside a file held whole.
424
+ *
425
+ * **Why it has to exist.** This store holds the bytes, so it owns the fact
426
+ * "this proxy has piece N". Something else kept a second copy of that fact —
427
+ * the torrent's own completion bitfield — and nothing reconciled them: the
428
+ * disk tier drops a piece behind every read head, correctly, and the bitfield
429
+ * goes on saying the piece is verified. A read then concludes the piece is
430
+ * had, asks for it, is told it is absent, and fails; and it is never
431
+ * re-downloaded either, because the library does not fetch what it believes
432
+ * it already owns. Field 2026-09-12: a film played 80 seconds, the encoder
433
+ * ran on to 725 s, `forgetBehind` dropped some 565 spilled pieces including
434
+ * piece 0, the encoder restarted and re-opened its input at byte 0, and
435
+ * `/stream` answered `0 of 2363497962 bytes: Piece 0 is verified but absent
436
+ * from the store` for 92 minutes while the browser retried one segment.
437
+ *
438
+ * So the bitfield becomes a projection of this fact, maintained by this
439
+ * announcement. This store does not know what a torrent is, who listens, or
440
+ * what they do about it — the same shape as `readPieceElsewhere` above.
441
+ *
442
+ * @type {((what: object) => void) | null}
443
+ */
444
+ #onPieceGone = null;
445
+
416
446
  /** Whether the last revision had to exceed the machine's share. */
417
447
  #beyondTheMachine = false;
418
448
  /**
@@ -477,6 +507,9 @@ export class SharedPieceStore {
477
507
  this.#isElsewhere = typeof options.isPieceElsewhere === "function"
478
508
  ? options.isPieceElsewhere
479
509
  : null;
510
+ this.#onPieceGone = typeof options.onPieceGone === "function"
511
+ ? options.onPieceGone
512
+ : null;
480
513
  // Plain data the layer above needs to answer those two, and which this
481
514
  // store is handed anyway: where each file of the torrent begins and how
482
515
  // long the whole of it is.
@@ -496,7 +529,12 @@ export class SharedPieceStore {
496
529
  allowanceBytes: null,
497
530
  // Where the live readers stand, so what goes first is decided by them and
498
531
  // not by which piece happened to be touched longest ago.
499
- readHeads: () => this.#lru.readHeads()
532
+ readHeads: () => this.#lru.readHeads(),
533
+ // EVERY WAY A PIECE LEAVES THE DISK COMES THROUGH HERE — behind the read
534
+ // heads, over the allowance, dropped as a duplicate, or forgotten while a
535
+ // spill finished. One listener instead of a list of call sites to
536
+ // remember, which is what let the announcement be missed at three of them.
537
+ onForgotten: (index) => this.#announceGoneIfNowhere(index)
500
538
  });
501
539
  liveStores.add(this);
502
540
  }
@@ -1131,6 +1169,38 @@ export class SharedPieceStore {
1131
1169
  : null;
1132
1170
  }
1133
1171
 
1172
+ /**
1173
+ * Say a piece has gone, but only once it has gone from everywhere.
1174
+ *
1175
+ * The disk tier announces its own loss, which is not the same statement: a
1176
+ * piece dropped as a duplicate of a file held whole has not been lost at all,
1177
+ * and one still resident is about to be spilled again rather than gone. Only
1178
+ * the third case — neither tier, no whole file — is a withdrawal of the claim
1179
+ * that this proxy has those bytes.
1180
+ *
1181
+ * Nothing is said while the store is closing: the torrent it belongs to is
1182
+ * being torn down, and a claim withdrawn against a dying torrent reaches
1183
+ * either a destroyed object or, worse, one that has already been added back.
1184
+ *
1185
+ * @param {number} index
1186
+ * @returns {void}
1187
+ */
1188
+ #announceGoneIfNowhere(index) {
1189
+ if (this.#closed || this.#onPieceGone === null) {
1190
+ return;
1191
+ }
1192
+ if (this.#buffers.has(index) || this.#disk.has(index) || this.isInWholeFiles(index)) {
1193
+ return;
1194
+ }
1195
+ this.#counters.withdrawn += 1;
1196
+ try {
1197
+ this.#onPieceGone({ index, files: this.#files, name: this.#name });
1198
+ } catch {
1199
+ // silent-ok: whoever listens is diagnosing or bookkeeping, and a listener
1200
+ // that throws must not be what fails an eviction the store needs to make.
1201
+ }
1202
+ }
1203
+
1134
1204
  /**
1135
1205
  * Whether a spilled piece is now a duplicate of bytes held elsewhere.
1136
1206
  *
@@ -18,6 +18,7 @@ import { logger } from "../utils/logger.js";
18
18
  import { SharedPieceStore, findSharedStore } from "./piece-store/shared-piece-store.js";
19
19
  import { Urgency, urgencyName } from "./demand/index.js";
20
20
  import { demandFor, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
21
+ import { withdrawClaim } from "./download/withdraw-claim.js";
21
22
  import { isAtAWatchingViewer, isBehindEverybody, isNobodyComingNow } from "./priority/PriorityMap.js";
22
23
  import { deriveSourceKey } from "./torrent-source-key.js";
23
24
 
@@ -1066,6 +1067,9 @@ export class TorrentPool {
1066
1067
  */
1067
1068
  #wholeSources = new Set();
1068
1069
 
1070
+ /** How many claims this pool has withdrawn, over its whole life. */
1071
+ #claimsWithdrawn = 0;
1072
+
1069
1073
  /**
1070
1074
  * Say that every file of a source is held whole.
1071
1075
  *
@@ -1098,6 +1102,76 @@ export class TorrentPool {
1098
1102
  this.#storeExtras = extras && typeof extras === "object" ? extras : {};
1099
1103
  }
1100
1104
 
1105
+ /**
1106
+ * Everything a store of this pool is built with, in ONE place.
1107
+ *
1108
+ * It was assembled at both `client.add` sites — the ordinary one and the one
1109
+ * that replaces a torrent the client no longer has — and a store built by the
1110
+ * second was missing whatever the first had gained. That is how a store came
1111
+ * to exist without the whole-file reader it needs, and it is why this is a
1112
+ * method rather than a literal.
1113
+ *
1114
+ * @returns {object}
1115
+ */
1116
+ #storeOptions() {
1117
+ return {
1118
+ memoryBytes: this.#memoryBytes,
1119
+ ...this.#storeExtras,
1120
+ // WIRED BY THE POOL ITSELF, not handed in: withdrawing the claim needs
1121
+ // only a torrent, and this class is the one that owns them. Last, so a
1122
+ // caller's extras cannot displace the one thing that keeps the library's
1123
+ // completion bitfield in step with the bytes.
1124
+ onPieceGone: (what) => this.withdrawPieceClaim(what)
1125
+ };
1126
+ }
1127
+
1128
+ /**
1129
+ * Withdraw the claim that this proxy has a piece, because the store that held
1130
+ * it no longer can produce it.
1131
+ *
1132
+ * **Whose job this is.** The store owns the bytes and therefore owns the fact;
1133
+ * the library keeps a second copy of that fact in its completion bitfield, and
1134
+ * this pool is the only thing here that owns a torrent, so reconciling the two
1135
+ * is this class's and nothing else's. The store announces and does not know
1136
+ * who listens.
1137
+ *
1138
+ * **What it costs.** The piece is re-created as incomplete, so the next read
1139
+ * that wants it waits for a download instead of failing. Field 2026-09-12,
1140
+ * which is what this is for: 565 spilled pieces were dropped behind the read
1141
+ * heads — correctly, they were behind every reader — the bitfield went on
1142
+ * saying they were verified, the encoder restarted and re-opened its input at
1143
+ * byte 0, and `Piece 0 is verified but absent from the store` then answered
1144
+ * every read for 92 minutes. Nothing re-downloaded it, because the library
1145
+ * does not fetch what it believes it owns.
1146
+ *
1147
+ * **What it deliberately does not do.** `_markUnverified` would also re-select
1148
+ * the piece, and does not here: every torrent is added with `deselect: true`,
1149
+ * which sets the library's own `_startAsDeselected` and makes it skip that
1150
+ * call. The download set has one owner — `SwarmSelection`, from the priority
1151
+ * map — and a piece withdrawn here is fetched again when a read states it,
1152
+ * which is the same statement every other piece waits on.
1153
+ *
1154
+ * @param {object} what
1155
+ * @param {number} what.index
1156
+ * @param {object[]} what.files - The store's own, from which the torrent that
1157
+ * owns them is found; the store has no idea what a torrent is.
1158
+ * @returns {void}
1159
+ */
1160
+ withdrawPieceClaim({ index, files }) {
1161
+ const outcome = withdrawClaim({
1162
+ index,
1163
+ files,
1164
+ warn: (line) => logger.warn(`torrent-pool: ${line}`)
1165
+ });
1166
+ if (outcome === "withdrawn") {
1167
+ this.#claimsWithdrawn += 1;
1168
+ }
1169
+ }
1170
+
1171
+ get claimsWithdrawn() {
1172
+ return this.#claimsWithdrawn;
1173
+ }
1174
+
1101
1175
  constructor({ maxDiskBytes, memoryBytes, dhtBootstrap } = {}) {
1102
1176
  this.#memoryBytes = Number.isFinite(memoryBytes) && memoryBytes > 0 ? memoryBytes : undefined;
1103
1177
 
@@ -1842,7 +1916,7 @@ export class TorrentPool {
1842
1916
  const addedReplacement = this.client.add(torrentId, {
1843
1917
  store: SharedPieceStore,
1844
1918
  storeCacheSlots: 0,
1845
- storeOpts: { memoryBytes: this.#memoryBytes, ...this.#storeExtras },
1919
+ storeOpts: this.#storeOptions(),
1846
1920
  deselect: true
1847
1921
  }, (replacement) => {
1848
1922
  this.torrents.set(key, replacement);
@@ -1875,7 +1949,7 @@ export class TorrentPool {
1875
1949
  const added = this.client.add(torrentId, {
1876
1950
  store: SharedPieceStore,
1877
1951
  storeCacheSlots: 0,
1878
- storeOpts: { memoryBytes: this.#memoryBytes, ...this.#storeExtras },
1952
+ storeOpts: this.#storeOptions(),
1879
1953
  // Nothing is fetched until somebody says they want it. WebTorrent's own
1880
1954
  // default is `this.select(0, this.pieces.length - 1)` — the whole
1881
1955
  // torrent — and this proxy used to undo that afterwards by deselecting
@@ -17,7 +17,7 @@
17
17
  import { Worker } from "node:worker_threads";
18
18
  import { Readable } from "node:stream";
19
19
  import { fileURLToPath } from "node:url";
20
- import { logger } from "../../utils/logger.js";
20
+ import { logger, writeAlreadyDecided } from "../../utils/logger.js";
21
21
  import { createCaller, createReceiveStream } from "./channel.js";
22
22
  import { Command, Event } from "./protocol.js";
23
23
 
@@ -218,7 +218,10 @@ export class TorrentWorkerClient {
218
218
  this.#fragmentReaders.delete(message.id);
219
219
  break;
220
220
  case Event.LOG:
221
- logger.info(`torrent-worker: ${message.message}`);
221
+ // Written as it is: the worker holds the same repeat rule and has
222
+ // already applied it, and deciding again here would be one decision
223
+ // taken twice on two different histories.
224
+ writeAlreadyDecided(message.level ?? "info", `torrent-worker: ${message.message}`);
222
225
  break;
223
226
  case Event.FILE_COMPLETE:
224
227
  // Recorded here so the stream route can answer from the file without
@@ -734,6 +734,10 @@ export async function* readFragments({
734
734
  }
735
735
  };
736
736
 
737
+ // How many times THIS piece has been asked for again after the store
738
+ // withdrew it. Reset once a piece is in hand, so the allowance is per piece.
739
+ let withdrawnRetries = 0;
740
+
737
741
  try {
738
742
  for (let pieceIndex = firstPiece; pieceIndex <= lastPiece; pieceIndex += 1) {
739
743
  if (cancellation.isCancelled()) {
@@ -986,8 +990,33 @@ export async function* readFragments({
986
990
 
987
991
  if (!located) {
988
992
  store.unpin(pieceIndex);
989
- throw new Error(`Piece ${pieceIndex} is verified but absent from the store.`);
993
+ // THE CLAIM HAS JUST BEEN WITHDRAWN, so this is a wait and not a
994
+ // failure. The store drops a piece behind every read head, and a read
995
+ // that re-opens an input at byte 0 — which is every encoder restart —
996
+ // asks for exactly those pieces. Until 2026-09-12 this threw, ffmpeg
997
+ // read the empty body as the end of the file, and the encoder died and
998
+ // was restarted into the same emptiness: field, `Piece 0 is verified
999
+ // but absent from the store` answered every read for 92 minutes while
1000
+ // the viewer looked at a still picture.
1001
+ //
1002
+ // Going back one step re-runs this piece's whole path rather than a
1003
+ // shortened copy of it: the bitfield now says the piece is missing, so
1004
+ // the block above declares it, steers it onto the fastest holders and
1005
+ // waits for it, exactly as it does for a piece that was never here.
1006
+ // Once per piece — a second emptiness means the bytes are not coming
1007
+ // and the caller must hear so.
1008
+ if (withdrawnRetries === 0) {
1009
+ withdrawnRetries += 1;
1010
+ pieceIndex -= 1;
1011
+ continue;
1012
+ }
1013
+ throw new Error(
1014
+ `Piece ${pieceIndex} was withdrawn from the store and did not come back.`
1015
+ );
990
1016
  }
1017
+ // Counted per piece: a long read may legitimately meet this more than
1018
+ // once, and each piece is entitled to its own second chance.
1019
+ withdrawnRetries = 0;
991
1020
 
992
1021
  let releasedThisPiece = false;
993
1022
  // Remembered so the generator can drop it itself. The pin is taken here
@@ -56,8 +56,12 @@ import { forwardLogsTo, logger } from "../../utils/logger.js";
56
56
  * copy of the logger that had no file, and every one of their lines was lost:
57
57
  * measured over a whole 49 938-line file, not one of them was in it.
58
58
  */
59
- forwardLogsTo((_level, message) => {
60
- parentPort.postMessage({ type: Event.LOG, message });
59
+ forwardLogsTo((level, message) => {
60
+ // THE LEVEL TRAVELS TOO. It was dropped here, so every line this thread wrote
61
+ // — a warning about a spill that failed, an error about a torrent that went
62
+ // away — arrived on the other side as information and was coloured and
63
+ // recorded as such.
64
+ parentPort.postMessage({ type: Event.LOG, level, message });
61
65
  });
62
66
 
63
67
  // Imported dynamically, and that is load-bearing: static imports are RESOLVED
@@ -658,6 +662,7 @@ startMemoryReport({
658
662
  const STORE_REPORT_INTERVAL_MS = 60_000;
659
663
 
660
664
  /** Last reported reserve, so an unchanged one stays silent. */
665
+ let lastClaimsWithdrawn = 0;
661
666
  let lastReserveBytes = 0;
662
667
 
663
668
  /** Last reported figures per store, so unchanged ones stay silent. */
@@ -695,6 +700,20 @@ setInterval(() => {
695
700
  );
696
701
  }
697
702
  }
703
+ // WHAT THE ANNOUNCEMENTS ACTUALLY DID. The stores count every piece they
704
+ // stopped being able to produce; this counts the ones where the library did
705
+ // still hold a claim and it was taken back. The GAP between the two is the
706
+ // reading: announcements far above withdrawals mean the stores are mostly
707
+ // dropping pieces that were never completed, and a withdrawal count stuck at
708
+ // zero while announcements climb means the mechanism is not reaching the
709
+ // torrent at all.
710
+ if (pool.claimsWithdrawn !== lastClaimsWithdrawn) {
711
+ lastClaimsWithdrawn = pool.claimsWithdrawn;
712
+ log(
713
+ `torrent-pool: ${pool.claimsWithdrawn} piece claim(s) withdrawn — pieces this proxy ` +
714
+ "had dropped and has now told the swarm it needs again"
715
+ );
716
+ }
698
717
  const reserveNow = machineReserveBytes();
699
718
  if (reserveNow !== reserveBefore || reserveNow !== lastReserveBytes) {
700
719
  lastReserveBytes = reserveNow;
@@ -708,7 +727,7 @@ setInterval(() => {
708
727
  `${stats.fromMemory}/${stats.fromDisk}/${stats.spills}/${stats.revivals}/` +
709
728
  `${stats.blockedByPins}/${stats.evictedOnRevise}/${stats.spillFailures}/` +
710
729
  `${stats.evictedProtected}/${stats.demand?.unionPieces ?? 0}/${stats.admittedToDisk}/` +
711
- `${stats.blocksAllocated}/${stats.blocksFree}/${stats.blocksReleased}`;
730
+ `${stats.blocksAllocated}/${stats.blocksFree}/${stats.blocksReleased}/${stats.withdrawn}`;
712
731
  if (lastReported.get(stats.name) === signature) {
713
732
  continue;
714
733
  }
@@ -729,6 +748,13 @@ setInterval(() => {
729
748
  (stats.admittedWithoutSlot > 0 ? ` to-disk-for-want-of-memory=${stats.admittedWithoutSlot}` : "") +
730
749
  (stats.stillMs > 1000 ? ` nothing-moved-for=${Math.round(stats.stillMs / 1000)}s` : "") +
731
750
  (stats.evictedOnRevise > 0 ? ` evictedOnRevise=${stats.evictedOnRevise}` : "") +
751
+ // How many pieces this store stopped being able to produce and said so,
752
+ // which is what makes the eviction's own bargain checkable: every one of
753
+ // these is a piece the torrent has been told to fetch again if it is ever
754
+ // wanted. A session where this climbs while reads keep succeeding is the
755
+ // bargain working; before 2026-09-12 the figure did not exist and the
756
+ // claim was never withdrawn at all.
757
+ (stats.withdrawn > 0 ? ` withdrawn=${stats.withdrawn}` : "") +
732
758
  (stats.spillFailures > 0 ? ` spill-failures=${stats.spillFailures}` : "") +
733
759
  (stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
734
760
  );