@torrent-tv/proxy 2.83.0 → 2.83.1

package/services/piece-store/shared-piece-store.js

@@ -306,10 +306,19 @@ export class SharedPieceStore {
   #counters = {
     fromMemory: 0,
     fromDisk: 0,
+    // Reads answered from the file this torrent has been assembled into. Not a
+    // miss and not a spill: the bytes were never lost, they are simply kept as
+    // a file now.
+    fromWholeFile: 0,
     spills: 0,
     revivals: 0,
     blockedByPins: 0,
     waitedForPins: 0,
+    // Pieces that went to disk because no block could be had for them. Not a
+    // failure — the piece is kept and a later read revives it — but it is the
+    // measure of how often the store is asked to hold more than it may, and
+    // before 2026-09-11 this case ended the torrent instead of being counted.
+    admittedWithoutSlot: 0,
     evictedOnRevise: 0,
     spillFailures: 0,
     // Whether the store is doing its job or being asked to hold more than it
@@ -375,8 +384,35 @@ export class SharedPieceStore {
   #freeBlocks = [];
   /** Blocks that exist at all: free plus holding a piece. */
   #blocksAllocated = 0;
-  /** Whether a reader has ever declared a window here. See `wantedBytes`. */
-  #everHadReader = false;
+  /**
+   * The widest window any reader has declared here, in pieces.
+   *
+   * The floor this store asks for between readers: one window is what the next
+   * read will ask for, and it is a measurement rather than a chosen number.
+   * See `wantedBytes`.
+   */
+  #widestSeenPieces = 0;
+  /** The torrent's files in order, passed through to whoever can read them. @type {object[]} */
+  #files = [];
+
+  /** What the whole torrent weighs, so a last piece is not read past its end. */
+  #torrentLength = 0;
+
+  /**
+   * How to read a piece neither tier has, or null when there is nowhere else.
+   *
+   * @type {((where: object) => Promise<Uint8Array | null>) | null}
+   */
+  #readElsewhere = null;
+
+  /**
+   * Whether a piece can be had elsewhere, asked without reading it — so a
+   * spilled copy that has become a duplicate can be dropped.
+   *
+   * @type {((where: object) => boolean) | null}
+   */
+  #isElsewhere = null;
+
   /** Whether the last revision had to exceed the machine's share. */
   #beyondTheMachine = false;
   /**
@@ -432,6 +468,20 @@ export class SharedPieceStore {
     this.#growthCeiling = openingCeiling;
     this.#lru = new PieceLru(openingCeiling);
     this.#name = options.name ?? "pieces";
+    // THE LAST PLACE A PIECE CAN COME FROM, handed in as two plain functions.
+    // Where those bytes are, and what a file is, is not this store's business:
+    // it holds pieces, and these say whether one can be had elsewhere and how.
+    this.#readElsewhere = typeof options.readPieceElsewhere === "function"
+      ? options.readPieceElsewhere
+      : null;
+    this.#isElsewhere = typeof options.isPieceElsewhere === "function"
+      ? options.isPieceElsewhere
+      : null;
+    // Plain data the layer above needs to answer those two, and which this
+    // store is handed anyway: where each file of the torrent begins and how
+    // long the whole of it is.
+    this.#files = Array.isArray(options.files) ? options.files : [];
+    this.#torrentLength = totalLength;
     // `options.disk` exists so a test can hold a write open or make one fail on
     // purpose. Four of the defects fixed here live in what happens when the
     // disk tier does not answer immediately or at all, and none of them is
@@ -471,6 +521,13 @@ export class SharedPieceStore {
       // is never returned is invisible until the store cannot admit anything,
       // and by then the reason is long gone. At rest this is zero.
       outstanding: this.#outstandingPieces,
+      // How long since ANYTHING in this store moved — a write finishing, a
+      // piece admitted, a pin released. A claim gives up after
+      // `PINNED_WAIT_MS` of exactly this, so it is the figure that says whether
+      // a refusal was the store being busy or the store being stuck, and until
+      // 2026-09-11 nothing printed it: the field had 152 refusals in 104
+      // seconds and no way to tell which.
+      stillMs: this.#lastProgressAt > 0 ? Date.now() - this.#lastProgressAt : 0,
       spilled: this.#disk.size,
       // What the spill file ACTUALLY weighs, piece by piece, rather than the
       // piece count times a full piece length. The last piece of a torrent is
@@ -568,7 +625,7 @@ export class SharedPieceStore {
   get wantedBytes() {
     const demand = this.#lru.demand();
     if (demand.readers > 0) {
-      this.#everHadReader = true;
+      this.#widestSeenPieces = Math.max(this.#widestSeenPieces, demand.widestPieces);
       const pieces = Math.max(MIN_RESIDENT_PIECES, demand.unionPieces, demand.widestPieces);
       // Plus room to absorb what arrives while a write is finishing. Asking for
       // exactly what the readers want leaves no free place ever, so every
@@ -577,16 +634,23 @@ export class SharedPieceStore {
       // that followed.
       return (pieces + this.slackPieces()) * this.#chunkLength;
     }
-    // Readers that have GONE are not the same as readers that have not arrived.
-    // A store whose readers ended has nothing to hold pieces for — its torrent
-    // sits until the pool's idle timer removes it, which needs a refcount of
-    // zero and can be a quarter of an hour away — so it asks for nothing and
-    // its memory goes back to the machine now. A store that has never had a
-    // reader is being filled for one that is on its way, and asks for what it
-    // was opened with until the first read says what it needs.
-    return this.#everHadReader
-      ? (MIN_RESIDENT_PIECES + this.slackPieces()) * this.#chunkLength
-      : this.#growthCeiling * this.#chunkLength;
+    // WITH NO READER, THE FLOOR IS ONE READER'S WINDOW — the widest this store
+    // has actually been asked for, which is measured rather than chosen, and
+    // `MIN_RESIDENT_PIECES` only until it has been asked for anything.
+    //
+    // It used to fall to the minimum the moment the last reader went, and the
+    // allowance is re-derived once a minute (`STORE_REPORT_INTERVAL_MS`) while
+    // a claim gives up after five seconds. A reader arriving into that minute
+    // met a store of two or three blocks: field 2026-09-11, the viewer switched
+    // to another file of the same torrent and the store was at
+    // `0/3 (0MB of 12MB allowed)` with the machine offering 4.3 GB.
+    //
+    // Holding one window's worth between readers is not waste: it is what the
+    // next read will ask for within seconds, and the pieces in it are the ones
+    // that reader left off at. A store that has never been read keeps nothing
+    // either — its pieces are arriving for a reader on the way, and they have
+    // the disk.
+    return (Math.max(MIN_RESIDENT_PIECES, this.#widestSeenPieces) + this.slackPieces()) * this.#chunkLength;
   }
 
   reviseGrowthCeiling(allowedBytes) {
@@ -720,6 +784,28 @@ export class SharedPieceStore {
    * @returns {Promise<() => void>} The release, which the caller MUST call in a
    *   `finally`. Calling it twice is harmless.
    */
+  /**
+   * The same reservation, answering null instead of throwing.
+   *
+   * For callers that have somewhere else to go: an arriving piece has the disk,
+   * so it writes through rather than failing. Keeping the throwing form for
+   * callers that genuinely have no alternative is what makes the difference
+   * visible at the call site rather than in a catch.
+   *
+   * @returns {Promise<(() => void) | null>}
+   */
+  async #claimSlotOrNull() {
+    try {
+      return await this.#claimSlot();
+    } catch (error) {
+      if (this.#closed) {
+        throw error;
+      }
+      this.#counters.admittedWithoutSlot += 1;
+      return null;
+    }
+  }
+
   async #claimSlot() {
     // How long THIS claim has been trying, kept here and not on the store.
     // It was a field, `#pinnedWaitStartedAt`, shared by the two waits inside
@@ -736,8 +822,8 @@ export class SharedPieceStore {
       if (this.#closed) {
         throw new Error("Piece store is closed.");
       }
-      const ok = await this.#claimSlotOnce(waitingSince);
-      if (ok) {
+      const isReserved = await this.#claimSlotOnce(waitingSince);
+      if (isReserved) {
         let released = false;
         return () => {
           if (released) {
@@ -1028,6 +1114,68 @@ export class SharedPieceStore {
     }
   }
 
+  /**
+   * One piece, from wherever else it can be had.
+   *
+   * @param {number} index
+   * @returns {Promise<Buffer | null>}
+   */
+  async #fromWholeFiles(index) {
+    return this.#readElsewhere
+      ? this.#readElsewhere({
+          index,
+          pieceLength: this.#chunkLength,
+          length: this.#torrentLength,
+          files: this.#files
+        })
+      : null;
+  }
+
+  /**
+   * Whether a spilled piece is now a duplicate of bytes held elsewhere.
+   *
+   * @param {number} index
+   * @returns {boolean}
+   */
+  isInWholeFiles(index) {
+    return this.#isElsewhere
+      ? this.#isElsewhere({
+          index,
+          pieceLength: this.#chunkLength,
+          length: this.#torrentLength,
+          files: this.#files
+        }) === true
+      : false;
+  }
+
+  /**
+   * Drop spilled pieces that are now duplicates of bytes held elsewhere.
+   *
+   * A film assembled into a file is on the disk twice until this runs: once as
+   * the file and once as the pieces it was built from. Field 2026-09-11, one
+   * episode: 1417 MB of segments beside 1424 MB of spilled pieces. Nothing is
+   * lost by dropping the second — a read that wants one of those pieces is
+   * answered from the file.
+   *
+   * @returns {number} How many were dropped.
+   */
+  dropDuplicatesHeldElsewhere() {
+    if (!this.#isElsewhere) {
+      return 0;
+    }
+    let dropped = 0;
+    for (const index of this.#disk.indexes()) {
+      if (this.#buffers.has(index) || this.#evicting.has(index)) {
+        continue;
+      }
+      if (this.isInWholeFiles(index)) {
+        this.#disk.forget(index);
+        dropped += 1;
+      }
+    }
+    return dropped;
+  }
+
   #wake() {
     const waiting = this.#waiters;
     this.#waiters = [];
@@ -1064,7 +1212,15 @@ export class SharedPieceStore {
     //
     // So when the disk is what the store is waiting for, it waits. A completing
     // write calls `#noteProgress`, which wakes whoever is here.
-    if (this.#blocksInFlight() > 0 && this.#blocksInUse() >= this.#growthCeiling) {
+    // THE SAME QUANTITY THE GRANT ABOVE TESTS. It used to ask whether
+    // `blocksInUse` alone had reached the ceiling, while what reaches the
+    // ceiling is `blocksInUse + outstanding` — so a store held at its ceiling
+    // by reservations fell past this branch and into the eviction below, which
+    // has nothing to evict when nothing is resident. Field 2026-09-11:
+    // `blocks=2 (0 spare)` with `outstanding=2` against a ceiling of 3, two
+    // writes in flight each of which would have returned a block, and the claim
+    // threw rather than waiting the moment out.
+    if (this.#blocksInFlight() > 0 && this.#blocksInUse() + this.#outstandingPieces >= this.#growthCeiling) {
       if (stillFor() < PINNED_WAIT_MS) {
         this.#counters.waitedForDisk += 1;
         return false;
@@ -1090,8 +1246,19 @@ export class SharedPieceStore {
         return false;
       }
       this.#counters.blockedByPins += 1;
+      // SAY WHICH STATE REFUSED, with the numbers it was read from. The old
+      // text named one state — every resident piece pinned — and the field case
+      // of 2026-09-11 was a different one: `resident=0 pinned=0`, nothing to
+      // evict because nothing was in memory at all, the ceiling held by
+      // reservations and by two writes on their way to disk. A message that
+      // names a state that did not exist costs a day of reading the wrong code.
       throw new Error(
-        `Every resident piece is pinned and nothing moved for ${PINNED_WAIT_MS}ms; no slot can be freed.`
+        `No block can be had for ${PINNED_WAIT_MS}ms: ` +
+        `${this.#buffers.size} resident (${this.#lru.pinnedCount} pinned, ` +
+        `${this.#lru.protectedCount} range(s) declared), ` +
+        `${this.#blocksInUse()} block(s) in use of ${this.#growthCeiling} allowed, ` +
+        `${this.#blocksInFlight()} on their way to disk, ` +
+        `${this.#outstandingPieces} reservation(s) outstanding.`
       );
     }
 
@@ -1135,7 +1302,25 @@ export class SharedPieceStore {
     }
 
     if (!this.#disk.has(index)) {
-      return null;
+      // Not spilled, but the file it belongs to may be here whole — which is
+      // the ordinary case once a film has been assembled and its spilled copy
+      // dropped as the duplicate it had become.
+      const whole = await this.#fromWholeFiles(index);
+      if (!whole) {
+        return null;
+      }
+      this.#counters.fromWholeFile += 1;
+      const release = await this.#claimSlotOrNull();
+      if (release === null) {
+        // No block to put it in. The caller gets the bytes anyway; what it
+        // loses is only that the next read pays for this one again.
+        return whole;
+      }
+      try {
+        return this.#registerPiece(index, this.#copyIntoNewBuffer(index, whole));
+      } finally {
+        release();
+      }
     }
 
     const release = await this.#claimSlot();
@@ -1382,8 +1567,8 @@ export class SharedPieceStore {
       }
 
       this.#noteArrival();
-      const declared = this.#lru.wants(index);
-      if (declared) {
+      const isDeclared = this.#lru.wants(index);
+      if (isDeclared) {
         this.#counters.admittedInsideWindow += 1;
       } else {
         this.#counters.admittedOutsideWindow += 1;
@@ -1407,7 +1592,7 @@ export class SharedPieceStore {
       //
       // Only when SOMETHING is stated: before that there is no basis for
       // calling one piece more wanted than another.
-      const worseThanTheVictim = () => {
+      const isWorseThanTheVictim = () => {
         const victim = this.#lru.nextVictim();
         if (victim.index === null) {
           return false;
@@ -1420,14 +1605,32 @@ export class SharedPieceStore {
         return arriving >= 0 && victim.wait >= 0 && arriving > victim.wait;
       };
       if (this.#lru.protectedCount > 0 && this.#isFullNow()
-        && (!declared || worseThanTheVictim())) {
+        && (!isDeclared || isWorseThanTheVictim())) {
         this.#counters.admittedToDisk += 1;
         await this.#writeThrough(index, bytes);
         this.#noteProgress();
         return;
       }
 
-      const release = await this.#claimSlot();
+      // MEMORY IS NEVER A REASON TO REFUSE A PIECE. The piece is verified and
+      // the disk will take it; failing here fails the torrent client's own
+      // write, and the client answers that by destroying the torrent — field
+      // 2026-09-11, `WebTorrent client error: Every resident piece is pinned`,
+      // after which every read of that torrent answered `File 1 not found` for
+      // the life of the process and the viewer could not open anything.
+      //
+      // The branch above already writes through when the store is full and the
+      // arrival is wanted less than what it would displace. It cannot fire
+      // where nothing is declared (`protectedCount > 0`), which is exactly the
+      // state a store between readers is in — and that state is where the
+      // refusal happened.
+      const release = await this.#claimSlotOrNull();
+      if (release === null) {
+        this.#counters.admittedToDisk += 1;
+        await this.#writeThrough(index, bytes);
+        this.#noteProgress();
+        return;
+      }
       try {
         this.#registerPiece(index, this.#copyIntoNewBuffer(index, bytes));
         await this.#forgetOnDisk(index);
@@ -1461,11 +1664,43 @@ export class SharedPieceStore {
         return Buffer.from(Buffer.from(buffer, offset, length));
       }
 
-      const revived = await this.#revive(index);
-      if (revived === null) {
-        throw new Error(`Piece ${index} is not in the store.`);
+      // NOT revived into memory. This entry point is the torrent client's: it
+      // reads to answer a peer, and a peer asks for 16 KB while a piece here is
+      // megabytes. Reviving would take a whole block for bytes nobody is going
+      // to read again — and it is the ONE read that can be answered without a
+      // block at all, since the caller is handed its own copy either way.
+      //
+      // Field 2026-09-11, and this is the whole reason the store could not
+      // serve anybody: an upload capped at 512 KB/s produced 63 416 reads of
+      // which 78.4 % missed memory, 49 696 pieces revived whole, and every one
+      // of those revivals competed for a ceiling of three blocks. Claiming a
+      // block for an upload is also how this path reached `#claimSlot`, whose
+      // failure the torrent client turns into a destroyed torrent.
+      // A spill that is still being written owns the only copy there is, so
+      // wait for it exactly as `#revive` does before asking the disk.
+      const spill = this.#evicting.get(index);
+      if (spill) {
+        await spill.catch(() => undefined);
       }
-      return Buffer.from(Buffer.from(revived, offset, length));
+      const resident = this.#buffers.get(index);
+      if (resident !== undefined) {
+        this.#lru.touch(index);
+        this.#counters.fromMemory += 1;
+        return Buffer.from(Buffer.from(resident, offset, length));
+      }
+      if (this.#disk.has(index)) {
+        const want = Math.max(0, Math.min(length, this.#lengthOf(index) - offset));
+        const target = Buffer.allocUnsafe(want);
+        await this.#disk.read(index, target, offset);
+        this.#counters.fromDisk += 1;
+        return target;
+      }
+      const whole = await this.#fromWholeFiles(index);
+      if (whole) {
+        this.#counters.fromWholeFile += 1;
+        return Buffer.from(whole.subarray(offset, offset + length));
+      }
+      throw new Error(`Piece ${index} is not in the store.`);
     };
 
     fetch().then((bytes) => done(null, bytes), (error) => done(error));
@@ -1484,6 +1719,18 @@ export class SharedPieceStore {
    */
   protectRange(readerId, from, to, urgency) {
     this.#lru.protect(readerId, from, to, urgency);
+    // A READER DECLARES ITSELF IN A MOMENT; the allowance was re-derived once a
+    // minute. Between the two a read met whatever the store had shrunk to while
+    // nobody was reading, and a claim gives up after five seconds — twelve
+    // times sooner than the store could have grown (field 2026-09-11).
+    //
+    // DELIBERATELY NOT raising the ceiling here. A reader declaring a window
+    // wider than the store may hold is a shortage to absorb, not an instruction
+    // to take more memory: with several viewers on several films the unions add
+    // up across stores, and each store's floor already wins over the machine's
+    // share at the next revision. What the shortage must not do is fail a read,
+    // and that is answered where it arises — no path in this store refuses for
+    // want of memory any more.
   }
 
   protectedRanges() {