@torrent-tv/proxy 2.83.3 → 2.83.5

package/services/torrent-worker/client.js

@@ -317,30 +317,6 @@ export class TorrentWorkerClient {
     return this.#caller.call(Command.LIST_FILES, { sourceKey });
   }
 
-  /**
-   * Claim a file so it is not evicted while being read.
-   *
-   * @param {string} sourceKey
-   * @param {number} fileIndex
-   * @returns {Promise<string>} The claim's identity, for {@link releaseFile}.
-   */
-  async acquireFile(sourceKey, fileIndex) {
-    return this.#caller.call(Command.ACQUIRE_FILE, { sourceKey, fileIndex });
-  }
-
-  /**
-   * Drop one claim taken with {@link acquireFile}.
-   *
-   * Named by claim rather than by file: several readers hold the same file at
-   * once, and releasing "the file" released somebody else's hold.
-   *
-   * @param {string} claimId
-   * @returns {Promise<void>}
-   */
-  async releaseFile(claimId) {
-    await this.#caller.call(Command.RELEASE_FILE, { claimId });
-  }
-
   /**
    * Live download figures for the progress display.
    *

package/services/torrent-worker/piece-reader.js

@@ -734,6 +734,10 @@ export async function* readFragments({
     }
   };
 
+  // How many times THIS piece has been asked for again after the store
+  // withdrew it. Reset once a piece is in hand, so the allowance is per piece.
+  let withdrawnRetries = 0;
+
   try {
     for (let pieceIndex = firstPiece; pieceIndex <= lastPiece; pieceIndex += 1) {
       if (cancellation.isCancelled()) {
@@ -986,8 +990,33 @@ export async function* readFragments({
 
       if (!located) {
         store.unpin(pieceIndex);
-        throw new Error(`Piece ${pieceIndex} is verified but absent from the store.`);
+        // THE CLAIM HAS JUST BEEN WITHDRAWN, so this is a wait and not a
+        // failure. The store drops a piece behind every read head, and a read
+        // that re-opens an input at byte 0 — which is every encoder restart —
+        // asks for exactly those pieces. Until 2026-09-12 this threw, ffmpeg
+        // read the empty body as the end of the file, and the encoder died and
+        // was restarted into the same emptiness: field, `Piece 0 is verified
+        // but absent from the store` answered every read for 92 minutes while
+        // the viewer looked at a still picture.
+        //
+        // Going back one step re-runs this piece's whole path rather than a
+        // shortened copy of it: the bitfield now says the piece is missing, so
+        // the block above declares it, steers it onto the fastest holders and
+        // waits for it, exactly as it does for a piece that was never here.
+        // Once per piece — a second emptiness means the bytes are not coming
+        // and the caller must hear so.
+        if (withdrawnRetries === 0) {
+          withdrawnRetries += 1;
+          pieceIndex -= 1;
+          continue;
+        }
+        throw new Error(
+          `Piece ${pieceIndex} was withdrawn from the store and did not come back.`
+        );
       }
+      // Counted per piece: a long read may legitimately meet this more than
+      // once, and each piece is entitled to its own second chance.
+      withdrawnRetries = 0;
 
       let releasedThisPiece = false;
       // Remembered so the generator can drop it itself. The pin is taken here

package/services/torrent-worker/pool-adapter.js

@@ -11,11 +11,9 @@
  *
  * Two accommodations are needed, and both are deliberate:
  *
- *  - **`acquireFile` and `prioritizeByteRange` stay synchronous.** They return
- *    nothing the caller inspects, so the command is dispatched and not awaited.
- *    `acquireFile` hands back a release function exactly as before, which sends
- *    its own command when called. Awaiting them would mean touching every call
- *    site for no observable gain.
+ *  - **`prioritizeByteRange` stays synchronous.** It returns nothing the caller
+ *    inspects, so the command is dispatched and not awaited. Awaiting it would
+ *    mean touching every call site for no observable gain.
  *  - **`getTorrent` needs a `sourceKey`.** Torrent objects cannot cross a
  *    thread, so the worker keys them. Callers that have one pass it; the rest
  *    get one derived from the source itself, so the identity stays stable
@@ -85,34 +83,6 @@ export class WorkerTorrentPool {
     return this.#client.allowSpillBytes(bytes);
   }
 
-  acquireFile(torrent, fileIndex) {
-    const sourceKey = torrent?.sourceKey;
-    if (!sourceKey) {
-      return () => undefined;
-    }
-    // Dispatched, not awaited — callers use the result immediately and inspect
-    // nothing. But the release MUST NOT overtake it: both are ordinary messages
-    // to the worker, and if release arrives first the reader count drops to zero
-    // while a read is still running. The idle sweep then removes the torrent AND
-    // its downloaded data out from under the encoder — field 2026-08-02:
-    // "removed idle torrent ... and its store" mid-playback, after which every
-    // read hung and ffmpeg saw an empty input ("Stream ends prematurely at 0").
-    // Chaining the release onto the acquire keeps them in order.
-    const acquired = this.#client.acquireFile(sourceKey, fileIndex).catch(() => null);
-    let released = false;
-    return () => {
-      if (released) {
-        return;
-      }
-      released = true;
-      // Release the claim this call opened, not "the file" — waiting for the
-      // acquire is also what tells us which claim that is.
-      void acquired
-        .then((claimId) => (claimId ? this.#client.releaseFile(claimId) : undefined))
-        .catch(() => undefined);
-    };
-  }
-
   /**
    * Bytes every torrent here has moved.
    *

package/services/torrent-worker/protocol.js

@@ -47,10 +47,6 @@
 export const Command = {
   /** Add (or join) a torrent; resolves when metadata is ready. */
   ADD_SOURCE: "add-source",
-  /** Claim a file for reading, so it is not evicted while in use. */
-  ACQUIRE_FILE: "acquire-file",
-  /** Drop a claim; the worker applies its own idle-removal policy. */
-  RELEASE_FILE: "release-file",
   /** File list and metadata for a source. */
   LIST_FILES: "list-files",
   /** Live download figures for the progress display. */

package/services/torrent-worker/worker.js

@@ -11,10 +11,9 @@
  * with the torrent for this thread, which is exactly the problem being solved.
  *
  * The existing `TorrentPool` is reused wholesale rather than reimplemented. It
- * already carries the parts that took field failures to get right — refcounted
- * file claims, idle removal, the global disk cap with LRU eviction, seek-aware
- * piece prioritisation, adaptive upload — and none of that changes by moving
- * threads.
+ * already carries the parts that took field failures to get right — idle
+ * removal, the global disk cap with LRU eviction, seek-aware piece
+ * prioritisation, adaptive upload — and none of that changes by moving threads.
  */
 
 // MUST stay first: it redirects `webrtc-polyfill` to a JavaScript WebRTC stack
@@ -25,7 +24,6 @@ import { isUsableTorrentHandle } from "./handle-state.js";
 import "./install-webrtc-shim.js";
 import { parentPort, workerData } from "node:worker_threads";
 import { createSendStream } from "./channel.js";
-import { createFileClaims } from "./file-claims.js";
 import { readFragments, supplyFiguresFor } from "./piece-reader.js";
 import { cuesHeldFor, declaredSubtitleTracksOf, subtitleTracksOf, warmSubtitleCues } from "./subtitle-cues.js";
 import {
@@ -36,6 +34,7 @@ import {
   warmResumePosition
 } from "./container-tracks.js";
 import { fillFileInBackground } from "./background-fill.js";
+import { demandFor } from "../download/registry.js";
 import { CompletedFiles, completedFilesRoot } from "../files/CompletedFiles.js";
 import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../files/piece-from-whole-file.js";
 import { Command, Event } from "./protocol.js";
@@ -93,8 +92,6 @@ const torrentsByKey = new Map();
  * @type {Map<string, { sourceType: string, source: string }>}
  */
 const sourceRecipes = new Map();
-/** File claims, each with its own identity — see `file-claims.js`. */
-const fileClaims = createFileClaims();
 /** In-flight reads, so a cancel can stop one mid-body. */
 const readsById = new Map();
 
@@ -127,6 +124,26 @@ function log(message) {
  * @param {string} sourceKey
  * @returns {Promise<import("webtorrent").Torrent>}
  */
+/**
+ * The torrent for a sourceKey IF there is one, and never one that has to be
+ * built to answer.
+ *
+ * The other half of the pair above, for the questions that are about something
+ * going away: a departure is not a reason to add anything, and the answer
+ * "there is nothing here" is a complete answer to them.
+ *
+ * @param {string} sourceKey
+ * @returns {Promise<import("webtorrent").Torrent | null>}
+ */
+async function knownTorrent(sourceKey) {
+  const pending = torrentsByKey.get(sourceKey);
+  if (!pending) {
+    return null;
+  }
+  const torrent = await pending.catch(() => null);
+  return isUsableTorrentHandle(torrent) ? torrent : null;
+}
+
 async function requireTorrent(sourceKey) {
   const pending = torrentsByKey.get(sourceKey);
   if (!pending) {
@@ -241,15 +258,6 @@ async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }
   const sender = createSendStream({ port: parentPort, requestId: id });
   readsById.set(id, sender);
 
-  // Hold the file for as long as this read runs. The caller also acquires it,
-  // but that acquire and its release are separate messages from another thread
-  // and can be reordered; this one cannot, because it lives entirely inside the
-  // read. Without it the idle sweep saw a zero reader count and removed the
-  // torrent AND its store mid-read — field 2026-08-02: "removed idle torrent
-  // ... and its store", after which every subsequent read hung and ffmpeg got
-  // an empty input.
-  const releaseRead = pool.acquireFile(torrent, fileIndex);
-
   const rangeStart = start ?? 0;
   const rangeEnd = end ?? file.length - 1;
 
@@ -290,7 +298,6 @@ async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }
     // Any fragment still awaiting confirmation will never get one now; settling
     // it here releases its pin rather than leaking a held slot.
     settleFragment(id);
-    releaseRead();
     if (!failed) {
       sender.end();
     }
@@ -356,27 +363,6 @@ async function runCommand(command, params, id) {
       }));
     }
 
-    case Command.ACQUIRE_FILE: {
-      const torrent = await requireTorrent(params.sourceKey);
-      // Every acquire is its own claim. Sharing one per file meant the first
-      // reader to finish released the hold while others were still reading.
-      return fileClaims.open(
-        params.sourceKey,
-        params.fileIndex,
-        pool.acquireFile(torrent, params.fileIndex)
-      );
-    }
-
-    case Command.RELEASE_FILE: {
-      const released = fileClaims.close(params.claimId);
-      if (!released) {
-        // Not fatal — but it means a release arrived twice or after teardown,
-        // and silence here is what let the previous scheme look healthy.
-        log(`release for unknown file claim ${params.claimId}`);
-      }
-      return released;
-    }
-
     case Command.HELD_TORRENTS: {
       // Which films this proxy has right now, and how much of each. Answered
       // from the live client rather than from the main thread's map of
@@ -537,13 +523,18 @@ async function runCommand(command, params, id) {
     }
 
     case Command.PRIORITY_MAP: {
-      const torrent = await requireTorrent(params.sourceKey);
-      pool.applyPriorityMap(
-        torrent,
-        params.fileIndex,
-        params.zones,
-        params.durationSeconds
-      );
+      // A MAP WITH NOTHING IN IT MUST NOT BRING A TORRENT BACK. It is what is
+      // said when the last viewer of a file leaves, which is also when the
+      // torrent may be on its way out — and `requireTorrent` rebuilds a dead
+      // handle from its recipe, so asking that way would re-add a torrent in
+      // order to be told that nothing is wanted of it.
+      const zones = Array.isArray(params.zones) ? params.zones : [];
+      const torrent = zones.length === 0
+        ? await knownTorrent(params.sourceKey)
+        : await requireTorrent(params.sourceKey);
+      if (torrent) {
+        pool.applyPriorityMap(torrent, params.fileIndex, zones, params.durationSeconds);
+      }
       return true;
     }
 
@@ -590,7 +581,6 @@ async function runCommand(command, params, id) {
     }
 
     case Command.DESTROY_ALL: {
-      fileClaims.closeAll();
       torrentsByKey.clear();
       sourceRecipes.clear();
       await pool.destroyAll();
@@ -718,7 +708,7 @@ setInterval(() => {
       `${stats.fromMemory}/${stats.fromDisk}/${stats.spills}/${stats.revivals}/` +
       `${stats.blockedByPins}/${stats.evictedOnRevise}/${stats.spillFailures}/` +
       `${stats.evictedProtected}/${stats.demand?.unionPieces ?? 0}/${stats.admittedToDisk}/` +
-      `${stats.blocksAllocated}/${stats.blocksFree}/${stats.blocksReleased}`;
+      `${stats.blocksAllocated}/${stats.blocksFree}/${stats.blocksReleased}/${stats.withdrawn}`;
     if (lastReported.get(stats.name) === signature) {
       continue;
     }
@@ -739,6 +729,13 @@ setInterval(() => {
       (stats.admittedWithoutSlot > 0 ? ` to-disk-for-want-of-memory=${stats.admittedWithoutSlot}` : "") +
       (stats.stillMs > 1000 ? ` nothing-moved-for=${Math.round(stats.stillMs / 1000)}s` : "") +
       (stats.evictedOnRevise > 0 ? ` evictedOnRevise=${stats.evictedOnRevise}` : "") +
+      // How many pieces this store stopped being able to produce and said so,
+      // which is what makes the eviction's own bargain checkable: every one of
+      // these is a piece the torrent has been told to fetch again if it is ever
+      // wanted. A session where this climbs while reads keep succeeding is the
+      // bargain working; before 2026-09-12 the figure did not exist and the
+      // claim was never withdrawn at all.
+      (stats.withdrawn > 0 ? ` withdrawn=${stats.withdrawn}` : "") +
       (stats.spillFailures > 0 ? ` spill-failures=${stats.spillFailures}` : "") +
       (stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
     );
@@ -827,11 +824,11 @@ function describePieceBuffers() {
  * @returns {void}
  */
 function warmActiveFiles(sourceKey, torrent) {
-  const usage = pool.fileUsageByTorrent.get(torrent);
-  if (!usage) {
-    return;
-  }
-  for (const fileIndex of usage.keys()) {
+  // The files anything is stated for — a viewer's own picture and soundtrack
+  // through the priority map, and the ends of a file that is open. It replaces
+  // a count of readers, which said the same thing by keeping a second copy of
+  // it.
+  for (const fileIndex of demandFor(torrent).register.files()) {
     const key = `${sourceKey}:${fileIndex}`;
     // A trigger that arrives while the previous pass is still walking is
     // dropped, not queued. `verified` fires per piece, so on a fast download
@@ -1009,18 +1006,21 @@ async function keepWholeFiles() {
     if (!infoHash || !Array.isArray(torrent.files)) {
       continue;
     }
-    const usage = pool.fileUsageByTorrent?.get?.(torrent);
+    // What anybody wants of this torrent. A file something is stated for is a
+    // file somebody may be reading, and this is the same list the reader counts
+    // used to give.
+    const wanted = new Set(demandFor(torrent).register.files());
     for (const [fileIndex, file] of torrent.files.entries()) {
       const key = `${infoHash}/${fileIndex}`;
       if (file?.done !== true || completedFiles.find(infoHash, fileIndex) || beingKept.has(key)) {
         continue;
       }
-      // NOT WHILE SOMEBODY IS READING IT. Writing a film out is a read of the
-      // whole of it and a write of the whole of it — a gigabyte and a half on
-      // the file this was measured against — and doing that beside a viewer
-      // takes the disk and the piece store from them for nothing they asked
-      // for. The file is complete; it will still be complete when they leave.
-      if (usage?.has?.(fileIndex)) {
+      // NOT WHILE ANYBODY WANTS IT. Writing a film out is a read of the whole
+      // of it and a write of the whole of it — a gigabyte and a half on the
+      // file this was measured against — and doing that beside a viewer takes
+      // the disk and the piece store from them for nothing they asked for. The
+      // file is complete; it will still be complete when they leave.
+      if (wanted.has(fileIndex)) {
         continue;
       }
       beingKept.add(key);
@@ -1074,7 +1074,7 @@ async function keepWholeFiles() {
     const isWhole =
       torrent.done === true &&
       torrent.files.every((unused, fileIndex) => completedFiles.find(infoHash, fileIndex) !== null);
-    if (isWhole && !(usage?.size > 0)) {
+    if (isWhole && wanted.size === 0) {
       logger.info(
         `whole files: "${torrent.name}" is downloaded whole and saved — removing the torrent, keeping the files`
       );

package/test/file-edges.test.js

@@ -0,0 +1,191 @@
+/**
+ * @file THE TWO ENDS OF A FILE ARE THE FILE'S, NOT A READ'S.
+ *
+ * A container keeps its directory at one end or the other: `ftyp` and an EBML
+ * header at the front, and for an MP4 that was not written for streaming the
+ * `moov` at the very back. Nothing can be read of such a file until they have
+ * arrived, and they go on being wanted for as long as it is open — a player
+ * asks for the file's shape again at every seek.
+ *
+ * Until 2.83.4 they were wanted by nobody. The prefetch that fetches them is an
+ * ordinary read, and a read withdraws what it states the moment it finishes, so
+ * the ends of an open film were held by nothing at all once the codec probe was
+ * done.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { TorrentPool, stateFileEdges, withdrawFileEdges } from "../services/torrent-pool.js";
+import { demandFor, forgetTorrent } from "../services/download/registry.js";
+import { Urgency } from "../services/demand/index.js";
+
+/** A torrent of two files, of which the second is the film. */
+function torrentOf({ length = 1_000_000 } = {}) {
+  return {
+    infoHash: `hash-${Math.random().toString(36).slice(2)}`,
+    pieceLength: 1024,
+    files: [
+      { offset: 0, length: 4096, name: "notes.nfo" },
+      { offset: 4096, length, name: "film.mkv" }
+    ],
+    _selections: { _items: [] },
+    _select() {},
+    _deselect() {},
+    critical() {}
+  };
+}
+
+/**
+ * @param {object} torrent
+ * @returns {object[]}
+ */
+function edges(torrent) {
+  return demandFor(torrent)
+    .register.windows()
+    .filter((one) => String(one.claimant).startsWith("file-edges:"));
+}
+
+test("one byte is claimed at each end, which is one piece at each end", () => {
+  const torrent = torrentOf({ length: 1_000_000 });
+  try {
+    assert.equal(stateFileEdges(torrent, 1, Urgency.TAIL), true);
+
+    const stated = edges(torrent);
+    assert.equal(stated.length, 2);
+    const head = stated.find((one) => String(one.claimant).endsWith(":head"));
+    const tail = stated.find((one) => String(one.claimant).endsWith(":tail"));
+    // File-relative, like everything else stated about a file: the claim says
+    // "the first byte of this file" and "its last", and what a byte costs to
+    // fetch — a whole piece — is the swarm's business and nobody else's.
+    assert.deepEqual([head.byteStart, head.byteEnd], [0, 0]);
+    assert.deepEqual([tail.byteStart, tail.byteEnd], [999_999, 999_999]);
+    assert.equal(head.fileIndex, 1);
+    assert.equal(tail.fileIndex, 1);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("while somebody is waiting for them they are urgent, and afterwards they are kept", () => {
+  const torrent = torrentOf();
+  try {
+    // The playback plan cannot answer until the file has said what is in it,
+    // and a person is watching a loading screen for as long as that takes.
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.NEAR, Urgency.NEAR]
+    );
+
+    // Once it has answered, nobody is waiting — but a seek will want them
+    // again, so they stay stated at the level of something nobody is waiting
+    // for. Re-stating replaces; it does not pile a second claim on the first.
+    stateFileEdges(torrent, 1, Urgency.TAIL, { lower: true });
+    assert.equal(edges(torrent).length, 2);
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.TAIL, Urgency.TAIL]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("each file of a torrent has its own ends", () => {
+  const torrent = torrentOf();
+  try {
+    stateFileEdges(torrent, 0, Urgency.TAIL);
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    assert.equal(edges(torrent).length, 4);
+
+    withdrawFileEdges(torrent, 0);
+    assert.deepEqual(
+      edges(torrent).map((one) => one.fileIndex),
+      [1, 1]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("a file of unknown length has no ends to claim", () => {
+  const torrent = torrentOf();
+  try {
+    assert.equal(stateFileEdges(torrent, 7, Urgency.TAIL), false);
+    assert.equal(stateFileEdges({ files: [{ length: 0 }] }, 0, Urgency.TAIL), false);
+    assert.equal(edges(torrent).length, 0);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("a map with nothing in it takes the ends back too", () => {
+  const torrent = torrentOf();
+  const applyPriorityMap = TorrentPool.prototype.applyPriorityMap;
+  try {
+    stateFileEdges(torrent, 1, Urgency.TAIL);
+    applyPriorityMap.call(null, torrent, 1, [{ from: 0, to: 600, priority: 100 }], 600);
+    assert.equal(edges(torrent).length, 2);
+
+    // Nobody wants this file any more, and its ends are kept only for as long
+    // as it is open.
+    applyPriorityMap.call(null, torrent, 1, [], 600);
+    assert.equal(edges(torrent).length, 0);
+    assert.deepEqual(demandFor(torrent).register.windows(), []);
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("another file's departure leaves these ends alone", () => {
+  const torrent = torrentOf();
+  const applyPriorityMap = TorrentPool.prototype.applyPriorityMap;
+  try {
+    stateFileEdges(torrent, 0, Urgency.TAIL);
+    stateFileEdges(torrent, 1, Urgency.TAIL);
+
+    applyPriorityMap.call(null, torrent, 0, [], 600);
+
+    assert.deepEqual(
+      edges(torrent).map((one) => one.fileIndex),
+      [1, 1]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("a warm-up cannot take the urgency away from a plan that is waiting", () => {
+  // The two callers of the same ends: a playback plan, which somebody is
+  // watching a loading screen for, and a warm-up, which by its whole purpose
+  // nobody is waiting for. The warm-up fires off the files beside the picture
+  // without awaiting them, so it can arrive second — and lowering the plan's
+  // level there would put a person behind a film somebody else is watching.
+  const torrent = torrentOf();
+  try {
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    stateFileEdges(torrent, 1, Urgency.TAIL);
+
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.NEAR, Urgency.NEAR]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});
+
+test("the read that is over is what puts them back down", () => {
+  const torrent = torrentOf();
+  try {
+    stateFileEdges(torrent, 1, Urgency.NEAR);
+    stateFileEdges(torrent, 1, Urgency.TAIL, { lower: true });
+
+    assert.deepEqual(
+      edges(torrent).map((one) => one.urgency),
+      [Urgency.TAIL, Urgency.TAIL]
+    );
+  } finally {
+    forgetTorrent(torrent);
+  }
+});