@torrent-tv/proxy 2.9.76 → 2.9.78

package/services/torrent-worker/file-claims.js

@@ -0,0 +1,91 @@
+/**
+ * @file Who is holding which file open.
+ *
+ * A claim keeps a file's data from being removed while something is reading it.
+ * Until 2.9.77 claims were keyed by `sourceKey:fileIndex`, which quietly made
+ * them **shared**: a second reader of the same file found the key taken and
+ * added nothing, so the first reader to finish released the claim out from
+ * under the second. The proxy reads one file from several places at once —
+ * ffmpeg's input, the keyframe index, the codec probe, a second viewer — so
+ * this was the normal case, not an edge one.
+ *
+ * The fix is not a counter. A counter restores the arithmetic but keeps the
+ * ambiguity: a release names a file, not a claim, so a duplicate or late
+ * release still decrements someone else's hold and nothing can detect it. Here
+ * every claim gets its own identity and a release names exactly that claim —
+ * so a stray release matches nothing, is reported, and harms no one.
+ */
+
+/**
+ * @typedef {object} FileClaims
+ * @property {(sourceKey: string, fileIndex: number, release: () => void) => string} open
+ * @property {(claimId: string) => boolean} close
+ * @property {() => void} closeAll
+ * @property {number} size
+ */
+
+/**
+ * Track file claims by identity.
+ *
+ * @returns {FileClaims}
+ */
+export function createFileClaims() {
+  /** Claim id → the function that releases that one claim. */
+  const claims = new Map();
+  let counter = 0;
+
+  return {
+    /**
+     * Record a new claim and return its identity.
+     *
+     * Each call is a distinct claim even for the same file — that is the whole
+     * point.
+     *
+     * @param {string} sourceKey
+     * @param {number} fileIndex
+     * @param {() => void} release
+     * @returns {string}
+     */
+    open(sourceKey, fileIndex, release) {
+      counter += 1;
+      // The file is in the id purely so a log line reads usefully; matching is
+      // on the whole string.
+      const claimId = `${sourceKey}:${fileIndex}:${counter}`;
+      claims.set(claimId, release);
+      return claimId;
+    },
+
+    /**
+     * Release one claim. Returns false when there was no such claim, which is
+     * worth logging: it means a release arrived twice, or after teardown.
+     *
+     * @param {string} claimId
+     * @returns {boolean}
+     */
+    close(claimId) {
+      const release = claims.get(claimId);
+      if (!release) {
+        return false;
+      }
+      claims.delete(claimId);
+      release();
+      return true;
+    },
+
+    /**
+     * Release everything — the worker is shutting down.
+     *
+     * @returns {void}
+     */
+    closeAll() {
+      for (const [, release] of claims) {
+        release();
+      }
+      claims.clear();
+    },
+
+    get size() {
+      return claims.size;
+    }
+  };
+}

package/services/torrent-worker/piece-reader.js

@@ -0,0 +1,175 @@
+/**
+ * @file Reading a byte range as positions in shared memory, not as bytes.
+ *
+ * The pieces already live in a `SharedArrayBuffer` the main thread can map. So
+ * the torrent thread does not need to hand over any bytes at all: it can say
+ * *where* a piece sits and let the other side read it there. What crosses the
+ * boundary is two numbers per piece.
+ *
+ * That is the whole point of the exercise. The alternative — copying each piece
+ * into memory we own and transferring it — costs 18.84 ms per 10 MB segment on
+ * the field host, and costs it **on the critical path**, in the thread that is
+ * also running the torrent, at the moment a viewer is waiting for that segment.
+ * Here the copy is gone entirely rather than moved.
+ *
+ * Two obligations come with it, and both are enforced rather than assumed:
+ *
+ *  - a piece being read is **pinned**, so eviction cannot take the memory out
+ *    from under the reader mid-read;
+ *  - the pin is released only once the other thread reports it has finished
+ *    with those bytes — not when they were sent, because nothing was sent.
+ */
+
+import { findSharedStore } from "../piece-store/shared-piece-store.js";
+
+/**
+ * Wait until a piece has been downloaded and verified.
+ *
+ * WebTorrent announces this as `verified`. The bitfield is re-checked after the
+ * listener is attached because the piece can complete in between, and a missed
+ * event here would wait forever.
+ *
+ * @param {import("webtorrent").Torrent} torrent
+ * @param {number} index
+ * @param {{ isCancelled: () => boolean }} cancellation
+ * @returns {Promise<void>}
+ */
+function whenPieceReady(torrent, index, cancellation) {
+  if (torrent.bitfield?.get(index)) {
+    return Promise.resolve();
+  }
+
+  return new Promise((resolve, reject) => {
+    /** @param {number} verifiedIndex */
+    const onVerified = (verifiedIndex) => {
+      if (verifiedIndex === index) {
+        cleanup();
+        resolve();
+      }
+    };
+    const onDestroyed = () => {
+      cleanup();
+      reject(new Error(`Torrent went away while waiting for piece ${index}.`));
+    };
+    // Cancellation is polled rather than pushed: a superseded seek destroys the
+    // read, and without this the wait would outlive it and hold a pin.
+    const poll = setInterval(() => {
+      if (cancellation.isCancelled()) {
+        cleanup();
+        reject(new Error(`Read cancelled while waiting for piece ${index}.`));
+      }
+    }, 250);
+
+    function cleanup() {
+      clearInterval(poll);
+      torrent.removeListener("verified", onVerified);
+      torrent.removeListener("close", onDestroyed);
+    }
+
+    torrent.on("verified", onVerified);
+    torrent.once("close", onDestroyed);
+
+    // The piece may have arrived between the check above and this listener.
+    if (torrent.bitfield?.get(index)) {
+      cleanup();
+      resolve();
+    }
+  });
+}
+
+/**
+ * A fragment of a read: where to find it, and how to let it go.
+ *
+ * @typedef {object} PieceFragment
+ * @property {number} pieceIndex
+ * @property {number} offset - Byte offset into the shared pool.
+ * @property {number} length
+ * @property {() => void} release - Drops this fragment's pin. Call exactly once.
+ */
+
+/**
+ * Walk a byte range of a file, yielding each piece's position in shared memory.
+ *
+ * Yields at most one fragment per piece; the first and last are usually partial.
+ * The caller must `release()` every fragment it receives, including on failure —
+ * an unreleased pin permanently costs a slot.
+ *
+ * @param {object} params
+ * @param {import("webtorrent").Torrent} params.torrent
+ * @param {number} params.fileIndex
+ * @param {number} params.start - Inclusive, relative to the file.
+ * @param {number} params.end - Inclusive, relative to the file.
+ * @param {{ isCancelled: () => boolean }} params.cancellation
+ * @returns {AsyncGenerator<PieceFragment>}
+ */
+export async function* readFragments({ torrent, fileIndex, start, end, cancellation }) {
+  const store = findSharedStore(torrent);
+  if (!store) {
+    throw new Error("This torrent is not backed by a shared piece store.");
+  }
+
+  const file = torrent.files?.[fileIndex];
+  if (!file) {
+    throw new Error(`File ${fileIndex} not found.`);
+  }
+
+  const pieceLength = torrent.pieceLength;
+  // Piece numbers are torrent-wide, so a file's own offsets have to be lifted
+  // into the torrent's address space first.
+  const absoluteStart = file.offset + start;
+  const absoluteEnd = file.offset + end;
+  const firstPiece = Math.floor(absoluteStart / pieceLength);
+  const lastPiece = Math.floor(absoluteEnd / pieceLength);
+
+  // Ask for these pieces first. `select` puts them in the download set at all;
+  // `critical` marks them as wanted now, which is what allows a piece to be
+  // fetched out of sequential order for a reader that is waiting on it.
+  if (typeof torrent.select === "function") {
+    torrent.select(firstPiece, lastPiece, 1);
+  }
+  if (typeof torrent.critical === "function") {
+    torrent.critical(firstPiece, lastPiece);
+  }
+
+  for (let pieceIndex = firstPiece; pieceIndex <= lastPiece; pieceIndex += 1) {
+    if (cancellation.isCancelled()) {
+      return;
+    }
+
+    const pieceStart = pieceIndex * pieceLength;
+    const fromWithinPiece = Math.max(absoluteStart, pieceStart) - pieceStart;
+    const toWithinPiece = Math.min(absoluteEnd, pieceStart + pieceLength - 1) - pieceStart;
+
+    await whenPieceReady(torrent, pieceIndex, cancellation);
+
+    // Pinned BEFORE it is located, and before any await that could let an
+    // eviction run: the offset is only meaningful while the piece is held.
+    store.pin(pieceIndex);
+    let located = null;
+    try {
+      located = await store.reside(pieceIndex);
+    } catch (error) {
+      store.unpin(pieceIndex);
+      throw error;
+    }
+
+    if (!located) {
+      store.unpin(pieceIndex);
+      throw new Error(`Piece ${pieceIndex} is verified but absent from the store.`);
+    }
+
+    let releasedThisPiece = false;
+    yield {
+      pieceIndex,
+      offset: located.offset + fromWithinPiece,
+      length: toWithinPiece - fromWithinPiece + 1,
+      release() {
+        if (releasedThisPiece) {
+          return;
+        }
+        releasedThisPiece = true;
+        store.unpin(pieceIndex);
+      }
+    };
+  }
+}

package/services/torrent-worker/pool-adapter.js

@@ -1,179 +1,188 @@
-/**
- * @file `TorrentPool`'s interface, served from the worker thread.
- *
- * The routes, the planner, the health report and the session manager all reach
- * for a torrent pool and use it the same handful of ways. Rather than rewrite
- * every one of them to thread a `sourceKey` through and await what used to be
- * immediate, this presents the shape they already expect and does the thread
- * hop behind it. Swapping the implementation is then a one-line change at
- * construction, and the call sites are untouched — which is what keeps a change
- * of this size reviewable.
- *
- * Two accommodations are needed, and both are deliberate:
- *
- *  - **`acquireFile` and `prioritizeByteRange` stay synchronous.** They return
- *    nothing the caller inspects, so the command is dispatched and not awaited.
- *    `acquireFile` hands back a release function exactly as before, which sends
- *    its own command when called. Awaiting them would mean touching every call
- *    site for no observable gain.
- *  - **`getTorrent` needs a `sourceKey`.** Torrent objects cannot cross a
- *    thread, so the worker keys them. Callers that have one pass it; the rest
- *    get one derived from the source itself, so the identity stays stable
- *    across calls for the same torrent.
- */
-
-import crypto from "node:crypto";
-import { TorrentWorkerClient } from "./client.js";
-
-/**
- * Stable key for a source, matching how the worker keys its torrents.
- *
- * Derived from the source itself rather than handed out per request, so two
- * routes asking for the same torrent name the same thing on the worker side.
- *
- * @param {"magnet" | "torrent"} sourceType
- * @param {string} source
- * @returns {string}
- */
-function deriveSourceKey(sourceType, source) {
-  return `${sourceType}:${crypto.createHash("sha1").update(source).digest("hex")}`;
-}
-
-/**
- * A torrent pool whose work happens on another thread.
- *
- * See `protocol.js` for why: the torrent was taking ~85% of the main thread and
- * everything owed to a viewer queued behind it.
- */
-export class WorkerTorrentPool {
-  #client;
-  /** Stand-ins by source key, so repeat calls return the same object. */
-  #torrents = new Map();
-
-  /**
-   * @param {{ maxDiskBytes?: number, memoryBytes?: number }} [options]
-   */
-  constructor(options = {}) {
-    this.#client = new TorrentWorkerClient(options);
-  }
-
-  /**
-   * Load (or join) a torrent and return a stand-in for it.
-   *
-   * @param {"magnet" | "torrent"} sourceType
-   * @param {string} source
-   * @returns {Promise<object>}
-   */
-  async getTorrent(sourceType, source) {
-    const sourceKey = deriveSourceKey(sourceType, source);
-    const existing = this.#torrents.get(sourceKey);
-    if (existing) {
-      return existing;
-    }
-    const torrent = await this.#client.getTorrent({ sourceKey, sourceType, source });
-    this.#torrents.set(sourceKey, torrent);
-    return torrent;
-  }
-
-  /**
-   * Claim a file for reading; the returned function releases it.
-   *
-   * Synchronous by design — see the file header.
-   *
-   * @param {object} torrent - A stand-in from {@link getTorrent}.
-   * @param {number} fileIndex
-   * @returns {() => void}
-   */
-  acquireFile(torrent, fileIndex) {
-    const sourceKey = torrent?.sourceKey;
-    if (!sourceKey) {
-      return () => undefined;
-    }
-    // Dispatched, not awaited — callers use the result immediately and inspect
-    // nothing. But the release MUST NOT overtake it: both are ordinary messages
-    // to the worker, and if release arrives first the reader count drops to zero
-    // while a read is still running. The idle sweep then removes the torrent AND
-    // its downloaded data out from under the encoder — field 2026-08-02:
-    // "removed idle torrent ... and its store" mid-playback, after which every
-    // read hung and ffmpeg saw an empty input ("Stream ends prematurely at 0").
-    // Chaining the release onto the acquire keeps them in order.
-    const acquired = this.#client.acquireFile(sourceKey, fileIndex).catch(() => undefined);
-    let released = false;
-    return () => {
-      if (released) {
-        return;
-      }
-      released = true;
-      void acquired.then(() => this.#client.releaseFile(sourceKey, fileIndex)).catch(() => undefined);
-    };
-  }
-
-  /**
-   * Live download figures for the progress display.
-   *
-   * @param {object} torrent
-   * @param {number | null} [fileIndex]
-   * @param {{ resumeAnchorByteStart?: number | null }} [options]
-   * @returns {Promise<object | null>}
-   */
-  async getFileStats(torrent, fileIndex = null, options = {}) {
-    const sourceKey = torrent?.sourceKey;
-    if (!sourceKey) {
-      return null;
-    }
-    return this.#client.getFileStats({
-      sourceKey,
-      fileIndex,
-      resumeAnchorByteStart: options?.resumeAnchorByteStart ?? null
-    });
-  }
-
-  /**
-   * Reorder piece selection around a read position.
-   *
-   * Synchronous by design — see the file header.
-   *
-   * @param {object} torrent
-   * @param {number} fileIndex
-   * @param {number} byteStart
-   * @param {number} [windowBytes]
-   * @returns {void}
-   */
-  prioritizeByteRange(torrent, fileIndex, byteStart, windowBytes) {
-    const sourceKey = torrent?.sourceKey;
-    if (!sourceKey) {
-      return;
-    }
-    void this.#client
-      .prioritizeByteRange({ sourceKey, fileIndex, byteStart, windowBytes })
-      .catch(() => undefined);
-  }
-
-  /**
-   * Pre-fetch the head and tail the codec probe needs.
-   *
-   * @param {object} torrent
-   * @param {number} fileIndex
-   * @param {number} [headBytes]
-   * @param {number} [tailBytes]
-   * @param {number} [timeoutMs]
-   * @returns {Promise<unknown>}
-   */
-  async prefetchFileEdges(torrent, fileIndex, headBytes, tailBytes, timeoutMs) {
-    const sourceKey = torrent?.sourceKey;
-    if (!sourceKey) {
-      return null;
-    }
-    return this.#client.prefetchFileEdges({ sourceKey, fileIndex, headBytes, tailBytes, timeoutMs });
-  }
-
-  /**
-   * Shut the torrent client down and stop the thread.
-   *
-   * @returns {Promise<void>}
-   */
-  async destroyAll() {
-    this.#torrents.clear();
-    await this.#client.destroyAll();
-  }
-}
+/**
+ * @file `TorrentPool`'s interface, served from the worker thread.
+ *
+ * The routes, the planner, the health report and the session manager all reach
+ * for a torrent pool and use it the same handful of ways. Rather than rewrite
+ * every one of them to thread a `sourceKey` through and await what used to be
+ * immediate, this presents the shape they already expect and does the thread
+ * hop behind it. Swapping the implementation is then a one-line change at
+ * construction, and the call sites are untouched — which is what keeps a change
+ * of this size reviewable.
+ *
+ * Two accommodations are needed, and both are deliberate:
+ *
+ *  - **`acquireFile` and `prioritizeByteRange` stay synchronous.** They return
+ *    nothing the caller inspects, so the command is dispatched and not awaited.
+ *    `acquireFile` hands back a release function exactly as before, which sends
+ *    its own command when called. Awaiting them would mean touching every call
+ *    site for no observable gain.
+ *  - **`getTorrent` needs a `sourceKey`.** Torrent objects cannot cross a
+ *    thread, so the worker keys them. Callers that have one pass it; the rest
+ *    get one derived from the source itself, so the identity stays stable
+ *    across calls for the same torrent.
+ */
+
+import crypto from "node:crypto";
+import { TorrentWorkerClient } from "./client.js";
+
+/**
+ * Stable key for a source, matching how the worker keys its torrents.
+ *
+ * Derived from the source itself rather than handed out per request, so two
+ * routes asking for the same torrent name the same thing on the worker side.
+ *
+ * @param {"magnet" | "torrent"} sourceType
+ * @param {string} source
+ * @returns {string}
+ */
+function deriveSourceKey(sourceType, source) {
+  return `${sourceType}:${crypto.createHash("sha1").update(source).digest("hex")}`;
+}
+
+/**
+ * A torrent pool whose work happens on another thread.
+ *
+ * See `protocol.js` for why: the torrent was taking ~85% of the main thread and
+ * everything owed to a viewer queued behind it.
+ */
+export class WorkerTorrentPool {
+  #client;
+  /** Stand-ins by source key, so repeat calls return the same object. */
+  #torrents = new Map();
+
+  /**
+   * @param {{ maxDiskBytes?: number, memoryBytes?: number }} [options]
+   */
+  constructor(options = {}) {
+    this.#client = new TorrentWorkerClient(options);
+  }
+
+  /**
+   * Load (or join) a torrent and return a stand-in for it.
+   *
+   * @param {"magnet" | "torrent"} sourceType
+   * @param {string} source
+   * @returns {Promise<object>}
+   */
+  async getTorrent(sourceType, source) {
+    const sourceKey = deriveSourceKey(sourceType, source);
+    const existing = this.#torrents.get(sourceKey);
+    if (existing) {
+      return existing;
+    }
+    const torrent = await this.#client.getTorrent({ sourceKey, sourceType, source });
+    this.#torrents.set(sourceKey, torrent);
+    return torrent;
+  }
+
+  /**
+   * Claim a file for reading; the returned function releases it.
+   *
+   * Synchronous by design — see the file header.
+   *
+   * @param {object} torrent - A stand-in from {@link getTorrent}.
+   * @param {number} fileIndex
+   * @returns {() => void}
+   */
+  acquireFile(torrent, fileIndex) {
+    const sourceKey = torrent?.sourceKey;
+    if (!sourceKey) {
+      return () => undefined;
+    }
+    // Dispatched, not awaited — callers use the result immediately and inspect
+    // nothing. But the release MUST NOT overtake it: both are ordinary messages
+    // to the worker, and if release arrives first the reader count drops to zero
+    // while a read is still running. The idle sweep then removes the torrent AND
+    // its downloaded data out from under the encoder — field 2026-08-02:
+    // "removed idle torrent ... and its store" mid-playback, after which every
+    // read hung and ffmpeg saw an empty input ("Stream ends prematurely at 0").
+    // Chaining the release onto the acquire keeps them in order.
+    const acquired = this.#client.acquireFile(sourceKey, fileIndex).catch(() => null);
+    let released = false;
+    return () => {
+      if (released) {
+        return;
+      }
+      released = true;
+      // Release the claim this call opened, not "the file" — waiting for the
+      // acquire is also what tells us which claim that is.
+      void acquired
+        .then((claimId) => (claimId ? this.#client.releaseFile(claimId) : undefined))
+        .catch(() => undefined);
+    };
+  }
+
+  /**
+   * Live download figures for the progress display.
+   *
+   * @param {object} torrent
+   * @param {number | null} [fileIndex]
+   * @param {{ resumeAnchorByteStart?: number | null }} [options]
+   * @returns {Promise<object | null>}
+   */
+  async getFileStats(torrent, fileIndex = null, options = {}) {
+    const sourceKey = torrent?.sourceKey;
+    if (!sourceKey) {
+      return null;
+    }
+    return this.#client.getFileStats({
+      sourceKey,
+      fileIndex,
+      resumeAnchorByteStart: options?.resumeAnchorByteStart ?? null
+    });
+  }
+
+  /**
+   * Reorder piece selection around a read position.
+   *
+   * Synchronous by design — see the file header.
+   *
+   * @param {object} torrent
+   * @param {number} fileIndex
+   * @param {number} byteStart
+   * @param {number} [windowBytes]
+   * @returns {void}
+   */
+  prioritizeByteRange(torrent, fileIndex, byteStart, windowBytes) {
+    const sourceKey = torrent?.sourceKey;
+    if (!sourceKey) {
+      return;
+    }
+    void this.#client
+      .prioritizeByteRange({ sourceKey, fileIndex, byteStart, windowBytes })
+      .catch(() => undefined);
+  }
+
+  /**
+   * Pre-fetch the head and tail the codec probe needs.
+   *
+   * Takes an options object, matching `TorrentPool.prefetchFileEdges` — this
+   * adapter exists to present that same interface. It previously declared
+   * positional parameters instead, so the planner's options object arrived as
+   * `headBytes` and only worked because it was passed along far enough to be
+   * destructured at the far end. Anyone calling it as documented got the
+   * defaults instead of the sizes they asked for.
+   *
+   * @param {object} torrent
+   * @param {number} fileIndex
+   * @param {{ headBytes?: number, tailBytes?: number, timeoutMs?: number }} [options]
+   * @returns {Promise<unknown>}
+   */
+  async prefetchFileEdges(torrent, fileIndex, options = {}) {
+    const sourceKey = torrent?.sourceKey;
+    if (!sourceKey) {
+      return null;
+    }
+    return this.#client.prefetchFileEdges({ sourceKey, fileIndex, options });
+  }
+
+  /**
+   * Shut the torrent client down and stop the thread.
+   *
+   * @returns {Promise<void>}
+   */
+  async destroyAll() {
+    this.#torrents.clear();
+    await this.#client.destroyAll();
+  }
+}

package/services/torrent-worker/protocol.js

@@ -80,6 +80,18 @@ export const Event = {
   ERROR: "error",
   /** One piece of a READ_RANGE body; its bytes are transferred, never copied. */
   CHUNK: "chunk",
+  /**
+   * Where a piece of the body sits in the torrent's shared pool — an offset and
+   * a length, no bytes at all. The main thread maps the same memory and reads it
+   * in place; see `piece-reader.js` for why this replaces sending the bytes.
+   */
+  FRAGMENT: "fragment",
+  /**
+   * The main thread has finished with a FRAGMENT and its pin may be dropped.
+   * Distinct from {@link CHUNK_ACK}, which only reports queue capacity: this one
+   * is a promise that nothing is reading those bytes any more.
+   */
+  FRAGMENT_DONE: "fragment-done",
   /** A READ_RANGE ended; no further CHUNKs bear that request id. */
   READ_END: "read-end",
   /** The main thread consumed a chunk — see {@link STREAM_HIGH_WATER_CHUNKS}. */