@torrent-tv/proxy 2.9.88 → 2.9.90

package/services/torrent-pool.js

@@ -306,17 +306,6 @@ export class TorrentPool {
    */
   #readPositionByTorrent = new Map();
 
-  /**
-   * Lowest piece currently selected for download, per torrent and fileIndex.
-   *
-   * Needed because selection is not readable back from WebTorrent, and a seek
-   * backward has to know whether the pieces it wants were deselected by an
-   * earlier seek forward.
-   *
-   * @type {Map<import("webtorrent").Torrent, Map<number, number>>}
-   */
-  #selectedFromPiece = new Map();
-
   /** Global disk cap in bytes (0 = disabled). */
   #maxDiskBytes = 0;
 
@@ -982,8 +971,20 @@ export class TorrentPool {
   }
 
   /**
-   * Update WebTorrent piece selection to match the current usage map.
-   * Files with at least one consumer are selected; all others are deselected.
+   * Drop the pieces of files nobody is reading from the download set.
+   *
+   * It does NOT select the files that ARE in use, and that is the point. What a
+   * file needs is decided by the readers walking it: each one claims a moving
+   * window around its own read head and gives it back when it ends (see
+   * `torrent-worker/piece-reader.js`). Selecting the whole file here as well
+   * put a second, contradictory claim on the same pieces — one that covered
+   * everything and therefore always outranked the window — and it was re-made
+   * on every single `/stream` request, so a seek's prioritisation survived at
+   * most until the next one. Measured consequence: a seek to 89.1% of a 4.7 GB
+   * film waited 93 s while the swarm fetched 2.47 GB in file order.
+   *
+   * A file with no reader is deselected outright, which is what stops a torrent
+   * downloading files the viewer never opened.
    *
    * @param {import("webtorrent").Torrent} torrent
    * @param {Map<number, number>} usage - fileIndex → refCount.
@@ -995,14 +996,7 @@ export class TorrentPool {
     }
     for (let index = 0; index < torrent.files.length; index += 1) {
       const file = torrent.files[index];
-      if (!file) {
-        continue;
-      }
-      const shouldSelect = (usage.get(index) ?? 0) > 0;
-      if (shouldSelect) {
-        if (typeof file.select === "function") {
-          file.select();
-        }
+      if (!file || (usage.get(index) ?? 0) > 0) {
         continue;
       }
       if (typeof file.deselect === "function") {
@@ -1012,42 +1006,27 @@ export class TorrentPool {
   }
 
   /**
-   * Bias the torrent's download toward the current read position, so a seek
-   * downloads the seek target first instead of waiting behind the sequential
-   * backlog (which caused ~15-18 s stalls when seeking into an undownloaded
-   * region). Called on every range request.
+   * Record where a file is being read from.
    *
-   * Two levers, matched to how WebTorrent's picker actually works:
+   * This used to also decide what the torrent should download, and that was the
+   * mistake: it was one of THREE places claiming pieces for the same file — the
+   * whole-file `file.select()` in `#syncSelections`, this method, and the reader
+   * itself — and they overwrote each other on every request. The claim now
+   * belongs to the reader alone, which holds a moving window around its own read
+   * head and gives it back when it ends
+   * (`torrent-worker/piece-reader.js`); several readers on one file therefore
+   * produce the union of their windows instead of the last caller's opinion.
    *
-   * 1. **Demote the gap BEHIND the playhead** — `deselect(fileStart, playhead-1)`.
-   *    The picker scans each selection sequentially from its first UNdownloaded
-   *    piece; with the whole file selected, a far forward seek would make it
-   *    fetch the undownloaded gap behind the new position first. Removing that
-   *    gap from the selection makes the scan START at the playhead, so all peer
-   *    capacity goes to the pieces the player needs next. This only STOPS
-   *    fetching the gap; already-downloaded pieces stay on disk (deleting them
-   *    is Disk hygiene Level 2), and a later backward seek re-selects the region
-   *    via this same call. The whole file is re-selected by `file.select()` on
-   *    the next `acquireFile`, so nothing is permanently dropped.
-   *
-   * 2. **Critical read-ahead window** — `critical(playhead, playhead+window)`.
-   *    `critical` does not reorder the scan; it enables HOTSWAP (re-request a
-   *    block from a faster peer when a slow one reserved it) over the near
-   *    window. Reset first so criticality stays a moving window rather than
-   *    accumulating over the whole file across seeks.
-   *
-   * Scope: single active reader per file (≈100% today). The multi-viewer union
-   * window — demote only where behind for ALL sessions — is deferred (roadmap
-   * item 23); here the latest read position wins.
-   *
-   * The pinned head/tail (prefetchFileEdges, codec probe) is downloaded up front
-   * and lives forward of the playhead (tail) or is already on disk (head), so
-   * demotion never costs the probe its data.
+   * What is left here is bookkeeping the readers cannot do: `getFileStats`
+   * reports how much of the window ahead of the read head is still missing, so
+   * the viewer can be shown how long a resume will take, and a jump in the read
+   * position is logged because a seek that never reaches the torrent is
+   * invisible otherwise.
    *
    * @param {import("webtorrent").Torrent} torrent
    * @param {number} fileIndex
    * @param {number} byteStart - Start offset within the file.
-   * @param {number} [windowBytes] - Bytes ahead of `byteStart` to mark critical.
+   * @param {number} [windowBytes] - Unused; kept so callers need not change.
    * @param {{ wholeFileRead?: boolean }} [options] - `wholeFileRead` marks a
    *   request that carried no byte range, i.e. one that merely opens the file at
    *   0 rather than asking to read from there. See the guard below.
@@ -1060,24 +1039,17 @@ export class TorrentPool {
     windowBytes = PRIORITY_WINDOW_BYTES,
     options = {}
   ) {
-    if (!torrent || typeof torrent.critical !== "function" || !Array.isArray(torrent.files)) {
-      return;
-    }
-    const pieceLength = Number(torrent.pieceLength);
-    if (!Number.isFinite(pieceLength) || pieceLength <= 0) {
+    if (!torrent || !Array.isArray(torrent.files)) {
       return;
     }
     const file = torrent.files[fileIndex];
     if (!file) {
       return;
     }
-    const fileOffset = Number.isFinite(file.offset) ? file.offset : 0;
     const fileLength = Number(file.length);
     if (!Number.isFinite(fileLength) || fileLength <= 0) {
       return;
     }
-    const fileStartPiece = Math.floor(fileOffset / pieceLength);
-    const fileEndPiece = Math.floor((fileOffset + fileLength - 1) / pieceLength);
 
     const safeStart = Math.max(0, Number(byteStart) || 0);
 
@@ -1120,70 +1092,6 @@ export class TorrentPool {
       );
     }
 
-    const absStart = fileOffset + safeStart;
-    const playheadPiece = Math.floor(absStart / pieceLength);
-    const absWindowEnd = Math.min(
-      fileOffset + fileLength - 1,
-      absStart + Math.max(1, windowBytes) - 1
-    );
-    const windowEndPiece = Math.floor(absWindowEnd / pieceLength);
-
-    // (1) Re-select from the playhead when it moved BACK behind what an earlier
-    //     seek deselected. `deselect` removes pieces from the download set, and
-    //     `critical` does NOT put them back — it only flags pieces already
-    //     selected. So without this, a seek forward followed by a seek backward
-    //     leaves the target pieces wanted by nobody: the encoder waits on data
-    //     the torrent was told to stop fetching, and waits forever.
-    //     Only on a backward move, so repeat calls do not pile up selections.
-    let selectedFrom = this.#selectedFromPiece.get(torrent)?.get(fileIndex);
-    if (selectedFrom === undefined || playheadPiece < selectedFrom) {
-      try {
-        torrent.select(playheadPiece, fileEndPiece, 1);
-      } catch {
-        // Best effort — never break streaming because selection failed.
-      }
-      let perFile = this.#selectedFromPiece.get(torrent);
-      if (!perFile) {
-        perFile = new Map();
-        this.#selectedFromPiece.set(torrent, perFile);
-      }
-      perFile.set(fileIndex, playheadPiece);
-      selectedFrom = playheadPiece;
-    }
-
-    // (2) Demote the gap behind the playhead so the picker scans forward from
-    //     the read position. Only when there IS a gap (not at the file start).
-    if (playheadPiece > fileStartPiece && typeof torrent.deselect === "function") {
-      try {
-        torrent.deselect(fileStartPiece, playheadPiece - 1);
-        // `deselect` subtracts the interval and copies the selection's `offset`
-        // — how many pieces from its start are already downloaded — into what
-        // remains. The picker scans from `from + offset`, so the surviving
-        // selection starts scanning far past its own end and can never yield a
-        // piece: measured with the library's own `Selections`, deselecting
-        // 0-522 from `{0-587, offset 226}` leaves `{523-587, offset 226}`,
-        // i.e. a scan starting at piece 749 of 587. The seek target ends up
-        // wanted by nobody. Re-selecting the same range replaces that dead
-        // entry with a fresh one whose offset is 0.
-        torrent.select(playheadPiece, fileEndPiece, 1);
-        this.#selectedFromPiece.get(torrent)?.set(fileIndex, playheadPiece);
-      } catch {
-        // Best effort — never break streaming because demotion failed.
-      }
-    }
-
-    // (3) Reset criticality to a moving read-ahead window (hotswap over the near
-    //     pieces), so it does not accumulate over the whole file across seeks.
-    if (Array.isArray(torrent._critical)) {
-      torrent._critical.length = 0;
-    }
-    if (windowEndPiece >= playheadPiece) {
-      try {
-        torrent.critical(playheadPiece, windowEndPiece);
-      } catch {
-        // Best effort.
-      }
-    }
   }
 
   /**

package/services/torrent-worker/piece-reader.js

@@ -22,6 +22,145 @@
 
 import { findSharedStore } from "../piece-store/shared-piece-store.js";
 
+/**
+ * How far ahead of the read head pieces are asked for.
+ *
+ * A read is open-ended — ffmpeg opens its input as `bytes <position>-<EOF>` and
+ * keeps it for the whole film — so taking the requested range literally asks
+ * for everything from the seek point to the end of the file at once. That is
+ * what a seek used to do: the swarm was told the entire tail was wanted, went
+ * at it from its first missing piece, and the one piece the decoder was blocked
+ * on arrived only when the sequential scan reached it. Measured on a 4.7 GB
+ * film: a seek to 89.1% took 93 s and pulled 2.47 GB.
+ *
+ * So the reader asks for a window and moves it as it goes. The size is a
+ * compromise the caller cannot yet express: the right unit is seconds of
+ * playback (duration and size are both known — to the transcode session, not to
+ * this thread), and 32 MB is about 34 s of a 1080p film but only a few seconds
+ * of a disc remux. Sizing it from the real byte rate is a follow-up; what
+ * matters here is that it is bounded and moving rather than "to the end".
+ */
+const READ_WINDOW_BYTES = 32 * 1024 * 1024;
+
+/**
+ * How many pieces past the one being waited for are marked critical.
+ *
+ * `critical` means "a reader is blocked on this now" — it is what lets a piece
+ * jump the sequential scan. Marking a whole range critical, as this did, says
+ * it about hundreds of pieces at once and the signal stops meaning anything.
+ * WebTorrent's own reader marks `min(1 MB / pieceLength, 2)` pieces, i.e. the
+ * one under the head and at most two more; the same rule is used here.
+ *
+ * @param {number} pieceLength
+ * @returns {number}
+ */
+function criticalRunLength(pieceLength) {
+  return Math.min(Math.floor((1024 * 1024) / Math.max(1, pieceLength)), 2);
+}
+
+/**
+ * The pieces a reader at `pieceIndex` wants next, clamped to its own range.
+ *
+ * @param {{ pieceIndex: number, lastPiece: number, windowPieces: number }} params
+ * @returns {{ from: number, to: number }}
+ */
+export function readWindowFor({ pieceIndex, lastPiece, windowPieces }) {
+  const span = Math.max(1, windowPieces);
+  return { from: pieceIndex, to: Math.min(lastPiece, pieceIndex + span - 1) };
+}
+
+/**
+ * Add this reader's window to the download set as a stream selection.
+ *
+ * `_select`/`_deselect` with the stream flag are what WebTorrent's own
+ * `FileIterator` uses; there is no public call for it, because the public
+ * `select` produces the merging, interval-subtracted kind whose bookkeeping
+ * cannot express "one of several readers wants this". Falls back to the public
+ * call if a future version drops the private one.
+ *
+ * @param {import("webtorrent").Torrent} torrent
+ * @param {{ from: number, to: number }} window
+ * @returns {void}
+ */
+function claimWindow(torrent, { from, to }) {
+  try {
+    if (typeof torrent._select === "function") {
+      torrent._select(from, to, 1, null, true);
+    } else if (typeof torrent.select === "function") {
+      torrent.select(from, to, 1);
+    }
+  } catch {
+    // Best effort — never fail a read because selection bookkeeping refused.
+  }
+}
+
+/**
+ * Take this reader's window back out of the download set.
+ *
+ * The bounds must match the ones given to {@link claimWindow} exactly: a stream
+ * selection is removed by equality, not by overlap.
+ *
+ * @param {import("webtorrent").Torrent} torrent
+ * @param {{ from: number, to: number }} window
+ * @returns {void}
+ */
+function releaseWindow(torrent, { from, to }) {
+  try {
+    if (typeof torrent._deselect === "function") {
+      torrent._deselect(from, to, true);
+    } else if (typeof torrent.deselect === "function") {
+      torrent.deselect(from, to);
+    }
+  } catch {
+    // Best effort.
+  }
+}
+
+/**
+ * Mark the piece a reader is blocked on, clearing the mark it set before.
+ *
+ * Criticality is never cleared by WebTorrent itself, so a reader that walked a
+ * film would leave every piece of it marked. Only the indices this reader set
+ * are cleared, so a second reader's mark on the same piece is not stolen — and
+ * the flag is advisory anyway.
+ *
+ * @param {import("webtorrent").Torrent} torrent
+ * @param {number} from
+ * @param {number} to
+ * @param {{ from: number, to: number } | null} previous
+ * @returns {{ from: number, to: number } | null}
+ */
+function markCritical(torrent, from, to, previous) {
+  if (previous && previous.from === from && previous.to === to) {
+    return previous;
+  }
+  if (previous) {
+    clearCritical(torrent, previous);
+  }
+  try {
+    torrent.critical?.(from, to);
+  } catch {
+    return null;
+  }
+  return { from, to };
+}
+
+/**
+ * Drop critical marks this reader set.
+ *
+ * @param {import("webtorrent").Torrent} torrent
+ * @param {{ from: number, to: number }} mark
+ * @returns {void}
+ */
+function clearCritical(torrent, { from, to }) {
+  if (!Array.isArray(torrent._critical)) {
+    return;
+  }
+  for (let index = from; index <= to; index += 1) {
+    torrent._critical[index] = false;
+  }
+}
+
 /**
  * Wait until a piece has been downloaded and verified.
  *
@@ -100,9 +239,19 @@ function whenPieceReady(torrent, index, cancellation) {
  * @param {number} params.start - Inclusive, relative to the file.
  * @param {number} params.end - Inclusive, relative to the file.
  * @param {{ isCancelled: () => boolean }} params.cancellation
+ * @param {number} [params.windowBytes] - How far ahead of the read head to ask
+ *   the swarm for. Defaults to {@link READ_WINDOW_BYTES}; a caller that knows
+ *   the media's byte rate should size it in seconds of playback instead.
  * @returns {AsyncGenerator<PieceFragment>}
  */
-export async function* readFragments({ torrent, fileIndex, start, end, cancellation }) {
+export async function* readFragments({
+  torrent,
+  fileIndex,
+  start,
+  end,
+  cancellation,
+  windowBytes = READ_WINDOW_BYTES
+}) {
   const store = findSharedStore(torrent);
   if (!store) {
     throw new Error("This torrent is not backed by a shared piece store.");
@@ -121,55 +270,95 @@ export async function* readFragments({ torrent, fileIndex, start, end, cancellat
   const firstPiece = Math.floor(absoluteStart / pieceLength);
   const lastPiece = Math.floor(absoluteEnd / pieceLength);
 
-  // Ask for these pieces first. `select` puts them in the download set at all;
-  // `critical` marks them as wanted now, which is what allows a piece to be
-  // fetched out of sequential order for a reader that is waiting on it.
-  if (typeof torrent.select === "function") {
-    torrent.select(firstPiece, lastPiece, 1);
-  }
-  if (typeof torrent.critical === "function") {
-    torrent.critical(firstPiece, lastPiece);
-  }
+  // This reader owns what it asks for, and gives it back when it is done. The
+  // window is a STREAM selection: those are removed by exact bounds and several
+  // identical ones coexist — WebTorrent's own source calls that "in a way a
+  // count" — so N readers on one torrent produce the union of their windows,
+  // and each one leaving takes away only its own. That is what makes several
+  // parallel readers (the codec probe's head and tail, subtitles, one input per
+  // viewer) cooperate instead of overwrite each other.
+  //
+  // The previous code selected the whole requested range, marked all of it
+  // critical, and never deselected anything — so ffmpeg's opening
+  // `bytes 0-<EOF>` left a permanent selection over the entire file, and no
+  // later prioritisation could outrank it.
+  const windowPieces = Math.max(1, Math.ceil(Math.max(1, windowBytes) / pieceLength));
+  const criticalRun = criticalRunLength(pieceLength);
+  /** @type {{ from: number, to: number } | null} */
+  let window = null;
+  /** @type {{ from: number, to: number } | null} */
+  let criticalMark = null;
 
-  for (let pieceIndex = firstPiece; pieceIndex <= lastPiece; pieceIndex += 1) {
-    if (cancellation.isCancelled()) {
+  const moveWindowTo = (pieceIndex) => {
+    const next = readWindowFor({ pieceIndex, lastPiece, windowPieces });
+    if (window && window.from === next.from && window.to === next.to) {
       return;
     }
-
-    const pieceStart = pieceIndex * pieceLength;
-    const fromWithinPiece = Math.max(absoluteStart, pieceStart) - pieceStart;
-    const toWithinPiece = Math.min(absoluteEnd, pieceStart + pieceLength - 1) - pieceStart;
-
-    await whenPieceReady(torrent, pieceIndex, cancellation);
-
-    // Pinned BEFORE it is located, and before any await that could let an
-    // eviction run: the offset is only meaningful while the piece is held.
-    store.pin(pieceIndex);
-    let located = null;
-    try {
-      located = await store.reside(pieceIndex);
-    } catch (error) {
-      store.unpin(pieceIndex);
-      throw error;
+    if (window) {
+      releaseWindow(torrent, window);
     }
+    claimWindow(torrent, next);
+    window = next;
+  };
 
-    if (!located) {
-      store.unpin(pieceIndex);
-      throw new Error(`Piece ${pieceIndex} is verified but absent from the store.`);
-    }
+  try {
+    for (let pieceIndex = firstPiece; pieceIndex <= lastPiece; pieceIndex += 1) {
+      if (cancellation.isCancelled()) {
+        return;
+      }
 
-    let releasedThisPiece = false;
-    yield {
-      pieceIndex,
-      offset: located.offset + fromWithinPiece,
-      length: toWithinPiece - fromWithinPiece + 1,
-      release() {
-        if (releasedThisPiece) {
-          return;
-        }
-        releasedThisPiece = true;
+      const pieceStart = pieceIndex * pieceLength;
+      const fromWithinPiece = Math.max(absoluteStart, pieceStart) - pieceStart;
+      const toWithinPiece = Math.min(absoluteEnd, pieceStart + pieceLength - 1) - pieceStart;
+
+      moveWindowTo(pieceIndex);
+
+      if (!torrent.bitfield?.get(pieceIndex)) {
+        // Blocked here and now — this is the one case `critical` is meant for.
+        criticalMark = markCritical(torrent, pieceIndex, Math.min(lastPiece, pieceIndex + criticalRun), criticalMark);
+      }
+
+      await whenPieceReady(torrent, pieceIndex, cancellation);
+
+      // Pinned BEFORE it is located, and before any await that could let an
+      // eviction run: the offset is only meaningful while the piece is held.
+      store.pin(pieceIndex);
+      let located = null;
+      try {
+        located = await store.reside(pieceIndex);
+      } catch (error) {
         store.unpin(pieceIndex);
+        throw error;
       }
-    };
+
+      if (!located) {
+        store.unpin(pieceIndex);
+        throw new Error(`Piece ${pieceIndex} is verified but absent from the store.`);
+      }
+
+      let releasedThisPiece = false;
+      yield {
+        pieceIndex,
+        offset: located.offset + fromWithinPiece,
+        length: toWithinPiece - fromWithinPiece + 1,
+        release() {
+          if (releasedThisPiece) {
+            return;
+          }
+          releasedThisPiece = true;
+          store.unpin(pieceIndex);
+        }
+      };
+    }
+  } finally {
+    // Reached on completion, on cancellation, on a throw, and when the consumer
+    // stops iterating — a window left behind would keep the swarm fetching for
+    // a reader that no longer exists.
+    if (window) {
+      releaseWindow(torrent, window);
+    }
+    if (criticalMark) {
+      clearCritical(torrent, criticalMark);
+    }
   }
 }