@torrent-tv/proxy 2.82.0 → 2.83.1

package/services/torrent-pool.js

@@ -8,6 +8,7 @@
  * download.
  */
 
+import { IDLE_KEEP_MS } from "./disk/keep.js";
 import dns from "node:dns/promises";
 import os from "node:os";
 import path from "node:path";
@@ -16,7 +17,7 @@ import WebTorrent from "webtorrent";
 import { logger } from "../utils/logger.js";
 import { SharedPieceStore, findSharedStore } from "./piece-store/shared-piece-store.js";
 import { Urgency, urgencyName } from "./demand/index.js";
-import { demandFor, forgetTorrent, reconcileAll } from "./download/registry.js";
+import { demandFor, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
 import { isAtAWatchingViewer, isBehindEverybody, isNobodyComingNow } from "./priority/PriorityMap.js";
 import { deriveSourceKey } from "./torrent-source-key.js";
 
@@ -61,13 +62,13 @@ const DHT_RESOLVE_TIMEOUT_MS = 2000;
 // torrent data lives under here.
 const WEBTORRENT_STORE_ROOT = path.join(os.tmpdir(), "webtorrent");
 
-// How long a torrent may sit with zero active file readers before it is
-// removed (with its on-disk store). Generous so brief gaps between ffmpeg
-// range reads — a pause, a backgrounded tab, or a phone turned off for a few
-// minutes — do not evict an in-use torrent's already-downloaded data, so a
-// resume plays from disk instead of re-downloading. A longer idle (viewer truly
-// gone) frees the disk; the global disk cap still evicts earlier under pressure.
-const TORRENT_IDLE_TTL_MS = 15 * 60 * 1000;
+// How long a torrent may sit with zero active file readers before it is removed
+// with its on-disk store. ONE NUMBER for everything nobody is using, shared with
+// the produced segments — see `services/disk/keep.js` for why it is one and what
+// it stands for. It was fifteen minutes here against thirty for the session it
+// feeds, so a viewer returning at the twentieth minute got a session whose source
+// had gone.
+const TORRENT_IDLE_TTL_MS = IDLE_KEEP_MS;
 
 // Bytes ahead of a read position to mark CRITICAL on each range request. In
 // WebTorrent, `critical` does NOT reorder the sequential piece scan — it enables
@@ -163,10 +164,19 @@ const STALL_REPORT_INTERVAL_MS = 30_000;
  * makes every two seconds.
  *
  * @param {import("webtorrent").Torrent} torrent
- * @returns {{ connectedPeers: number, knownPeers: number | null, queuedPeers: number | null }}
+ * @returns {{ connectedPeers: number, deliveringPeers: number, knownPeers: number | null,
+ *   queuedPeers: number | null }}
  */
 export function describeSwarmReach(torrent) {
   const wires = Array.isArray(torrent?.wires) ? torrent.wires.length : 0;
+  // HOW MANY OF THEM ARE DOING ANYTHING. Connections accumulated without bound
+  // in the field — 249 to 596 over one viewing, 15 862 more queued — and
+  // whether that helped or merely cost memory and requests is not answerable
+  // from a count of wires. A wire that has delivered a byte is the measured
+  // unit; the rest are held open for nothing.
+  const delivering = Array.isArray(torrent?.wires)
+    ? torrent.wires.filter((wire) => Number(wire?.downloaded) > 0).length
+    : 0;
   const read = (value) => (typeof value === "number" && Number.isFinite(value) ? value : null);
   let knownPeers = null;
   let queuedPeers = null;
@@ -178,7 +188,7 @@ export function describeSwarmReach(torrent) {
     // destroyed torrent, and the reading is a diagnostic. Nothing here is
     // worth failing a stats poll for.
   }
-  return { connectedPeers: wires, knownPeers, queuedPeers };
+  return { connectedPeers: wires, deliveringPeers: delivering, knownPeers, queuedPeers };
 }
 
 /**
@@ -344,6 +354,11 @@ export function torrentsForUploadPolicy(torrents, usageByTorrent, now) {
       // Recorded so the policy can tell "nothing is arriving and somebody is
       // waiting" from "nothing is arriving because nobody asked".
       torrent.hasActiveReader = hasReader;
+      // Whether anybody is still short of bytes of it. Upload is bought with
+      // reciprocity and reciprocity is only worth buying while something is
+      // missing; a torrent whose declared windows are all present wants nothing
+      // from the swarm, and what it gives it gives for nobody.
+      torrent.hasUnmetDemand = hasUnmetDemand(torrent);
       chosen.push(torrent);
     }
   }
@@ -389,7 +404,8 @@ export function decideUploadLimit(activeTorrents, opts = {}) {
     // 2026-08-04: four cycles of 512 -> 50 KB/s in three minutes, each
     // reported as `earn unchoke ... down=0KB/s`, all of them raising the
     // upload at moments when no byte was wanted by anyone.
-    const starving = notDone && torrent?.hasActiveReader !== false && downloadSpeed < starvingSpeed;
+    const starving = notDone && torrent?.hasActiveReader !== false
+      && torrent?.hasUnmetDemand !== false && downloadSpeed < starvingSpeed;
     if (starving && chokedInterested >= chokedThreshold) {
       const name = typeof torrent?.name === "string" ? torrent.name : "?";
       return {
@@ -401,6 +417,14 @@ export function decideUploadLimit(activeTorrents, opts = {}) {
     }
   }
 
+  // NOTHING TO BUY. Every torrent here has a reader, and not one of them is
+  // short of a byte anybody asked for: the upload would be given for nobody.
+  // Field 2026-09-11 — 512 KB/s held for forty-eight minutes on a file that was
+  // complete, 596 peers connected and climbing, and every 16 KB served read a
+  // 4 MB piece back off the disk.
+  if (activeTorrents.every((torrent) => torrent?.hasUnmetDemand === false)) {
+    return { bytesPerSec: idleFloor, reason: "nothing anybody asked for is missing — upload buys nothing" };
+  }
   return { bytesPerSec: floor, reason: "active readers, not choke-starved" };
 }
 
@@ -782,6 +806,52 @@ export class TorrentPool {
    *   default is computed from free disk (min(10 GB, half free)). Pass 0 to
    *   disable the cap.
    */
+  /**
+   * Sources every file of which this proxy holds whole.
+   *
+   * A torrent added again for one of these has nothing to fetch and nothing to
+   * check: what it would verify was written out of pieces this client had
+   * already hashed, and the file's size was checked against what the torrent
+   * says. So it is added with verification skipped — otherwise a viewer
+   * returning to a film waits for a gigabyte and a half to be hashed to learn
+   * what we wrote down.
+   *
+   * @type {Set<string>}
+   */
+  #wholeSources = new Set();
+
+  /**
+   * Say that every file of a source is held whole.
+   *
+   * @param {string} sourceKey
+   * @returns {void}
+   */
+  addWholeSource(sourceKey) {
+    this.#wholeSources.add(String(sourceKey));
+  }
+
+  /**
+   * Anything else every store of this pool is built with.
+   *
+   * Today: where a piece can be had when neither of the store's own tiers has
+   * it — the files this proxy has assembled. Given to the pool rather than
+   * discovered by the store, which knows nothing about files.
+   *
+   * @type {object}
+   */
+  #storeExtras = {};
+
+  /**
+   * Say where a piece can be had when the store has neither a resident nor a
+   * spilled copy.
+   *
+   * @param {object} extras
+   * @returns {void}
+   */
+  buildStoresWith(extras) {
+    this.#storeExtras = extras && typeof extras === "object" ? extras : {};
+  }
+
   constructor({ maxDiskBytes, memoryBytes, dhtBootstrap } = {}) {
     this.#memoryBytes = Number.isFinite(memoryBytes) && memoryBytes > 0 ? memoryBytes : undefined;
 
@@ -850,8 +920,8 @@ export class TorrentPool {
       ? maxDiskBytes
       : computeDefaultDiskCap(os.tmpdir());
     if (this.#maxDiskBytes > 0) {
-      const gb = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
-      logger.info(`torrent-pool: disk cap ${gb} GB (LRU eviction of idle torrents above it)`);
+      const gigabytes = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
+      logger.info(`torrent-pool: disk cap ${gigabytes} GB (LRU eviction of idle torrents above it)`);
       this.#diskSweepTimer = setInterval(() => this.#enforceDiskCap(), DISK_CAP_SWEEP_INTERVAL_MS);
       this.#diskSweepTimer.unref?.();
     }
@@ -868,6 +938,67 @@ export class TorrentPool {
     this.#uploadAdjustTimer.unref?.();
   }
 
+  /**
+   * Leave the swarm of a torrent nobody is reading, and rejoin it when
+   * somebody is.
+   *
+   * A PROXY THAT NEEDS NOTHING FROM A SWARM SHOULD NOT BE IN THAT SWARM. While
+   * a file is being watched every connection is worth keeping — the one that
+   * has delivered nothing yet may deliver next — so this is not a limit on
+   * connections and never fires during playback. It fires when NOTHING is
+   * stated about this torrent at all: no window from any reader, no file held.
+   *
+   * What that state cost until now: WebTorrent enforces `maxConns` only on
+   * peers it dials (`_drain`), while `_addIncomingPeer` checks that the torrent
+   * is neither destroyed nor paused and registers the peer. A proxy with its
+   * port mapped is reachable, so connections arrive and are never turned away —
+   * field 2026-09-11, 249 connected at the start of one viewing and 596 at the
+   * end, 15 862 more queued, on a file complete for three quarters of an hour,
+   * each of them served by reading a 4 MB piece off the disk for every 16 KB
+   * sent.
+   *
+   * `paused` is the library's own word for this and the path it already checks,
+   * so nothing here fights it. The pause alone does not close what is already
+   * open, so the peers are let go by hand; the data stays exactly where it is,
+   * and the next reader resumes the torrent rather than fetching it again.
+   *
+   * @param {import("webtorrent").Torrent} torrent
+   * @returns {void}
+   */
+  followTheReaders(torrent) {
+    if (!torrent || torrent.destroyed) {
+      return;
+    }
+    const usage = this.fileUsageByTorrent.get(torrent);
+    const isRead = Boolean(usage && usage.size > 0);
+    const isWanted = isRead || demandFor(torrent).register.windows().length > 0;
+    if (isWanted && torrent.paused === true) {
+      torrent.resume();
+      logger.info(
+        `torrent-pool: [${String(torrent.infoHash ?? "?").slice(0, 8)}] rejoined the swarm — somebody is reading it again`
+      );
+      return;
+    }
+    if (isWanted || torrent.paused === true) {
+      return;
+    }
+    torrent.pause();
+    const open = Array.isArray(torrent.wires) ? torrent.wires.length : 0;
+    let closed = 0;
+    for (const peer of [...(torrent._peers?.values?.() ?? [])]) {
+      try {
+        peer.destroy();
+        closed += 1;
+      } catch {
+        // A peer already going: nothing to do, and nothing worth failing for.
+      }
+    }
+    logger.info(
+      `torrent-pool: [${String(torrent.infoHash ?? "?").slice(0, 8)}] left the swarm — nobody is reading it, ` +
+      `${open} connection(s) open, ${closed} let go; the data stays and the next reader rejoins`
+    );
+  }
+
   /**
    * Re-evaluate and apply the client-wide upload limit from current swarm state
    * (see {@link decideUploadLimit}). Runs on a timer; only calls into WebTorrent
@@ -1200,6 +1331,9 @@ export class TorrentPool {
       this.#stateBackgroundFill(torrent);
     }
     this.#reportStalledDownloads();
+    for (const torrent of this.torrents.values()) {
+      this.followTheReaders(torrent);
+    }
     const { bytesPerSec, reason } = decideUploadLimit(active);
     if (bytesPerSec === this.#uploadLimit) {
       return;
@@ -1240,11 +1374,14 @@ export class TorrentPool {
     }
     // Candidates: pooled torrents with zero active readers, LRU first.
     const candidates = [...this.torrents.values()]
-      .filter((t) => {
-        const usage = this.fileUsageByTorrent.get(t);
+      .filter((torrent) => {
+        const usage = this.fileUsageByTorrent.get(torrent);
         return !usage || usage.size === 0;
       })
-      .sort((a, b) => (this.#lastAccess.get(a) ?? 0) - (this.#lastAccess.get(b) ?? 0));
+      .sort(
+        (earlier, later) =>
+          (this.#lastAccess.get(earlier) ?? 0) - (this.#lastAccess.get(later) ?? 0)
+      );
 
     for (const torrent of candidates) {
       if (used <= this.#maxDiskBytes) {
@@ -1252,9 +1389,9 @@ export class TorrentPool {
       }
       const freed = Math.max(0, torrentDownloadedBytes(torrent));
       const name = typeof torrent?.name === "string" ? torrent.name : "(unknown)";
-      const gb = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
+      const gigabytes = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
       logger.info(
-        `torrent-pool: disk cap ${gb} GB exceeded — evicting idle torrent "${name}" ` +
+        `torrent-pool: disk cap ${gigabytes} GB exceeded — evicting idle torrent "${name}" ` +
           `(~${(freed / (1024 * 1024)).toFixed(0)} MB)`
       );
       this.#cancelIdleRemoval(torrent);
@@ -1329,6 +1466,21 @@ export class TorrentPool {
       logger.warn(`torrent-pool: [${label()}] warning: ${formatWarning(warning)}`);
     });
 
+    // A TORRENT THAT DIED WITHOUT US ASKING LEAVES ITS RECORD BEHIND, and the
+    // record goes on answering. Field 2026-09-11: the store refused a block,
+    // the client destroyed the torrent, and for the rest of the process every
+    // read answered `File 1 not found in torrent:d4022ff4…` while `/stats`
+    // reported `peers=0 connected of 1186 known` — so the browser's own remedy,
+    // adding the source again, returned the corpse and the viewer could not
+    // open anything until the addon was restarted.
+    //
+    // Forgetting the record is the whole fix: the next request builds the
+    // torrent again, from the same magnet, against the same data on disk.
+    torrent.on("error", (error) => {
+      logger.error(`torrent-pool: [${label()}] died: ${formatWarning(error)}`);
+      this.#forgetDeadTorrent(torrent);
+    });
+
     // Everything below needs the torrent to have been PARSED, and `add` returns
     // before that: `announce`, `files` and `private` are all still empty, and
     // `discovery` — which owns the tracker client — is not created until
@@ -1436,7 +1588,7 @@ export class TorrentPool {
         const message = error instanceof Error ? error.message : String(error);
         const dupMatch = /duplicate torrent ([0-9a-f]{40})/i.exec(message);
         if (dupMatch) {
-          const existing = this.client.torrents.find((t) => t?.infoHash === dupMatch[1]);
+          const existing = this.client.torrents.find((candidate) => candidate?.infoHash === dupMatch[1]);
           // A torrent already here but WITHOUT metadata is not an answer to a
           // request that carries metadata. A magnet whose swarm never answered
           // leaves exactly that: an entry with the right infohash, no file
@@ -1500,7 +1652,7 @@ export class TorrentPool {
                 const addedReplacement = this.client.add(torrentId, {
                   store: SharedPieceStore,
                   storeCacheSlots: 0,
-                  storeOpts: { memoryBytes: this.#memoryBytes },
+                  storeOpts: { memoryBytes: this.#memoryBytes, ...this.#storeExtras },
                   deselect: true
                 }, (replacement) => {
                   this.torrents.set(key, replacement);
@@ -1533,14 +1685,19 @@ export class TorrentPool {
       const added = this.client.add(torrentId, {
         store: SharedPieceStore,
         storeCacheSlots: 0,
-        storeOpts: { memoryBytes: this.#memoryBytes },
+        storeOpts: { memoryBytes: this.#memoryBytes, ...this.#storeExtras },
         // Nothing is fetched until somebody says they want it. WebTorrent's own
         // default is `this.select(0, this.pieces.length - 1)` — the whole
         // torrent — and this proxy used to undo that afterwards by deselecting
         // the files nobody had opened. On a season pack that meant every
         // episode was being fetched for as long as the viewer took to choose
         // one. The download set is built up from stated needs instead.
-        deselect: true
+        deselect: true,
+        // Nothing to check when every file of this source is already here
+        // whole: what would be verified was written out of pieces this client
+        // had hashed, and the file's size was checked against what the torrent
+        // says. See `#wholeSources`.
+        skipVerify: this.#wholeSources.has(key)
       }, (readyTorrent) => {
         this.client.off("error", onError);
         this.torrents.set(key, readyTorrent);
@@ -1591,6 +1748,15 @@ export class TorrentPool {
     // it recently accessed so LRU eviction keeps it.
     this.#cancelIdleRemoval(torrent);
     this.#lastAccess.set(torrent, Date.now());
+    // And rejoin its swarm HERE rather than at the next five-second pass: a
+    // reader that has just arrived is about to ask for bytes, and a torrent
+    // still paused answers by fetching nothing at all.
+    if (torrent.paused === true) {
+      torrent.resume();
+      logger.info(
+        `torrent-pool: [${String(torrent.infoHash ?? "?").slice(0, 8)}] rejoined the swarm — a reader arrived`
+      );
+    }
     usage.set(fileIndex, (usage.get(fileIndex) ?? 0) + 1);
 
     let released = false;
@@ -1627,18 +1793,18 @@ export class TorrentPool {
     }
     this.#cancelIdleRemoval(torrent);
     const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
-    const ih = String(torrent.infoHash ?? "?").slice(0, 8);
-    logger.info(`torrent-pool: scheduling idle removal for "${name}" [${ih}] in ${TORRENT_IDLE_TTL_MS / 1000}s`);
+    const infoHashShort = String(torrent.infoHash ?? "?").slice(0, 8);
+    logger.info(`torrent-pool: scheduling idle removal for "${name}" [${infoHashShort}] in ${TORRENT_IDLE_TTL_MS / 1000}s`);
     const timer = setTimeout(() => {
       this.#idleTimers.delete(torrent);
       // Re-check: a new acquire since scheduling would have cancelled this
       // timer, but guard anyway against a race.
       const usage = this.fileUsageByTorrent.get(torrent);
       if (usage && usage.size > 0) {
-        logger.info(`torrent-pool: idle timer fired for "${name}" [${ih}] but refcount ${usage.size} >0 — keep`);
+        logger.info(`torrent-pool: idle timer fired for "${name}" [${infoHashShort}] but refcount ${usage.size} >0 — keep`);
         return;
       }
-      logger.warn(`torrent-pool: idle TTL fired for "${name}" [${ih}] — removing torrent (reason=idle-ttl caller=scheduleIdleRemoval)`);
+      logger.warn(`torrent-pool: idle TTL fired for "${name}" [${infoHashShort}] — removing torrent (reason=idle-ttl caller=scheduleIdleRemoval)`);
       this.#removeTorrent(torrent, "idle-ttl");
     }, TORRENT_IDLE_TTL_MS);
     timer.unref?.();
@@ -1667,6 +1833,59 @@ export class TorrentPool {
    * @param {string} [reason="unknown"] - Why removal was requested (idle-ttl, disk-cap, evict, api).
    * @returns {void}
    */
+  /**
+   * Forget a torrent that died on its own, keeping its bytes.
+   *
+   * NOT `#removeTorrent`: that one destroys the store as well, and the data on
+   * disk is the expensive thing here — re-downloading a film costs the swarm
+   * and the viewer, while re-adding a torrent costs a tracker announce. What
+   * has to go is only the RECORD, which otherwise answers every later request
+   * with a corpse.
+   *
+   * @param {import("webtorrent").Torrent} torrent
+   * @returns {void}
+   */
+  #forgetDeadTorrent(torrent) {
+    if (!torrent) {
+      return;
+    }
+    let forgotten = false;
+    for (const [key, value] of this.torrents) {
+      if (value === torrent) {
+        this.torrents.delete(key);
+        forgotten = true;
+        break;
+      }
+    }
+    if (!forgotten) {
+      return;
+    }
+    forgetTorrent(torrent);
+    this.fileUsageByTorrent.delete(torrent);
+    this.#lastAccess.delete(torrent);
+    this.#readPositionByTorrent.delete(torrent);
+    const name = typeof torrent.name === "string" ? torrent.name : "(unknown)";
+    logger.warn(
+      `torrent-pool: forgot "${name}" [${String(torrent.infoHash ?? "?").slice(0, 8)}] — ` +
+      "it died without being asked to; its data stays on disk and the next request adds it again"
+    );
+  }
+
+  /**
+   * Remove a torrent and its store, by name of the reason.
+   *
+   * The public form of what the idle timer and the disk cap already do. Taking
+   * the store with it is the point where a torrent downloaded whole is
+   * concerned: its pieces are a duplicate of the files kept elsewhere.
+   *
+   * @param {import("webtorrent").Torrent} torrent
+   * @param {string} reason
+   * @returns {void}
+   */
+  remove(torrent, reason) {
+    this.#removeTorrent(torrent, reason);
+  }
+
   #removeTorrent(torrent, reason = "unknown") {
     // Everything anybody stated for this torrent goes with it.
     forgetTorrent(torrent);
@@ -1754,6 +1973,7 @@ export class TorrentPool {
       downloadSpeed,
       uploadSpeed,
       connectedPeers: reach.connectedPeers,
+      deliveringPeers: reach.deliveringPeers,
       knownPeers: reach.knownPeers,
       queuedPeers: reach.queuedPeers,
       trackerSeeders: announce.seeders,

package/services/torrent-worker/client.js

@@ -94,6 +94,18 @@ export class TorrentWorkerClient {
   /** Reads consuming fragments in place, keyed by request id. */
   #fragmentReaders = new Map();
 
+  /**
+   * Files downloaded whole, by `${infoHash}/${fileIndex}`.
+   *
+   * Held on THIS thread because it is this thread that serves them: a file that
+   * is whole is read with an ordinary file read, and asking the torrent thread
+   * for anything about it would put back the very hop this removes. The torrent
+   * thread writes the files and says so; this is what it says.
+   *
+   * @type {Map<string, { path: string, length: number, name: string }>}
+   */
+  wholeFiles = new Map();
+
   /** @type {(event: { sourceKey: string, fileIndex: number, trackIndex: number, cues: object[], language: string }) => void} */
   #onSubtitleCues;
 
@@ -208,6 +220,15 @@ export class TorrentWorkerClient {
         case Event.LOG:
           logger.info(`torrent-worker: ${message.message}`);
           break;
+        case Event.FILE_COMPLETE:
+          // Recorded here so the stream route can answer from the file without
+          // going near the torrent thread at all.
+          this.wholeFiles.set(`${message.infoHash}/${message.fileIndex}`, {
+            path: message.path,
+            length: message.length,
+            name: message.name
+          });
+          break;
         case Event.SUBTITLE_CUES_READY:
           this.#onSubtitleCues({
             sourceKey: message.sourceKey,

package/services/torrent-worker/protocol.js

@@ -164,7 +164,15 @@ export const Event = {
    * {@link Command.SUBTITLE_CUES} call. Lets the main thread PUSH them to
    * whichever browser is watching instead of waiting to be asked.
    */
-  SUBTITLE_CUES_READY: "subtitle-cues-ready"
+  SUBTITLE_CUES_READY: "subtitle-cues-ready",
+  /**
+   * A file has been downloaded whole and written out as a file.
+   *
+   * The main thread serves it from disk after this, without asking this thread
+   * for anything: an ordinary read of an ordinary file, with no piece store
+   * between them and nothing that can refuse it for want of memory.
+   */
+  FILE_COMPLETE: "file-complete"
 };
 
 /**