@torrent-tv/proxy 2.86.0 → 2.87.0

package/services/storage/wire.js

@@ -0,0 +1,175 @@
+/**
+ * @file Who takes what of this machine, and how each of them is told its share.
+ *
+ * Kept apart from the owner because the owner knows nothing about this proxy —
+ * it reads what is free, divides it and hands out shares — and kept out of the
+ * session manager because a list of claimants is not a fact about a session.
+ *
+ * A CLAIMANT IS WHOEVER HOLDS BYTES, and every one of them is here. Four do:
+ * the torrent's pieces in memory, the pieces spilled from there to disk, the
+ * segments an encoder produced, the files downloaded whole — and the evidence,
+ * which is a claimant with a rule of its own (`Diagnostics.js`). Three of the
+ * five live on the torrent thread, so they arrive as pairs of closures over its
+ * channel rather than as the objects themselves.
+ *
+ * **Which RESOURCE each takes is read, not assumed.** Memory is one. Disk is
+ * one per device, because two filesystems cannot pay for each other: measured
+ * on the addon host 2026-09-05, `/tmp` — the segments and the spill — is the
+ * overlay, and `/data` — the evidence — is ext4 on the nvme. One figure divided
+ * between claimants on both gave each a share of a disk it does not write to.
+ *
+ * What is NOT a claimant is anything that merely counts somebody else's bytes.
+ * The torrent pool held a "disk cap" of 10 GB over `torrentDownloadedBytes` — a
+ * sum over WebTorrent's bitfield, so a piece held purely in MEMORY told against
+ * a ceiling called disk, while the bytes it was meant to bound belong to the
+ * spill and to the whole files, which have owners of their own. It owned
+ * nothing and is gone.
+ */
+
+import { deviceOf } from "./free.js";
+import { MachineBudget } from "./MachineBudget.js";
+import { availableMemoryBytes } from "./machine-memory.js";
+
+/**
+ * Build the owner of this machine's room and register everything that takes any.
+ *
+ * @param {object} params
+ * @param {{ root: string, stats: () => { bytes: number } }} params.segmentStore
+ * @param {{ held: () => number, allow: (bytes: number) => unknown }} [params.spill] -
+ *   The pieces the memory store spills. They live on the torrent thread, so
+ *   this is a pair of closures over the channel rather than the pool itself.
+ * @param {{ held: () => number, wanted: () => number, allow: (bytes: number) => unknown }} [params.memory] -
+ *   The pieces held IN memory, on the torrent thread as well. Told their share
+ *   by the same owner that divides the disk, because the two trade: what does
+ *   not fit in memory is spilled, so this share decides how much disk is needed.
+ * @param {{ held: () => number, allow: (bytes: number) => unknown, root?: string }} [params.wholeFiles] -
+ *   Files downloaded whole and kept as files, on the torrent thread as well.
+ * @param {import("./Diagnostics.js").Diagnostics} [params.diagnostics] - The
+ *   evidence. Built where the collectors are, so one object is both registered
+ *   here and consulted by them.
+ * @param {string} [params.diagnosticsRoot] - Where the evidence is written,
+ *   which on the addon host is a different device from everything else.
+ * @param {(directory: string) => Promise<number | null>} params.readFree
+ * @param {import("./MachineBudget.js").BudgetPolicy} [params.policy] - What the
+ *   operator has said this proxy may take. Absent, the measured default.
+ * @param {{ info: Function, warn?: Function }} [params.logger]
+ * @returns {{ revise: () => Promise<unknown>, segmentBytes: () => number, diagnostics: object | null, describe: () => string }}
+ *   What the segments may hold is asked for rather than pushed: zero until the
+ *   first revision, and zero stops growth rather than licensing it.
+ */
+export function wireMachineBudget({
+  segmentStore,
+  spill,
+  memory,
+  wholeFiles,
+  diagnostics,
+  diagnosticsRoot = "",
+  readFree,
+  policy,
+  logger
+}) {
+  // THE OPERATOR NAMES ONE FIGURE FOR DISK, and disk is one resource per
+  // device — so the floor they named is given to every disk resource as it is
+  // defined, rather than asking them to name a device they have never heard of.
+  const floors = { ...(policy?.floors ?? {}) };
+  const diskFloorBytes = Math.max(0, Number(policy?.diskFloorBytes) || 0);
+  const effective = { ...(policy ?? {}), floors };
+  const budget = new MachineBudget({ policy: effective, logger });
+  /**
+   * @param {string} name
+   * @param {string} directory
+   */
+  const defineDisk = (name, directory) => {
+    if (diskFloorBytes > 0 && !(name in floors)) {
+      floors[name] = diskFloorBytes;
+    }
+    budget.defineResource({ name, readFree: () => readFree(directory) });
+  };
+
+  budget.defineResource({ name: "memory", readFree: () => availableMemoryBytes() });
+
+  const segmentsOn = deviceOf(segmentStore.root);
+  defineDisk(segmentsOn, segmentStore.root);
+
+  let segmentBytes = 0;
+  budget.register({
+    name: "segments",
+    resource: segmentsOn,
+    held: () => segmentStore.stats().bytes,
+    // The whole of every film anybody is watching. There is no smaller honest
+    // answer, so it asks for everything and is cut in proportion like the rest.
+    wanted: () => Number.MAX_SAFE_INTEGER,
+    allow: (bytes) => {
+      segmentBytes = bytes;
+    }
+  });
+
+  if (typeof memory?.allow === "function") {
+    budget.register({
+      name: "pieces in memory",
+      resource: "memory",
+      held: () => memory.held?.() ?? 0,
+      wanted: () => memory.wanted?.() ?? 0,
+      allow: (bytes) => {
+        void memory.allow(bytes);
+      }
+    });
+  }
+
+  if (typeof spill?.allow === "function") {
+    // The share travels the channel that already carries everything else, and
+    // the reply says what they hold — one exchange, both directions.
+    budget.register({
+      name: "spilled pieces",
+      resource: segmentsOn,
+      held: () => spill.held?.() ?? 0,
+      wanted: () => Number.MAX_SAFE_INTEGER,
+      allow: (bytes) => {
+        void spill.allow(bytes);
+      }
+    });
+  }
+
+  if (typeof wholeFiles?.allow === "function") {
+    const filesOn = wholeFiles.root ? deviceOf(wholeFiles.root) : segmentsOn;
+    defineDisk(filesOn, wholeFiles.root || segmentStore.root);
+    budget.register({
+      name: "whole files",
+      resource: filesOn,
+      held: () => wholeFiles.held?.() ?? 0,
+      // A whole file is worth keeping while anybody may read it again, and
+      // there is no smaller honest answer than "all of them" — like the rest,
+      // it is cut in proportion.
+      wanted: () => Number.MAX_SAFE_INTEGER,
+      allow: (bytes) => {
+        void wholeFiles.allow(bytes);
+      }
+    });
+  }
+
+  if (diagnostics) {
+    const evidenceOn = diagnosticsRoot ? deviceOf(diagnosticsRoot) : segmentsOn;
+    defineDisk(evidenceOn, diagnosticsRoot || segmentStore.root);
+    budget.register({
+      name: "diagnostics",
+      resource: evidenceOn,
+      held: () => diagnostics.held(),
+      // What it holds plus one more of the largest kind seen — measured, and
+      // nothing at all before a fault has happened.
+      wanted: () => diagnostics.wanted(),
+      allow: (bytes) => diagnostics.allow(bytes)
+    });
+  }
+
+  return {
+    revise: async () => {
+      // Re-read the evidence before dividing: it is the one claimant whose
+      // bytes are written by something that never reports them.
+      await diagnostics?.measure();
+      return budget.revise();
+    },
+    segmentBytes: () => segmentBytes,
+    diagnostics: diagnostics ?? null,
+    describe: () => budget.describe()
+  };
+}

package/services/{download → torrent/download}/SwarmSelection.js

@@ -29,7 +29,7 @@ import {
   Urgency,
   urgencyName
 } from "../demand/index.js";
-import { findSharedStore } from "../piece-store/shared-piece-store.js";
+import { pieceStoreOf } from "../piece-store-of.js";
 
 export class SwarmSelection {
   #torrent;
@@ -50,7 +50,7 @@ export class SwarmSelection {
    *   piece store is reached. Injectable so a test can drive the memory
    *   projection without constructing a real store.
    */
-  constructor({ torrent, register, findStore = findSharedStore }) {
+  constructor({ torrent, register, findStore = pieceStoreOf }) {
     this.#torrent = torrent;
     this.#register = register;
     this.#findStore = findStore;

package/services/{download → torrent/download}/registry.js

@@ -2,11 +2,11 @@
  * @file One demand register and one swarm selection per torrent, found from the
  * torrent itself.
  *
- * The same shape the piece store already uses — `findSharedStore(torrent)`
+ * The same shape the piece store already uses — `pieceStoreOf(torrent)`
  * reaches the store without it being threaded through every call — and for the
  * same reason: the reader, the pool and the background fill all need the same
  * instance, and passing it through six layers of arguments would make the
- * plumbing bigger than the thing.
+ * argument lists bigger than the thing they carry.
  *
  * Kept in a live set as well as a weak map, because one question cannot be
  * answered per torrent: whether ANYTHING anywhere is still missing something

package/services/torrent/files.js

@@ -6,7 +6,7 @@
  * picture — `Rus Sound/<name>.mka`, `Sub/[group]/<name>.ass` — and a viewer who
  * cannot reach them is watching the release without half of what it carries.
  * This module answers only which file goes with which; what is inside a file is
- * the container layer's business (`services/container/`), and what a viewer is
+ * the container layer's business (`services/media/container/`), and what a viewer is
  * shown is composed in the browser, where the locale is known.
  *
  * **Why it lives in `torrent/`.** A container knows only itself, and a track

package/services/torrent/piece-store-of.js

@@ -0,0 +1,29 @@
+/**
+ * @file The piece store behind a torrent, reached through what it does.
+ *
+ * WebTorrent wraps the store it was given in stores of its own, each holding
+ * the next one as `store`, so the store the torrent component talks to is found
+ * by walking that chain. It is recognised by the interface this component uses
+ * — which pieces the readers have asked to keep (`protectedRanges`) — and not by
+ * its class: the torrent component does not import the storage component's
+ * implementation, it is handed it where the thread is assembled
+ * (`worker/worker.js`).
+ */
+
+/** How deep WebTorrent's own wrappers are allowed to go before the walk gives up. */
+const WRAPPER_DEPTH = 8;
+
+/**
+ * @param {{ store?: object } | null | undefined} torrent
+ * @returns {object | null}
+ */
+export function pieceStoreOf(torrent) {
+  let candidate = torrent?.store;
+  for (let depth = 0; candidate && depth < WRAPPER_DEPTH; depth += 1) {
+    if (typeof candidate.protectedRanges === "function") {
+      return candidate;
+    }
+    candidate = candidate.store;
+  }
+  return null;
+}

package/services/{torrent-pool.js → torrent/torrent-pool.js}

@@ -2,24 +2,24 @@
  * @file WebTorrent client pool.
  *
  * Manages a shared WebTorrent client instance and a map of active torrents
- * keyed by their own infohash (see `torrent-source-key.js`), so a magnet and
+ * keyed by their own infohash (see `torrent/torrent-source-key.js`), so a magnet and
  * a `.torrent` file for the same content share one entry. Tracks file-level
  * usage so that only the pieces needed by active streams are selected for
  * download.
  */
 
-import { IDLE_KEEP_MS } from "./disk/keep.js";
+import { IDLE_KEEP_MS } from "../storage/keep.js";
 import dns from "node:dns/promises";
 import os from "node:os";
 import path from "node:path";
-import { rmSync, statfsSync } from "node:fs";
+import { rmSync } from "node:fs";
 import WebTorrent from "webtorrent";
-import { logger } from "../utils/logger.js";
-import { SharedPieceStore, findSharedStore } from "./piece-store/shared-piece-store.js";
+import { logger } from "../../utils/logger.js";
+import { pieceStoreOf } from "./piece-store-of.js";
 import { Urgency, urgencyName } from "./demand/index.js";
 import { demandFor, forgetTorrent, reconcileAll, hasUnmetDemand } from "./download/registry.js";
 import { withdrawClaim } from "./download/withdraw-claim.js";
-import { isAtAWatchingViewer, isBehindEverybody, isNobodyComingNow } from "./priority/PriorityMap.js";
+import { isAtAWatchingViewer, isBehindEverybody, isNobodyComingNow } from "../viewer/PriorityMap.js";
 import { deriveSourceKey } from "./torrent-source-key.js";
 
 /** How a window stated from the priority map names itself. */
@@ -89,15 +89,6 @@ const PRIORITY_WINDOW_BYTES = 16 * 1024 * 1024;
 const HEADER_HEAD_BYTES = 256 * 1024;
 const HEADER_TAIL_BYTES = 2 * 1024 * 1024;
 
-// Global disk cap. Downloaded torrent data is removed on idle TTL and at
-// shutdown, but under pressure (several large files within the TTL window)
-// it can still fill a small HA host's disk (SD/eMMC), which can take down
-// Home Assistant itself. When the total exceeds the cap, whole torrents with
-// no active reader are evicted least-recently-used first. Active torrents are
-// never evicted (we cannot delete what is playing).
-const DISK_CAP_ABSOLUTE_MAX_BYTES = 10 * 1024 * 1024 * 1024; // 10 GB
-const DISK_CAP_SWEEP_INTERVAL_MS = 30_000;
-
 // Adaptive upload. Seeding to the BitTorrent swarm does not help our viewer (we
 // deliver over our own WebRTC/HTTPS channel) — it is pure uplink cost and the
 // riskiest legal act (active distribution). So the default is minimal: a
@@ -528,23 +519,6 @@ function formatWarning(warning) {
   return stack || warning.message;
 }
 
-/**
- * @param {string} storePath
- * @returns {number}
- */
-function computeDefaultDiskCap(storePath) {
-  try {
-    const stat = statfsSync(storePath);
-    const freeBytes = stat.bavail * stat.bsize;
-    if (Number.isFinite(freeBytes) && freeBytes > 0) {
-      return Math.min(DISK_CAP_ABSOLUTE_MAX_BYTES, Math.floor(freeBytes / 2));
-    }
-  } catch {
-    // statfs unavailable (old Node / odd FS) — fall back to the fixed max.
-  }
-  return DISK_CAP_ABSOLUTE_MAX_BYTES;
-}
-
 /**
  * Decode a raw torrent source value into the format expected by WebTorrent.
  *
@@ -1009,15 +983,9 @@ export class TorrentPool {
    */
   #edgePrefetches = new Map();
 
-  /** Global disk cap in bytes (0 = disabled). */
-  #maxDiskBytes = 0;
-
   /** Memory budget per torrent for resident pieces; undefined = store default. */
   #memoryBytes;
 
-  /** Periodic disk-cap enforcement timer. */
-  #diskSweepTimer = null;
-
   /** Current client-wide upload limit in bytes/sec (adaptive). -1 = not yet set. */
   #uploadLimit = -1;
 
@@ -1047,12 +1015,6 @@ export class TorrentPool {
    */
   #swarmTimingByTorrent = new WeakMap();
 
-  /**
-   * @param {{ maxDiskBytes?: number }} [options]
-   *   `maxDiskBytes` caps total downloaded torrent data; when omitted a
-   *   default is computed from free disk (min(10 GB, half free)). Pass 0 to
-   *   disable the cap.
-   */
   /**
    * Sources every file of which this proxy holds whole.
    *
@@ -1168,11 +1130,26 @@ export class TorrentPool {
     }
   }
 
+  /** The chunk store class every torrent is built with. @type {Function} */
+  #pieceStore;
+
   get claimsWithdrawn() {
     return this.#claimsWithdrawn;
   }
 
-  constructor({ maxDiskBytes, memoryBytes, dhtBootstrap } = {}) {
+  /**
+   * @param {object} params
+   * @param {number} [params.memoryBytes]
+   * @param {string[]} [params.dhtBootstrap]
+   * @param {Function} params.pieceStore - The chunk store WebTorrent builds per
+   *   torrent: the storage component's implementation, handed in where the
+   *   thread is assembled.
+   */
+  constructor({ memoryBytes, dhtBootstrap, pieceStore } = {}) {
+    if (typeof pieceStore !== "function") {
+      throw new TypeError("TorrentPool needs the piece store to build torrents with");
+    }
+    this.#pieceStore = pieceStore;
     this.#memoryBytes = Number.isFinite(memoryBytes) && memoryBytes > 0 ? memoryBytes : undefined;
 
     // Sweep orphaned torrent data left by a previous hard kill (no graceful
@@ -1235,16 +1212,6 @@ export class TorrentPool {
       logger.warn(`torrent-pool: client warning: ${formatWarning(warning)}`);
     });
 
-    this.#maxDiskBytes = Number.isFinite(maxDiskBytes) && maxDiskBytes >= 0
-      ? maxDiskBytes
-      : computeDefaultDiskCap(os.tmpdir());
-    if (this.#maxDiskBytes > 0) {
-      const gigabytes = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
-      logger.info(`torrent-pool: disk cap ${gigabytes} GB (LRU eviction of idle torrents above it)`);
-      this.#diskSweepTimer = setInterval(() => this.#enforceDiskCap(), DISK_CAP_SWEEP_INTERVAL_MS);
-      this.#diskSweepTimer.unref?.();
-    }
-
     // Adaptive upload: start with seeding OFF (nothing is being watched yet),
     // then let the periodic adjuster raise it to the floor while a reader is
     // active and to the boost when download is choke-starved. WebTorrent's
@@ -1321,7 +1288,7 @@ export class TorrentPool {
    * @returns {{ ranges: Array<{ from: number, to: number }>, missing: boolean }}
    */
   #readerDemand(torrent) {
-    const store = findSharedStore(torrent);
+    const store = pieceStoreOf(torrent);
     const ranges = typeof store?.protectedRanges === "function" ? store.protectedRanges() : [];
     let missing = false;
     for (const range of ranges) {
@@ -1610,61 +1577,6 @@ export class TorrentPool {
     logger.info(`torrent-pool: upload limit -> ${Math.round(bytesPerSec / 1024)} KB/s (${reason})`);
   }
 
-  /**
-   * Sum of downloaded bytes across pooled torrents — a cheap proxy for the
-   * on-disk footprint (the FS store writes downloaded pieces).
-   *
-   * @returns {number}
-   */
-  #currentDiskBytes() {
-    let total = 0;
-    for (const torrent of this.torrents.values()) {
-      total += Math.max(0, torrentDownloadedBytes(torrent));
-    }
-    return total;
-  }
-
-  /**
-   * Evict whole torrents, least-recently-used first, while the total on-disk
-   * footprint exceeds the cap. Only torrents with NO active file reader are
-   * evictable — a playing torrent cannot be deleted. Best-effort.
-   *
-   * @returns {void}
-   */
-  #enforceDiskCap() {
-    if (this.#maxDiskBytes <= 0) {
-      return;
-    }
-    let used = this.#currentDiskBytes();
-    if (used <= this.#maxDiskBytes) {
-      return;
-    }
-    // Candidates: pooled torrents nothing is wanted of, the longest unwanted
-    // first.
-    const candidates = [...this.torrents.values()]
-      .filter((torrent) => !isWanted(torrent))
-      .sort(
-        (earlier, later) =>
-          (this.#lastAccess.get(earlier) ?? 0) - (this.#lastAccess.get(later) ?? 0)
-      );
-
-    for (const torrent of candidates) {
-      if (used <= this.#maxDiskBytes) {
-        break;
-      }
-      const freed = Math.max(0, torrentDownloadedBytes(torrent));
-      const name = typeof torrent?.name === "string" ? torrent.name : "(unknown)";
-      const gigabytes = (this.#maxDiskBytes / (1024 * 1024 * 1024)).toFixed(1);
-      logger.info(
-        `torrent-pool: disk cap ${gigabytes} GB exceeded — evicting idle torrent "${name}" ` +
-          `(~${(freed / (1024 * 1024)).toFixed(0)} MB)`
-      );
-      this.#cancelIdleRemoval(torrent);
-      this.#removeTorrent(torrent, "disk-cap");
-      used -= freed;
-    }
-  }
-
   /**
    * Attach peer-discovery diagnostics to a freshly added torrent: tracker
    * announce results (seeders/leechers per announce) and torrent-level
@@ -1914,7 +1826,7 @@ export class TorrentPool {
               this.#readPositionByTorrent.delete(existing);
               this.client.remove(existing, { destroyStore: true }, () => {
                 const addedReplacement = this.client.add(torrentId, {
-                  store: SharedPieceStore,
+                  store: this.#pieceStore,
                   storeCacheSlots: 0,
                   storeOpts: this.#storeOptions(),
                   deselect: true
@@ -1947,7 +1859,7 @@ export class TorrentPool {
       // hands out, holds pieces in shared memory the main thread can read
       // directly, and spills to disk instead of losing them.
       const added = this.client.add(torrentId, {
-        store: SharedPieceStore,
+        store: this.#pieceStore,
         storeCacheSlots: 0,
         storeOpts: this.#storeOptions(),
         // Nothing is fetched until somebody says they want it. WebTorrent's own
@@ -2561,7 +2473,7 @@ export class TorrentPool {
    * whole-file selection, a window around the read head, and the reader itself
    * — and they overwrote each other on every request. Every need is now stated
    * in one register (`services/demand/`) and one class turns the register into
-   * requests to the swarm (`services/download/SwarmSelection.js`); several
+   * requests to the swarm (`services/torrent/download/SwarmSelection.js`); several
    * readers on one file therefore produce the union of their windows instead of
    * the last caller's opinion.
    *
@@ -2667,11 +2579,6 @@ export class TorrentPool {
    * @returns {Promise<void>}
    */
   async destroyAll() {
-    // Stop periodic disk-cap enforcement.
-    if (this.#diskSweepTimer) {
-      clearInterval(this.#diskSweepTimer);
-      this.#diskSweepTimer = null;
-    }
     // Stop periodic adaptive-upload adjustment.
     if (this.#uploadAdjustTimer) {
       clearInterval(this.#uploadAdjustTimer);

package/services/{torrent-worker → torrent/worker}/background-fill.js

@@ -19,7 +19,7 @@
  * this must not be asking the swarm for anything.
  *
  * WHY IT IS A READ AND NOT A SELECTION. `file.select()` claims every piece of a
- * file at once. `#syncSelections` in `torrent-pool.js` records what that cost
+ * file at once. `#syncSelections` in `torrent/torrent-pool.js` records what that cost
  * when it was done alongside the readers' own moving windows: a claim covering
  * everything always outranked the window, and a seek to 89.1 % of a 4.7 GB film
  * waited 93 s while the swarm fetched 2.47 GB in file order. So this walks the
@@ -27,7 +27,7 @@
  * is reading and gives it back.
  */
 
-import { logger } from "../../utils/logger.js";
+import { logger } from "../../../utils/logger.js";
 import { readersAreBlockedOn, stallsSeenOn } from "./piece-reader.js";
 
 /**