@torrent-tv/proxy 2.80.19 → 2.81.1

package/services/hls-session-manager.js

@@ -8,7 +8,7 @@
  */
 
 import { createReadStream, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
-import { access, readdir, readFile, rm, stat, unlink } from "node:fs/promises";
+import { access, readFile, rm, stat, unlink } from "node:fs/promises";
 import { Readable } from "node:stream";
 import os from "node:os";
 import path from "node:path";
@@ -88,11 +88,13 @@ import {
 // — did not move when the code did.
 export { ffmpegSeconds, onKeyframeGridFor, seekLandingOffsetFor, segmentCutTimesFrom };
 import { viewersOf } from "./viewer/Viewer.js";
+import { viewerSegmentsOn } from "./viewer/positions.js";
 import { Viewers } from "./viewer/Viewers.js";
 import { LiveOutputs } from "./output/LiveOutputs.js";
 import { variantHeightsFor } from "./output/ladder.js";
 import { EncodeOrchestrator } from "./orchestrators/EncodeOrchestrator.js";
 import { readDiskFree } from "./memory-report.js";
+import { wireDiskSpace } from "./disk/wire.js";
 
 /**
  * Whether an encoder run died because its INPUT went away, rather than because
@@ -473,18 +475,6 @@ const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
  * there for the life of the process.
  */
 const SEGMENT_STORE_IDLE_MS = 6 * 60 * 60 * 1000;
-/**
- * The share of FREE disk the produced segments may take.
- *
- * A share of what is free NOW, re-read on every sweep, for the same reason the
- * piece store re-derives its memory allowance every minute: a machine that
- * fills up after this proxy started would otherwise go on spending an allowance
- * taken when it was empty. A Home Assistant install often runs from a 32 GB
- * card carrying everything else in the house.
- */
-const SEGMENT_STORE_FREE_SHARE = 0.25;
-/** What the store may hold where the free space cannot be read at all. */
-const SEGMENT_STORE_FALLBACK_BYTES = 2 * 1024 * 1024 * 1024;
 const DEFAULT_STARTUP_WAIT_MS = 5_000;
 // Realtime budget — runtime downswitch (software encoder only). Periodically
 // check each active software-transcode session's ffmpeg `speed`; when it stays
@@ -1604,6 +1594,13 @@ export class HlsSessionManager {
       void this.cleanupExpired();
     }, CLEANUP_INTERVAL_MS);
     this.cleanupTimer.unref();
+    // One owner of the disk, and the list of what takes it lives with the owner.
+    this.diskSpace = wireDiskSpace({
+      segmentStore: this.segmentStore,
+      torrentPool: this.torrentPool,
+      readFree: readDiskFree,
+      logger
+    });
     // Realtime-budget monitor: only meaningful for the software encoder with a
     // benchmark (the only path that can pick/step resolution). Cheap no-op scan
     // otherwise.
@@ -5530,6 +5527,9 @@ export class HlsSessionManager {
       inputUnavailable: (message) => isInputUnavailable(message),
       onProgress: (report) => this.#noteRunProgress(session, run, report),
       indexOfName: (name) => session.segmentFormat.segmentIndexFromName(name),
+      // Why this encoder exists, recorded with its argument list. It used to be
+      // handed to a separate `start` call; there is no separate call now.
+      because,
       onClosed: (name) => this.segmentStore.publish(session.outputKey ?? "", name, session.segmentFormat),
       onEnded: (ended) => this.noteRunEnded(session, run, ended)
     });
@@ -5543,8 +5543,6 @@ export class HlsSessionManager {
     // premature downscale.
     session.budgetSlowSince = 0;
 
-    run.start(because);
-
     logger.info(
       `transcode ${session.id} encode-run #${safeIndex}..#${runEnd} from segment #${safeIndex} ` +
       `(+${Date.now() - restartEnteredAt}ms since the restart was asked for) ` +
@@ -9667,32 +9665,22 @@ export class HlsSessionManager {
     // The segments outlive every session on them, so what they cost is decided
     // here rather than by anybody's departure: how long ago each output was
     // last read, and how much room the disk has for the lot.
+    // The room is the disk owner's to divide; this asks what the share is now.
+    await this.diskSpace.revise();
     this.segmentStore.enforce({
       idleMs: SEGMENT_STORE_IDLE_MS,
-      maxBytes: await this.#segmentStoreAllowance()
+      maxBytes: this.diskSpace.segmentBytes(),
+      viewersAt: (key) =>
+        viewerSegmentsOn({
+          sessions: this.sessionsById.values(),
+          outputKey: key,
+          segmentAt: (session, seconds) => this.#segmentIndexForTime(session, seconds),
+          now: Date.now(),
+          staleAfterMs: this.presenceStaleAfterMs()
+        })
     });
   }
 
-  /**
-   * How much disk the produced segments may hold.
-   *
-   * A share of what is FREE now rather than a figure fixed at startup, for the
-   * same reason the piece store's memory allowance is re-derived every minute:
-   * a machine that fills up after this proxy started would otherwise go on
-   * spending an allowance taken when it was empty. On a Home Assistant install
-   * that disk is often a 32 GB card carrying everything else the household
-   * runs.
-   *
-   * @returns {Promise<number>}
-   */
-  async #segmentStoreAllowance() {
-    const free = await readDiskFree(this.segmentStore.root);
-    if (!Number.isFinite(free) || free <= 0) {
-      return SEGMENT_STORE_FALLBACK_BYTES;
-    }
-    return Math.max(SEGMENT_STORE_FALLBACK_BYTES, Math.floor(free * SEGMENT_STORE_FREE_SHARE));
-  }
-
   /**
    * Return a progress snapshot for the given session, or `null` if not found.
    * Also refreshes `lastAccessedAt` to prevent the session from expiring.
@@ -10078,14 +10066,7 @@ export class HlsSessionManager {
     for (const sessionId of activeIds) {
       await this.disposeSession(sessionId);
     }
-    const rootDir = path.join(os.tmpdir(), "torrent-tv-hls");
-    try {
-      const dirs = await readdir(rootDir);
-      if (dirs.length === 0) {
-        await rm(rootDir, { recursive: true, force: true });
-      }
-    } catch (_error) {
-      // Best effort cleanup.
-    }
+    // Everything this process owns, root included. See SegmentStore.dropAll.
+    this.segmentStore.dropAll("the proxy is shutting down");
   }
 }

package/services/orchestrators/EncodeOrchestrator.js

@@ -572,8 +572,11 @@ export class EncodeOrchestrator {
     // a run writing that number again writes under a working name until it
     // closes its own — so the file standing there is a complete piece made by
     // somebody, and serving it is right until the newer one replaces it whole.
+    // The claim comes after the run exists, and that is not a race: everything
+    // between is synchronous, and a process cannot say a word before the next
+    // tick. What it removes is the second act — building a run and starting it
+    // were two steps, and two owners each performed the second one.
     this.coverageOf(address).claim(run, from, endOfRun({ from, to }));
-    run.start(because);
   }
 
   /**

package/services/piece-store/allowance.js

@@ -0,0 +1,107 @@
+/**
+ * @file How much of a machine's resource the stores may hold between them.
+ *
+ * Written once and used twice, because memory and disk are the same question
+ * asked of two resources: what is free now, plus what we already hold, less
+ * what everything that is not us has recently been seen to need.
+ *
+ * It used to exist only for memory, inside the memory store, keyed on module
+ * state. Disk had no such rule at all — the spill file grew until the machine
+ * did, 14.4 GB in a single viewing on 2026-08-31 — and copying the memory rule
+ * across would have made two rules to keep in step. One rule, two readings.
+ */
+
+/**
+ * How many observations of other processes' demand are kept.
+ *
+ * A window rather than a high-water: a single spike would otherwise stand for
+ * the life of the process and hold the stores down long after whatever caused
+ * it had finished.
+ */
+const OTHER_DEMAND_SAMPLES = 60;
+
+/**
+ * What everything that is not us has recently been seen to need.
+ *
+ * One instance per resource. A fall in what is free that we did not cause is a
+ * measurement of somebody else's demand; a fall we did cause is our own doing
+ * and says nothing about the machine.
+ */
+export class OtherDemand {
+  /** @type {number[]} */
+  #falls = [];
+
+  #lastFreeBytes = 0;
+
+  #lastHeldBytes = 0;
+
+  /**
+   * Take a reading, and answer what to reserve for others.
+   *
+   * @param {number} freeBytes - What the machine says is free now.
+   * @param {number} heldBytes - What the stores hold of this resource now.
+   * @returns {number} The reserve, in bytes.
+   */
+  note(freeBytes, heldBytes) {
+    if (this.#lastFreeBytes > 0) {
+      const fell = this.#lastFreeBytes - freeBytes;
+      const ours = heldBytes - this.#lastHeldBytes;
+      this.#falls.push(Math.max(0, fell - ours));
+      if (this.#falls.length > OTHER_DEMAND_SAMPLES) {
+        this.#falls.shift();
+      }
+    }
+    this.#lastFreeBytes = freeBytes;
+    this.#lastHeldBytes = heldBytes;
+    return this.reserve();
+  }
+
+  /** @returns {number} */
+  reserve() {
+    return this.#falls.length === 0 ? 0 : Math.max(...this.#falls);
+  }
+
+  /** Forget the readings. For tests, which share a module. */
+  forget() {
+    this.#falls = [];
+    this.#lastFreeBytes = 0;
+    this.#lastHeldBytes = 0;
+  }
+}
+
+/**
+ * How much of the resource the stores may hold between them.
+ *
+ * What the machine reports free is what could be taken ON TOP of what is
+ * already held, so the stores' own bytes are added back: the pair is the
+ * ceiling the stores could reach.
+ *
+ * @param {number} freeBytes
+ * @param {number} heldBytes
+ * @param {number} reserveBytes
+ * @returns {number}
+ */
+export function machineAllowanceBytes(freeBytes, heldBytes, reserveBytes) {
+  return Math.max(0, Math.max(freeBytes, 0) + Math.max(heldBytes, 0) - Math.max(reserveBytes, 0));
+}
+
+/**
+ * Divide what the machine allows between the stores, by what each is asking
+ * for.
+ *
+ * When everyone's ask fits, everyone gets it and the machine's limit never
+ * binds. When the asks do not fit, each store is cut in proportion to what it
+ * asked, so a store wanting little is not cut to make room for one wanting
+ * much.
+ *
+ * @param {number[]} wantedBytes - What each store is asking for, in order.
+ * @param {number} allowanceBytes
+ * @returns {number[]} What each store may hold, in the same order.
+ */
+export function divideAllowance(wantedBytes, allowanceBytes) {
+  const total = wantedBytes.reduce((sum, want) => sum + Math.max(0, want), 0);
+  if (total <= allowanceBytes || total === 0) {
+    return wantedBytes.map((want) => Math.max(0, want));
+  }
+  return wantedBytes.map((want) => Math.floor(allowanceBytes * (Math.max(0, want) / total)));
+}

package/services/piece-store/piece-disk-store.js

@@ -0,0 +1,365 @@
+/**
+ * @file Where downloaded pieces live once memory cannot hold them.
+ *
+ * The second tier of the piece store, and — unlike what it replaces — an owner
+ * of what it takes: it knows how many bytes it holds, it is told how many it
+ * may hold, and when it is over that it gives disk back.
+ *
+ * WHAT IT REPLACED, because the difference is the whole of the design. `DiskTier`
+ * wrote every piece into ONE sparse file at `index * chunkLength` and answered
+ * `forget(index)` by dropping the number from a set. The bytes stayed: a sparse
+ * file's blocks are returned only by hole punching, which Node exposes no
+ * binding for, so nothing this process could do returned a single block before
+ * the whole file was removed. Measured 2026-08-31: a store holding 312-424 MB
+ * of pieces had written **14 400 MB** to that file in fifty minutes, and free
+ * space on the host fell by every megabyte of it until the session ended. On a
+ * Home Assistant install with a 32 GB card that is the card.
+ *
+ * A PIECE IS A FILE, and that is what makes the ceiling real. Removing a file
+ * returns exactly its blocks, needs no binding this runtime lacks, and makes
+ * the unit of eviction the same as the unit of storage — so the order pieces
+ * leave in is the order we choose rather than the order they happen to lie in.
+ * The read that the single file was chosen for is unaffected: it is still one
+ * `read` into a buffer the caller already owns, which is what the 22.08 ms →
+ * 7.63 ms measurement on the field host was about. What it adds is an `open`
+ * per read, tens of microseconds against those milliseconds.
+ *
+ * Nothing here decides what the allowance should be. It is told, because the
+ * disk is one and this store is not its only user — the segments an encoder
+ * produces are on it too — and a ceiling that one of two users sets for itself
+ * is not a ceiling.
+ */
+
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * How a removal is asked for.
+ *
+ * Windows keeps a deleted file's name reserved until the last handle on it is
+ * closed, and answers `rm` of the name — or `rmdir` of the directory holding it
+ * — with EPERM until then. These are Node's own documented options for exactly
+ * that, not a wait invented here; on POSIX they never come into play.
+ */
+const REMOVAL = { force: true, maxRetries: 10, retryDelay: 20 };
+
+/**
+ * One torrent's pieces on disk.
+ *
+ * Pieces are numbered by the torrent, so a file is named by its number. The
+ * directory is the store: nothing else writes into it, and destroying the store
+ * removes it whole.
+ */
+export class PieceDiskStore {
+  #directory;
+
+  #chunkLength;
+
+  /** Piece index → its length on disk. @type {Map<number, number>} */
+  #stored = new Map();
+
+  /** Piece index → when it was last written or read. @type {Map<number, number>} */
+  #touched = new Map();
+
+  /** Pieces being read right now, which eviction leaves alone. @type {Map<number, number>} */
+  #reading = new Map();
+
+  /** Removals still in flight, by piece. @type {Map<number, Promise<unknown>>} */
+  #removing = new Map();
+
+  /** What this store may hold, or null while nobody has said. @type {number | null} */
+  #allowanceBytes = null;
+
+  #evictions = 0;
+
+  #bytes = 0;
+
+  #now;
+
+  /**
+   * @param {object} params
+   * @param {string} params.directory - Where this torrent's pieces live.
+   * @param {string} params.name - A name unique to the torrent; it becomes the
+   *   directory inside `directory`.
+   * @param {number} params.chunkLength - The torrent's piece length. Kept for
+   *   the caller's arithmetic; a piece's own length is recorded as it is written,
+   *   because the last piece of a torrent is shorter.
+   * @param {number | null} [params.allowanceBytes] - What it may hold. Null
+   *   means nobody has said yet, and nothing is evicted until somebody does.
+   * @param {() => number} [params.now]
+   */
+  constructor({ directory, name, chunkLength, allowanceBytes = null, now = Date.now }) {
+    this.#directory = path.join(directory, name);
+    this.#chunkLength = chunkLength;
+    this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
+    this.#now = now;
+  }
+
+  /** Where this store's pieces live, for logging and cleanup. */
+  get path() {
+    return this.#directory;
+  }
+
+  /** How many pieces are on disk. */
+  get size() {
+    return this.#stored.size;
+  }
+
+  /** What those pieces weigh. */
+  get bytes() {
+    return this.#bytes;
+  }
+
+  /** What it may hold, or null while nobody has said. */
+  get allowanceBytes() {
+    return this.#allowanceBytes;
+  }
+
+  /**
+   * Say what it may hold from now on.
+   *
+   * Lowering it does not free anything by itself: what is already written stays
+   * until the next write needs room. A store that is over its allowance and
+   * never written to again is holding disk nobody has asked for, which is the
+   * same bargain memory makes.
+   *
+   * @param {number | null} bytes
+   * @returns {number | null} What it may hold now.
+   */
+  reviseAllowance(bytes) {
+    this.#allowanceBytes = Number.isFinite(bytes) && bytes >= 0 ? bytes : null;
+    return this.#allowanceBytes;
+  }
+
+  /**
+   * @param {number} index
+   * @returns {boolean}
+   */
+  has(index) {
+    return this.#stored.has(index);
+  }
+
+  /**
+   * Write a piece out, making room for it first.
+   *
+   * @param {number} index
+   * @param {Uint8Array} bytes
+   * @returns {Promise<void>}
+   */
+  async write(index, bytes) {
+    // A piece thrown away and wanted again before its file has gone. Windows
+    // holds a deleted-but-still-open file in a pending state and answers the
+    // next `open` of that name with EPERM, so writing it again has to wait for
+    // the removal to finish. On POSIX the wait costs a resolved promise.
+    await this.#removing.get(index);
+    await this.#ensureDirectory();
+    await this.#makeRoomFor(bytes.length, index);
+    await fs.writeFile(this.#pathOf(index), bytes);
+    if (!this.#stored.has(index)) {
+      this.#bytes += bytes.length;
+    } else {
+      this.#bytes += bytes.length - (this.#stored.get(index) ?? 0);
+    }
+    this.#stored.set(index, bytes.length);
+    this.#touched.set(index, this.#now());
+  }
+
+  /**
+   * Read a piece back into a buffer the caller already owns.
+   *
+   * @param {number} index
+   * @param {Uint8Array} target - Destination; its length is what gets read.
+   * @returns {Promise<number>} Bytes read.
+   */
+  async read(index, target) {
+    if (!this.#stored.has(index)) {
+      throw new Error(`Piece ${index} is not on disk.`);
+    }
+    // While this runs, eviction leaves the piece alone. Removing a file that is
+    // open is safe on POSIX and the read would finish — but the open itself
+    // happens below, and between the check above and that open a file removed
+    // is a read that fails for a piece the caller was told is there.
+    this.#reading.set(index, (this.#reading.get(index) ?? 0) + 1);
+    let handle = null;
+    try {
+      handle = await fs.open(this.#pathOf(index), "r");
+      const { bytesRead } = await handle.read(target, 0, target.length, 0);
+      this.#touched.set(index, this.#now());
+      return bytesRead;
+    } finally {
+      await handle?.close().catch(() => undefined);
+      const outstanding = (this.#reading.get(index) ?? 1) - 1;
+      if (outstanding > 0) {
+        this.#reading.set(index, outstanding);
+      } else {
+        this.#reading.delete(index);
+      }
+    }
+  }
+
+  /**
+   * Forget a piece and give its disk back.
+   *
+   * The removal itself is left to run: the caller's contract is synchronous —
+   * after this returns, the store no longer has the piece — and the blocks come
+   * back a moment later. A failure to remove leaves a file nobody will read;
+   * the directory goes whole when the store is destroyed.
+   *
+   * @param {number} index
+   * @returns {void}
+   */
+  forget(index) {
+    const length = this.#stored.get(index);
+    if (length === undefined) {
+      return;
+    }
+    this.#stored.delete(index);
+    this.#touched.delete(index);
+    this.#bytes -= length;
+    // Kept track of, because a removal still in flight holds the directory: on
+    // Windows `rmdir` refuses while any handle inside is open, so `destroy`
+    // waits for these before removing the directory. On Linux it would succeed
+    // and the removals would then fail silently against a directory that is
+    // gone — tidy either way, and correct on both.
+    const removal = fs.rm(this.#pathOf(index), REMOVAL)
+      .catch(() => undefined)
+      .finally(() => {
+        if (this.#removing.get(index) === removal) {
+          this.#removing.delete(index);
+        }
+      });
+    this.#removing.set(index, removal);
+  }
+
+  /**
+   * What it holds, what it may hold, and what it has had to throw away.
+   *
+   * @returns {{ pieces: number, bytes: number, allowanceBytes: number | null, evictions: number }}
+   */
+  stats() {
+    return {
+      pieces: this.#stored.size,
+      bytes: this.#bytes,
+      allowanceBytes: this.#allowanceBytes,
+      evictions: this.#evictions
+    };
+  }
+
+  /**
+   * Close it, leaving its contents in place.
+   *
+   * There is no handle to close — a read opens and closes its own — so this
+   * exists for the caller's lifecycle and does nothing else.
+   *
+   * @returns {Promise<void>}
+   */
+  async close() {
+    // Nothing to release: a read opens and closes its own handle.
+  }
+
+  /**
+   * Close and remove everything.
+   *
+   * @returns {Promise<void>}
+   */
+  async destroy() {
+    await this.close();
+    await this.settled();
+    this.#stored.clear();
+    this.#touched.clear();
+    this.#bytes = 0;
+    await fs.rm(this.#directory, { ...REMOVAL, recursive: true });
+  }
+
+  /**
+   * Wait for every removal this store has started.
+   *
+   * Removing a piece answers at once and frees the disk a moment later; this is
+   * how a caller that needs the disk back NOW — a test, or `destroy` — waits for
+   * it without making `forget` asynchronous for everybody else.
+   *
+   * @returns {Promise<void>}
+   */
+  async settled() {
+    while (this.#removing.size > 0) {
+      await Promise.all([...this.#removing.values()]);
+    }
+  }
+
+  /**
+   * @param {number} index
+   * @returns {string}
+   */
+  #pathOf(index) {
+    return path.join(this.#directory, `${index}.piece`);
+  }
+
+  /**
+   * Make sure the directory is there, before every write.
+   *
+   * Deliberately not remembered. A remembered "it exists" is a statement about
+   * the past: this store's own `destroy` removes the directory, a startup sweep
+   * removes what a killed process left, and an operator clearing a full disk
+   * removes anything. Each of those turns the memory into a lie and every write
+   * after it into `ENOENT`. Creating a directory that already exists costs tens
+   * of microseconds against a write measured in milliseconds.
+   *
+   * @returns {Promise<void>}
+   */
+  async #ensureDirectory() {
+    await fs.mkdir(this.#directory, { recursive: true });
+  }
+
+  /**
+   * Throw away the least recently used pieces until one more will fit.
+   *
+   * Least recently used first, and for the reason the memory tier uses the same
+   * order: what nobody has read for the longest is what a viewer is least
+   * likely to want next. A piece being read now is never a victim, and neither
+   * is the piece about to be written.
+   *
+   * A piece thrown away is not lost, only un-had: the store answers `has` with
+   * false, the read that wanted it gets nothing, and the torrent fetches it
+   * again. That is the same bargain the memory tier makes when it spills.
+   *
+   * @param {number} incomingBytes
+   * @param {number} incomingIndex
+   * @returns {Promise<void>}
+   */
+  async #makeRoomFor(incomingBytes, incomingIndex) {
+    if (this.#allowanceBytes === null) {
+      return;
+    }
+    const already = this.#stored.get(incomingIndex) ?? 0;
+    while (this.#bytes - already + incomingBytes > this.#allowanceBytes) {
+      const victim = this.#leastRecentlyUsed(incomingIndex);
+      if (victim === null) {
+        // Everything left is either being read or is the piece coming in. The
+        // write goes ahead: refusing it would lose a piece the swarm has
+        // already paid for, and the next write finds the readers gone.
+        return;
+      }
+      this.forget(victim);
+      this.#evictions += 1;
+    }
+  }
+
+  /**
+   * @param {number} except
+   * @returns {number | null}
+   */
+  #leastRecentlyUsed(except) {
+    let victim = null;
+    let oldest = Number.POSITIVE_INFINITY;
+    for (const [index, at] of this.#touched) {
+      if (index === except || this.#reading.has(index)) {
+        continue;
+      }
+      if (at < oldest) {
+        oldest = at;
+        victim = index;
+      }
+    }
+    return victim;
+  }
+}