@torrent-tv/proxy 2.86.0 → 2.87.0

package/services/{piece-store → storage/piece-store}/piece-disk-store.js

@@ -54,8 +54,6 @@ const REMOVAL = { force: true, maxRetries: 10, retryDelay: 20 };
 export class PieceDiskStore {
   #directory;
 
-  #chunkLength;
-
   /** Piece index → its length on disk. @type {Map<number, number>} */
   #stored = new Map();
 
@@ -99,9 +97,6 @@ export class PieceDiskStore {
    * @param {string} params.directory - Where this torrent's pieces live.
    * @param {string} params.name - A name unique to the torrent; it becomes the
    *   directory inside `directory`.
-   * @param {number} params.chunkLength - The torrent's piece length. Kept for
-   *   the caller's arithmetic; a piece's own length is recorded as it is written,
-   *   because the last piece of a torrent is shorter.
    * @param {number | null} [params.allowanceBytes] - What it may hold. Null
    *   means nobody has said yet, and nothing is evicted until somebody does.
    * @param {() => number} [params.now]
@@ -109,14 +104,12 @@ export class PieceDiskStore {
   constructor({
     directory,
     name,
-    chunkLength,
     allowanceBytes = null,
     now = Date.now,
     readHeads = () => [],
     onForgotten = () => undefined
   }) {
     this.#directory = path.join(directory, name);
-    this.#chunkLength = chunkLength;
     this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
     this.#now = now;
     this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];

package/services/{piece-store → storage/piece-store}/shared-piece-store.js

@@ -41,9 +41,8 @@
  * and nothing here disagrees.
  */
 
-import os from "node:os";
-import { readFileSync } from "node:fs";
-import { divideAllowance, machineAllowanceBytes, OtherDemand } from "./allowance.js";
+import { divideAllowance, machineAllowanceBytes, OtherDemand } from "../allowance.js";
+import { availableMemoryBytes as availableMemorySync } from "../machine-memory.js";
 import { PieceLru } from "./piece-lru.js";
 import { PieceDiskStore } from "./piece-disk-store.js";
 
@@ -112,12 +111,20 @@ export function forgetMachineMemory() {
 
 export { divideAllowance, machineAllowanceBytes };
 
-export function reviseStoreBudgets() {
+export function reviseStoreBudgets(allowanceBytes = null) {
   const stores = [...liveStores];
   const held = stores.reduce((sum, store) => sum + store.residentBytes, 0);
-  const available = availableMemorySync();
-  const reserve = noteMachineMemory(available, held);
-  const allowance = machineAllowanceBytes(available, held, reserve);
+  // TOLD, NOT WORKED OUT. Memory has one owner and it is on the main thread,
+  // where the disk's owner is — and it must be the same one, because the two
+  // resources trade: pieces that do not fit in memory are spilled to disk, so
+  // what this store is given decides how much disk it needs. What arrives is
+  // this thread's whole share; the stores divide it between themselves.
+  //
+  // Null is a thread that has not been told yet, and it works its own share out
+  // for that one pass rather than growing without a bound.
+  const allowance = Number.isFinite(allowanceBytes) && allowanceBytes !== null
+    ? Math.max(0, allowanceBytes)
+    : machineAllowanceBytes(availableMemorySync(), held, noteMachineMemory(availableMemorySync(), held));
   const shares = divideAllowance(stores.map((store) => store.wantedBytes), allowance);
   const revised = [];
   for (const [position, store] of stores.entries()) {
@@ -175,18 +182,6 @@ function defaultMemoryBytes() {
   return Math.floor(allowance / (stores.length + 1));
 }
 
-function availableMemorySync() {
-  try {
-    const text = readFileSync("/proc/meminfo", "utf8");
-    const match = /^MemAvailable:\s+(\d+)\s+kB$/m.exec(text);
-    if (match) {
-      return Number(match[1]) * 1024;
-    }
-  } catch {
-  }
-  return os.freemem();
-}
-
 /**
  * How many blocks of piece memory this thread has let go of, and how many the
  * collector has actually taken back.
@@ -204,7 +199,7 @@ function availableMemorySync() {
  *
  * What it does NOT prove: a `SharedArrayBuffer`'s memory is shared between the
  * isolates, so this thread's handle going means only that THIS thread let go.
- * The main thread counts its own (`torrent-worker/client.js`), and the pair is
+ * The main thread counts its own (`torrent/worker/client.js`), and the pair is
  * what answers the question.
  */
 const released = { count: 0, collected: 0 };
@@ -522,7 +517,6 @@ export class SharedPieceStore {
     this.#disk = options.disk ?? new PieceDiskStore({
       directory: options.path ?? ".",
       name: `${this.#name}.pieces`,
-      chunkLength,
       // What it may hold is settled by the same revision that settles memory,
       // within a minute of the store existing. Until then it is unbounded, which
       // is what it has always been — the difference is that it now stops.
@@ -1871,3 +1865,16 @@ export class SharedPieceStore {
     this.#disk.destroy().then(() => callback(null), (error) => callback(error));
   }
 }
+
+/**
+ * What the stores in this thread hold and would take, for the owner of memory.
+ *
+ * @returns {{ held: number, wanted: number }}
+ */
+export function memoryClaim() {
+  const stores = [...liveStores];
+  return {
+    held: stores.reduce((sum, store) => sum + store.residentBytes, 0),
+    wanted: stores.reduce((sum, store) => sum + store.wantedBytes, 0)
+  };
+}

package/services/{encode → storage/segment-store}/SegmentStore.js

@@ -40,7 +40,7 @@
  */
 
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, watch, writeFileSync } from "node:fs";
 import { rmSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
@@ -90,6 +90,27 @@ export class SegmentStore {
   /** Output key → when it was last asked for. @type {Map<string, number>} */
   #touched = new Map();
 
+  /**
+   * Output key → the init segment served for it, once one has been read whole.
+   *
+   * A property of the output's produced material, so it lives with that
+   * material and goes when it goes. The player fetches it once and never again,
+   * so the first whole one is kept and every later request gets the same bytes,
+   * even though a re-encoding run rewrites the file on disk.
+   *
+   * @type {Map<string, Buffer>}
+   */
+  #inits = new Map();
+
+  /** Told of every segment published, whatever it is. @type {Set<(key: string, index: number) => void>} */
+  #publishedListeners = new Set();
+
+  /** Segment publication waiters by output key and segment number. @type {Map<string, Map<number, Set<Function>>>} */
+  #waiters = new Map();
+
+  /** Output key → the watch on its directory. @type {Map<string, import("node:fs").FSWatcher>} */
+  #watchers = new Map();
+
   /** @type {{ info: Function, warn: Function }} */
   #logger;
 
@@ -144,6 +165,9 @@ export class SegmentStore {
       writeFileSync(path.join(dir, KEY_FILE), `${key}\n`, "utf8");
     }
     this.#touched.set(key, this.#now());
+    // Production is about to begin here, and a piece closing in this directory
+    // is what every held request is waiting for.
+    this.#watch(key);
     return dir;
   }
 
@@ -187,6 +211,7 @@ export class SegmentStore {
       return known;
     }
     const byNumber = new Map();
+    const sizes = new Map();
     let bytes = 0;
     let highest = -1;
     // The biggest piece and which number it is. What reads it is the figure the
@@ -220,6 +245,7 @@ export class SegmentStore {
           continue;
         }
         byNumber.set(index, full);
+        sizes.set(index, size);
         bytes += size;
         if (index > highest) {
           highest = index;
@@ -232,7 +258,7 @@ export class SegmentStore {
       this.#held.delete(key);
       return empty;
     }
-    const contents = { readAt: mtime, byNumber, bytes, unproven: highest, largest };
+    const contents = { readAt: mtime, byNumber, sizes, bytes, unproven: highest, largest };
     this.#held.set(key, contents);
     return contents;
   }
@@ -276,6 +302,20 @@ export class SegmentStore {
     return this.refresh(key).byNumber.size;
   }
 
+  /**
+   * The size in bytes of every finished piece of this output, by number.
+   *
+   * Read by whoever keeps what an output's segments carried (roadmap item 97,
+   * step 14). A reading, not a request: it does not mark the output as read,
+   * so it does not keep an output nobody is watching.
+   *
+   * @param {string} key
+   * @returns {Map<number, number>}
+   */
+  sizesOf(key) {
+    return new Map(this.refresh(key).sizes ?? []);
+  }
+
   /**
    * The biggest piece this output has made, and which number it is.
    *
@@ -292,6 +332,16 @@ export class SegmentStore {
   }
 
 
+  /**
+   * Every output whose pieces this store knows how to read: those in use and
+   * those a previous life of the process left.
+   *
+   * @returns {string[]}
+   */
+  addresses() {
+    return [...this.#formats.keys()];
+  }
+
   /**
    * Whether this piece is finished, and may therefore be served.
    *
@@ -395,9 +445,287 @@ export class SegmentStore {
     }
     // What the directory holds has changed, so the memory of it is stale.
     this.#held.delete(key);
+    const index = format?.segmentIndexFromName?.(served) ?? -1;
+    if (index >= 0) {
+      this.announce(key, index);
+      for (const listener of this.#publishedListeners) {
+        listener(key, index);
+      }
+    }
     return served;
   }
 
+  /**
+   * The init segment kept for this output, or null until one was read whole.
+   *
+   * @param {string} key
+   * @returns {Buffer | null}
+   */
+  initOf(key) {
+    return this.#inits.get(key) ?? null;
+  }
+
+  /**
+   * Keep the init segment served for this output. The first whole one stands;
+   * an empty read is not an init (ffmpeg creates the file before it writes the
+   * header into it) and is refused.
+   *
+   * @param {string} key
+   * @param {Buffer | null} bytes
+   * @returns {Buffer | null} The init that now stands for this output.
+   */
+  keepInit(key, bytes) {
+    if (!this.#inits.has(key) && bytes && bytes.length > 0) {
+      this.#inits.set(key, bytes);
+    }
+    return this.initOf(key);
+  }
+
+  /**
+   * Be told of every segment published: what the store holds has grown, which
+   * is when whether it still fits its share is worth asking.
+   *
+   * @param {(key: string, index: number) => void} listener
+   * @returns {() => void} Stop being told.
+   */
+  onPublished(listener) {
+    this.#publishedListeners.add(listener);
+    return () => this.#publishedListeners.delete(listener);
+  }
+
+  /**
+   * Watch this output's directory, so a piece closed by ANY writer is an event.
+   *
+   * WHY IT EXISTS, and it is one fault rather than a precaution. A piece is
+   * finished in one of two ways, and only one of them said so. The `segment`
+   * muxer writes under a working name and reports the closure on a channel of
+   * its own (`publish`); the `hls` muxer writes through a temporary name of its
+   * own (`+temp_file`) and reports nothing, so the file simply appears under the
+   * name it is served as. That branch is taken by every re-encoded output on the
+   * even grid — the ordinary quality step — and for those `waitFor` could only
+   * end on its deadline: the segment was on disk and the request that wanted it
+   * went on waiting, up to a full minute, and was then answered 503.
+   *
+   * So the store watches what it owns. The rename into place moves the
+   * directory's modification time, the kernel says so, and both branches reach
+   * one statement: this number is now present. Nothing polls.
+   *
+   * Idempotent, and started from the two places that need it — the directory
+   * being made for a run, and a wait beginning on an output this process has
+   * adopted rather than made.
+   *
+   * WHOEVER REMOVES ONE OF THESE DIRECTORIES GOES THROUGH `drop`, which stops
+   * the watch first. On Linux, where this runs, an open watch does not prevent
+   * a directory being removed; on Windows it does, and `rmSync` then retries
+   * until it gives up — measured, a check that removed a store's directory
+   * behind its back took 110 s instead of 3.
+   *
+   * @param {string} key
+   * @returns {void}
+   */
+  #watch(key) {
+    if (this.#watchers.has(key)) {
+      return;
+    }
+    const dir = path.join(this.#root, directoryNameFor(key));
+    // Nothing has been made for this output yet. A wait that begins first will
+    // ask again — `directoryFor` is what a run calls before it writes — and
+    // asking the kernel to watch what is not there throws, once for every wait
+    // that begins meanwhile.
+    if (!existsSync(dir)) {
+      return;
+    }
+    try {
+      const watcher = watch(dir, { persistent: false }, (_event, name) => {
+        this.#noticed(key, typeof name === "string" ? name : null);
+      });
+      // A directory that goes away takes its watch with it rather than leaving
+      // an error nobody reads.
+      watcher.on("error", () => this.#unwatch(key));
+      this.#watchers.set(key, watcher);
+    } catch (error) {
+      // Watching is how a wait ends early, not how a segment is found: every
+      // reader still reads the disk. Said rather than swallowed, because a
+      // store that cannot watch holds each request for its whole deadline, and
+      // that is worth knowing before the field says it.
+      this.#logger?.warn?.(
+        `segment store: cannot watch ${key.slice(0, 60)}: ` +
+        `${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  /**
+   * Stop watching one output's directory.
+   *
+   * @param {string} key
+   * @returns {void}
+   */
+  #unwatch(key) {
+    const watcher = this.#watchers.get(key);
+    if (!watcher) {
+      return;
+    }
+    this.#watchers.delete(key);
+    try {
+      watcher.close();
+    } catch {
+      // Already closed with its directory.
+    }
+  }
+
+  /**
+   * The directory moved: say what is now there.
+   *
+   * A REMOVAL MOVES IT TOO, so the name is checked against the disk before
+   * anything is announced — otherwise clearing up after a dead run would read
+   * as that run's pieces arriving.
+   *
+   * @param {string} key
+   * @param {string | null} name - What changed, where the platform says.
+   * @returns {void}
+   */
+  #noticed(key, name) {
+    // WHAT WAS READ OF THIS DIRECTORY IS STALE. Dropped rather than re-read:
+    // re-reading HERE is quadratic in the pieces of a film — one listing per
+    // file written, of a directory that grows with every file, measured at 110 s
+    // against 3 s over 482 pieces — while dropping costs one listing to whoever
+    // asks next, which is bounded by the questions and not by the writes.
+    //
+    // IT IS NOT ENOUGH TO LET `refresh` NOTICE BY ITSELF. It re-reads when the
+    // directory's modification time has moved, and that time is stated in
+    // milliseconds: a file written in the same millisecond as the last reading
+    // leaves it unmoved, and the piece is then reported absent although it is
+    // there. That is the hazard `refresh` documents and `publish` has always
+    // guarded against by dropping the memory.
+    this.#held.delete(key);
+    const format = this.#formats.get(key);
+    const dir = path.join(this.#root, directoryNameFor(key));
+    const index = name && format?.isSegmentFileName?.(name)
+      ? (format.segmentIndexFromName?.(name) ?? -1)
+      : -1;
+    if (index >= 0) {
+      // A REMOVAL MOVES THE DIRECTORY TOO, so the one file is asked about
+      // before anything is announced — otherwise clearing up after a dead run
+      // would read as that run's pieces arriving.
+      if (!existsSync(path.join(dir, name))) {
+        return;
+      }
+      this.#announceArrival(key, index);
+      return;
+    }
+    if (name) {
+      return;
+    }
+    // The platform named nothing. Then whoever is waiting asks about their own
+    // piece, which is the same question with the same answer and no guessing.
+    const byIndex = this.#waiters.get(key);
+    if (!byIndex || !format?.segmentFileName) {
+      return;
+    }
+    for (const waiting of [...byIndex.keys()]) {
+      if (existsSync(path.join(dir, format.segmentFileName(waiting)))) {
+        this.#announceArrival(key, waiting);
+      }
+    }
+  }
+
+  /**
+   * One statement that a number is now present here.
+   *
+   * @param {string} key
+   * @param {number} index
+   * @returns {void}
+   */
+  #announceArrival(key, index) {
+    this.announce(key, index);
+    for (const listener of this.#publishedListeners) {
+      listener(key, index);
+    }
+  }
+
+  /**
+   * Announce that a segment is now present. This stores no readiness state;
+   * the file under its served name remains the only fact.
+   *
+   * @param {string} key
+   * @param {number} index
+   */
+  announce(key, index) {
+    const byIndex = this.#waiters.get(key);
+    const waiting = byIndex?.get(index);
+    if (!waiting) return;
+    byIndex.delete(index);
+    if (byIndex.size === 0) this.#waiters.delete(key);
+    for (const resolve of waiting) resolve(true);
+  }
+
+  /**
+   * How many requests are waiting on this output right now.
+   *
+   * The store's own state, and the one fact that says a wait EXISTS rather than
+   * is about to: a check that ends a wait it has not yet registered proves
+   * nothing, and passes just as well without the thing it is checking.
+   *
+   * @param {string} key
+   * @returns {number}
+   */
+  waitingFor(key) {
+    let waiting = 0;
+    for (const holders of this.#waiters.get(key)?.values() ?? []) {
+      waiting += holders.size;
+    }
+    return waiting;
+  }
+
+  /**
+   * Wait for a segment publication event or one deadline. This does not poll
+   * the filesystem.
+   *
+   * @param {string} key
+   * @param {number} index
+   * @param {number} timeoutMs - May be infinite: then only the publication or
+   *   `cancelled` ends the wait.
+   * @param {Promise<unknown> | null} [cancelled] - Settles when whoever waits
+   *   has gone, so the waiter does not outlive them.
+   * @returns {Promise<boolean>}
+   */
+  waitFor(key, index, timeoutMs, cancelled = null) {
+    if (this.pathOf(key, index)) return Promise.resolve(true);
+    // An output adopted from a previous process has a directory nobody made in
+    // this one, so the wait itself is the first moment it is watched.
+    this.#watch(key);
+    return new Promise((resolve) => {
+      const byIndex = this.#waiters.get(key) ?? new Map();
+      const waiting = byIndex.get(index) ?? new Set();
+      this.#waiters.set(key, byIndex);
+      byIndex.set(index, waiting);
+      let settled = false;
+      const finish = (published) => {
+        if (settled) return;
+        settled = true;
+        if (timer) clearTimeout(timer);
+        waiting.delete(finish);
+        // Removed only if they are still the sets this waiter joined: an
+        // announcement takes the set out before calling its waiters, and a new
+        // waiter may have registered a fresh one under the same number since.
+        if (waiting.size === 0 && byIndex.get(index) === waiting) byIndex.delete(index);
+        if (byIndex.size === 0 && this.#waiters.get(key) === byIndex) this.#waiters.delete(key);
+        resolve(published);
+      };
+      waiting.add(finish);
+      // No deadline is a wait that ends only on the publication: a timer given
+      // an infinite delay fires at once, which would turn the wait into a poll.
+      const timer = Number.isFinite(timeoutMs) ? setTimeout(() => finish(false), Math.max(0, timeoutMs)) : null;
+      timer?.unref?.();
+      cancelled?.then(() => finish(false));
+      // Publication may have landed after the first path check and before this
+      // waiter was registered. Recheck once after registration so that window
+      // cannot turn a present segment into a full-deadline wait.
+      if (this.pathOf(key, index)) finish(true);
+    });
+  }
+
   /**
    * Clear up after a run that has ended: remove what it left unfinished.
    *
@@ -437,6 +765,42 @@ export class SegmentStore {
   }
 
 
+  /**
+   * Where the file of this name is, or null when this output does not hold it.
+   *
+   * By NAME rather than by number, because that is what a request carries. The
+   * conversion is the store's own business: it is told how this output's files
+   * are named (`useFormat`) and therefore needs nobody else to read one.
+   *
+   * @param {string} key
+   * @param {string} fileName
+   * @returns {string | null}
+   */
+  pathOfName(key, fileName) {
+    const format = this.#formats.get(key);
+    const index = format?.segmentIndexFromName?.(fileName) ?? -1;
+    if (!Number.isInteger(index) || index < 0) {
+      return null;
+    }
+    return this.pathOf(key, index);
+  }
+
+  /**
+   * Forget what this output's directory was last seen to hold.
+   *
+   * What is held is re-read when the directory's modification time moves, which
+   * is how a quiet request costs one `stat`. A caller that has just removed a
+   * file and asks again in the same tick would otherwise be told it is still
+   * there — the time has moved, but only by as much as the clock's own
+   * granularity, and that is not something to depend on.
+   *
+   * @param {string} key
+   * @returns {void}
+   */
+  forget(key) {
+    this.#held.delete(key);
+  }
+
   /**
    * Where a segment is, or null when this output does not hold it.
    *
@@ -475,9 +839,11 @@ export class SegmentStore {
     } catch {
       // Already gone, or in use; the next sweep sees it either way.
     }
+    this.#unwatch(key);
     this.#held.delete(key);
     this.#formats.delete(key);
     this.#touched.delete(key);
+    this.#inits.delete(key);
     this.#logger.info(`segment-store dropped ${directoryNameFor(key)} (${because})`);
   }
 
@@ -496,12 +862,30 @@ export class SegmentStore {
   }
 
   /**
-   * Throw away everything this store owns, and the root with it.
+   * Throw away everything this store owns.
    *
    * For a clean exit. What is left on disk afterwards is by definition from a
    * kill, which is the case the startup sweep exists for — and without this the
    * sweep adopts, the exit leaves, and the next start adopts again, for ever.
    *
+   * WHAT IT OWNS IS ITS DIRECTORIES, NOT THE ROOT. The root is a fixed path —
+   * `os.tmpdir()/torrent-tv-hls` — so it belongs to the machine and not to any
+   * one process, and this used to remove it whole. The comment that stood here
+   * named the hazard exactly ("another process may share the root") and did it
+   * anyway.
+   *
+   * Measured 2026-09-14, where a second process is ordinary: `node --test` runs
+   * every test file in its own process in parallel, and four files call this
+   * forty times between them. `segment-serve-wiring.test.js` then reported a
+   * segment missing that it had written itself a moment earlier, 2 to 4 checks
+   * per run, never the same ones — because another process had deleted the
+   * directory under it. The same thing happens to two proxies on one machine:
+   * a bare npm run beside the addon, or the wedge stand beside the product, and
+   * one of them exiting takes the other's segments mid-serve.
+   *
+   * A store removes what it made. It does not remove the container it made it
+   * in, and an empty root costs nothing.
+   *
    * @param {string} because
    * @returns {number} How many outputs went.
    */
@@ -511,11 +895,14 @@ export class SegmentStore {
       this.drop(key, because);
       dropped += 1;
     }
-    try {
-      rmSync(this.#root, { recursive: true, force: true });
-    } catch {
-      // Another process may share the root and hold a directory open. What is
-      // ours is gone either way.
+    // EVERY WATCH GOES, not only the watches of outputs that named their file
+    // format. A directory is made before anything says how its files are named
+    // — `directoryFor` is what a run calls, `useFormat` is a separate statement
+    // — so a store can hold a watch on a key that is in no format list, and
+    // dropping by that list left it open: on Windows the process then does not
+    // exit, which is how this was found.
+    for (const key of [...this.#watchers.keys()]) {
+      this.#unwatch(key);
     }
     return dropped;
   }