@torrent-tv/proxy 2.83.0 → 2.83.2

package/services/files/CompletedFiles.js

@@ -0,0 +1,276 @@
+/**
+ * @file Files this proxy has downloaded whole, kept as files.
+ *
+ * A torrent is a way of GETTING bytes. Once every byte of a file is here, the
+ * torrent has nothing left to do for it: the file is a file, and reading it is
+ * an ordinary read of an ordinary file — no piece store, no memory ceiling, no
+ * eviction, no revival from a spill, and nothing that can refuse a read for
+ * want of memory.
+ *
+ * WHY THIS EXISTS AT ALL, in the words it was asked for (2026-09-11): as soon
+ * as a torrent is fully downloaded, downloading stops, the torrent is deleted,
+ * and the artefacts — what was downloaded — stay for as long as they are
+ * wanted.
+ *
+ * WHERE THEY LIVE, and it is not beside the torrent's own store. Destroying a
+ * torrent with `destroyStore` removes that directory whole, which is exactly
+ * what the instruction above ends with — so a file kept inside it would be
+ * deleted by the very act it is meant to survive.
+ *
+ * WHAT IS HERE IS ADOPTED AT STARTUP, for the same reason the piece store
+ * adopts its own directory: a proxy that has restarted has these files and must
+ * not fetch them again. A file whose size does not match what the torrent says
+ * is not adopted — it was being written when the process died.
+ */
+
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+/**
+ * Where whole files live.
+ *
+ * Beside the torrents' own directories and not inside them: destroying a
+ * torrent with its store removes that directory whole, and these files exist to
+ * survive exactly that.
+ *
+ * @returns {string}
+ */
+export function completedFilesRoot() {
+  return path.join(os.tmpdir(), "torrent-tv-files");
+}
+
+/** How a removal is asked for; Windows needs the retries, POSIX ignores them. */
+const REMOVAL = { force: true, maxRetries: 10, retryDelay: 20 };
+
+/**
+ * One directory of whole files, keyed by the torrent they came from.
+ */
+export class CompletedFiles {
+  #root;
+
+  /** `${infoHash}/${fileIndex}` → what is held. @type {Map<string, { path: string, length: number, name: string }>} */
+  #held = new Map();
+
+  /** Assemblies in flight, so two passes cannot write one file at once. @type {Set<string>} */
+  #writing = new Set();
+
+  /**
+   * @param {object} params
+   * @param {string} params.root - Where whole files live. Outside any torrent's
+   *   own store directory, which is removed with the torrent.
+   */
+  constructor({ root }) {
+    this.#root = root;
+  }
+
+  /** Where these files live, for logging. */
+  get root() {
+    return this.#root;
+  }
+
+  /** How many whole files are held. */
+  get size() {
+    return this.#held.size;
+  }
+
+  /** What they weigh. */
+  get bytes() {
+    let total = 0;
+    for (const file of this.#held.values()) {
+      total += file.length;
+    }
+    return total;
+  }
+
+  /**
+   * @param {string} infoHash
+   * @param {number} fileIndex
+   * @returns {string}
+   */
+  #keyOf(infoHash, fileIndex) {
+    return `${String(infoHash)}/${fileIndex}`;
+  }
+
+  /**
+   * The whole file for this torrent and index, or null.
+   *
+   * @param {string} infoHash
+   * @param {number} fileIndex
+   * @returns {{ path: string, length: number, name: string } | null}
+   */
+  find(infoHash, fileIndex) {
+    return this.#held.get(this.#keyOf(infoHash, fileIndex)) ?? null;
+  }
+
+  /**
+   * Take up whole files a previous life of this proxy left here.
+   *
+   * @param {(infoHash: string, fileIndex: number) => number | null} lengthOf -
+   *   What the torrent says this file weighs, or null when it is not known. A
+   *   file of the wrong size was being written when the process died and is not
+   *   adopted.
+   * @returns {Promise<number>} How many were taken up.
+   */
+  async adopt(lengthOf) {
+    let adopted = 0;
+    let torrents = [];
+    try {
+      torrents = await fs.readdir(this.#root, { withFileTypes: true });
+    } catch {
+      return 0;
+    }
+    for (const entry of torrents) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      const infoHash = entry.name;
+      let files = [];
+      try {
+        files = await fs.readdir(path.join(this.#root, infoHash), { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      let manifest = {};
+      try {
+        manifest = JSON.parse(
+          await fs.readFile(path.join(this.#root, infoHash, "manifest.json"), "utf8")
+        );
+      } catch {
+        // No manifest, or an unreadable one: the bytes are still servable, and
+        // a file named by its number is better than a file thrown away.
+      }
+      for (const held of files) {
+        const fileIndex = Number.parseInt(held.name, 10);
+        if (!held.isFile() || !Number.isInteger(fileIndex) || fileIndex < 0) {
+          continue;
+        }
+        const where = path.join(this.#root, infoHash, held.name);
+        try {
+          const { size } = await fs.stat(where);
+          const expected = lengthOf(infoHash, fileIndex);
+          if (expected !== null && size !== expected) {
+            await fs.rm(where, REMOVAL);
+            continue;
+          }
+          this.#held.set(this.#keyOf(infoHash, fileIndex), {
+            path: where,
+            length: size,
+            name: String(manifest?.[String(fileIndex)]?.name ?? fileIndex)
+          });
+          adopted += 1;
+        } catch {
+          // Gone between the listing and the reading: not ours to worry about.
+        }
+      }
+    }
+    return adopted;
+  }
+
+  /**
+   * Write one whole file out of whatever the torrent can read it from.
+   *
+   * Written under a name nothing serves and renamed when it is closed, so a
+   * process killed halfway leaves no file that looks complete. The rename is
+   * the one moment at which the file becomes servable, which is also why
+   * nothing has to be locked against readers.
+   *
+   * @param {object} params
+   * @param {string} params.infoHash
+   * @param {number} params.fileIndex
+   * @param {number} params.length - What the torrent says the file weighs.
+   * @param {string} params.name - What the torrent calls it. Kept because the
+   *   torrent is what is about to be deleted, and a file with no name can only
+   *   be served as a number.
+   * @param {() => NodeJS.ReadableStream} params.open - The torrent's own read of
+   *   the whole file.
+   * @returns {Promise<{ path: string, length: number } | null>} Null when
+   *   another pass is already writing it, or when what was read does not weigh
+   *   what the torrent said.
+   */
+  async keep({ infoHash, fileIndex, length, name, open }) {
+    const key = this.#keyOf(infoHash, fileIndex);
+    const held = this.#held.get(key);
+    if (held) {
+      return held;
+    }
+    if (this.#writing.has(key)) {
+      return null;
+    }
+    this.#writing.add(key);
+    const directory = path.join(this.#root, String(infoHash));
+    const where = path.join(directory, String(fileIndex));
+    const partial = `${where}.partial`;
+    try {
+      await fs.mkdir(directory, { recursive: true });
+      await fs.rm(partial, REMOVAL);
+      const source = open();
+      const handle = await fs.open(partial, "w");
+      let written = 0;
+      try {
+        for await (const chunk of source) {
+          await handle.write(chunk);
+          written += chunk.length;
+        }
+      } finally {
+        await handle.close();
+      }
+      if (written !== length) {
+        // The read ended early — the data went away mid-write, which over a
+        // torrent is ordinary. What must not happen is a short file under a
+        // name that says it is whole.
+        await fs.rm(partial, REMOVAL);
+        return null;
+      }
+      await fs.rename(partial, where);
+      const file = { path: where, length, name: String(name ?? fileIndex) };
+      this.#held.set(key, file);
+      await this.#writeManifest(infoHash);
+      return file;
+    } catch {
+      await fs.rm(partial, REMOVAL).catch(() => undefined);
+      return null;
+    } finally {
+      this.#writing.delete(key);
+    }
+  }
+
+  /**
+   * Write down what each file of one torrent is called and weighs.
+   *
+   * The torrent is what is about to be deleted, and it is the only thing that
+   * knows either. A number on disk is enough to serve the bytes and not enough
+   * to say what they are.
+   *
+   * @param {string} infoHash
+   * @returns {Promise<void>}
+   */
+  async #writeManifest(infoHash) {
+    const named = {};
+    for (const [key, file] of this.#held) {
+      if (key.startsWith(`${infoHash}/`)) {
+        named[key.slice(infoHash.length + 1)] = { length: file.length, name: file.name };
+      }
+    }
+    await fs
+      .writeFile(path.join(this.#root, String(infoHash), "manifest.json"), JSON.stringify(named))
+      .catch(() => undefined);
+  }
+
+  /**
+   * Forget and remove every whole file of one torrent.
+   *
+   * @param {string} infoHash
+   * @returns {Promise<void>}
+   */
+  async forget(infoHash) {
+    for (const key of [...this.#held.keys()]) {
+      if (key.startsWith(`${infoHash}/`)) {
+        this.#held.delete(key);
+      }
+    }
+    await fs.rm(path.join(this.#root, String(infoHash)), { ...REMOVAL, recursive: true }).catch(
+      () => undefined
+    );
+  }
+}

package/services/files/piece-from-whole-file.js

@@ -0,0 +1,118 @@
+/**
+ * @file Reading one piece of a torrent out of the files it has already been
+ * assembled into.
+ *
+ * THE KEYSTONE OF KEEPING WHOLE FILES AT ALL. Without it a whole file is a
+ * second copy of bytes the piece store is also holding, and neither copy can be
+ * dropped: the store cannot drop its own because it is what every piece read
+ * goes to, and the file cannot be dropped because it is what a torrent-free read
+ * goes to. With it the store has a place to fall back to, so
+ *
+ *   1. the spilled copy of a whole file is redundant and can go — the film stops
+ *      being on the disk twice, which on the field host of 2026-09-11 was
+ *      1417 MB of segments plus 1424 MB of spilled pieces for one episode;
+ *   2. the torrent can be destroyed with its store and added again later
+ *      without fetching a byte: what it verifies, it reads from here.
+ *
+ * A piece is a byte range of the torrent, and the torrent's files are laid end
+ * to end in that same space — so a piece belongs to one file, or straddles the
+ * boundary between two. Both cases are the same walk.
+ */
+
+import fs from "node:fs/promises";
+
+/**
+ * Read one piece out of whole files, or answer null.
+ *
+ * Null when any part of the piece is in a file this proxy does not hold whole:
+ * a piece half read is worse than a piece not read, because the layer above
+ * would hash it and mark the piece bad.
+ *
+ * @param {object} params
+ * @param {number} params.index - The piece.
+ * @param {number} params.pieceLength - Every piece but the last is this long.
+ * @param {number} params.length - What the whole torrent weighs.
+ * @param {Array<{ offset: number, length: number }>} params.files - The
+ *   torrent's files in order, as it lays them out.
+ * @param {(fileIndex: number) => { path: string, length: number } | null} params.wholeFileAt
+ * @returns {Promise<Buffer | null>}
+ */
+export async function pieceFromWholeFiles({ index, pieceLength, length, files, wholeFileAt }) {
+  if (!Array.isArray(files) || files.length === 0 || !(pieceLength > 0)) {
+    return null;
+  }
+  const pieceStart = index * pieceLength;
+  const pieceEnd = Math.min(pieceStart + pieceLength, length) - 1;
+  if (pieceStart > pieceEnd) {
+    return null;
+  }
+  const piece = Buffer.allocUnsafe(pieceEnd - pieceStart + 1);
+  let filled = 0;
+  for (const [fileIndex, file] of files.entries()) {
+    const fileStart = Number(file?.offset ?? 0);
+    const fileEnd = fileStart + Number(file?.length ?? 0) - 1;
+    const from = Math.max(pieceStart, fileStart);
+    const to = Math.min(pieceEnd, fileEnd);
+    if (from > to) {
+      continue;
+    }
+    const whole = wholeFileAt(fileIndex);
+    if (!whole) {
+      // Part of this piece is in a file this proxy does not hold whole.
+      return null;
+    }
+    let handle = null;
+    try {
+      handle = await fs.open(whole.path, "r");
+      const { bytesRead } = await handle.read(piece, from - pieceStart, to - from + 1, from - fileStart);
+      if (bytesRead !== to - from + 1) {
+        return null;
+      }
+      filled += bytesRead;
+    } catch {
+      return null;
+    } finally {
+      await handle?.close().catch(() => undefined);
+    }
+  }
+  return filled === piece.length ? piece : null;
+}
+
+/**
+ * Whether every byte of one piece is in files this proxy holds whole.
+ *
+ * Asked before dropping a spilled copy, and answered without reading anything.
+ *
+ * @param {object} params
+ * @param {number} params.index
+ * @param {number} params.pieceLength
+ * @param {number} params.length
+ * @param {Array<{ offset: number, length: number }>} params.files
+ * @param {(fileIndex: number) => { path: string, length: number } | null} params.wholeFileAt
+ * @returns {boolean}
+ */
+export function pieceIsInWholeFiles({ index, pieceLength, length, files, wholeFileAt }) {
+  if (!Array.isArray(files) || files.length === 0 || !(pieceLength > 0)) {
+    return false;
+  }
+  const pieceStart = index * pieceLength;
+  const pieceEnd = Math.min(pieceStart + pieceLength, length) - 1;
+  if (pieceStart > pieceEnd) {
+    return false;
+  }
+  let covered = 0;
+  for (const [fileIndex, file] of files.entries()) {
+    const fileStart = Number(file?.offset ?? 0);
+    const fileEnd = fileStart + Number(file?.length ?? 0) - 1;
+    const from = Math.max(pieceStart, fileStart);
+    const to = Math.min(pieceEnd, fileEnd);
+    if (from > to) {
+      continue;
+    }
+    if (!wholeFileAt(fileIndex)) {
+      return false;
+    }
+    covered += to - from + 1;
+  }
+  return covered === pieceEnd - pieceStart + 1;
+}

package/services/output/cut-grid.js

@@ -60,8 +60,8 @@ export function computeCutGrid({ useKeyframeGrid, durationSeconds, segDur, keyfr
   const base = Number.isFinite(startTime) ? startTime : 0;
   const uniform = () => {
     const boundaries = [];
-    for (let t = 0; t < total - 0.001; t += step) {
-      boundaries.push(Number(t.toFixed(6)));
+    for (let cutAt = 0; cutAt < total - 0.001; cutAt += step) {
+      boundaries.push(Number(cutAt.toFixed(6)));
     }
     boundaries.push(total);
     // One clock: nothing here is a keyframe of the source, so nothing is owed
@@ -74,7 +74,17 @@ export function computeCutGrid({ useKeyframeGrid, durationSeconds, segDur, keyfr
   const kept = keyframeTimes
     .filter((time) => Number.isFinite(time))
     .map((time) => ({ source: time, published: time - base }))
-    .filter((cut) => cut.published >= -0.001 && cut.published < total - 0.05)
+    // THE END IS MEASURED THE SAME WAY AS EVERY OTHER CUT. A keyframe nearer to
+    // the end than one segment leaves a tail too short to be a segment, and the
+    // rule below — no cut closer than a step to the one before it — never looks
+    // at the end at all. It used to be guarded by fifty milliseconds, a number
+    // from nowhere: field 2026-09-11, a keyframe 160 ms before the end of a
+    // 54-minute film passed it and left segment #541 lasting 0.16 s. The sound
+    // has no data in such a tail, so its run made 541 segments where the picture
+    // made 542, was marked short, and the repair that followed was handed a
+    // start later than its own end — 190 bytes that are not a fragment, and a
+    // viewer held 23 s at the last minute of the film for a 404.
+    .filter((cut) => cut.published >= -0.001 && cut.published < total - step)
     .sort((left, right) => left.published - right.published);
   // The first cut is the start of the file, whatever the container's own clock
   // says that is; the last is its end. Neither is a keyframe, and a run never

package/services/piece-store/piece-disk-store.js

@@ -31,6 +31,7 @@
  */
 
 import fs from "node:fs/promises";
+import { readdirSync, statSync } from "node:fs";
 import path from "node:path";
 
 /**
@@ -100,6 +101,59 @@ export class PieceDiskStore {
     this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
     this.#now = now;
     this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];
+    this.#adoptWhatIsAlreadyHere();
+  }
+
+  /**
+   * Take up the pieces a previous life of this torrent left in this directory.
+   *
+   * The directory is the torrent's own, so what is in it belongs to it — and
+   * without this, nothing ever reads those files again: a torrent that is torn
+   * down and added back gets a store whose index starts empty, answers "not on
+   * disk" for every piece it in fact has, and downloads the film a second time
+   * while the first copy sits beside it. That is what a torrent destroyed by an
+   * error left behind until 2026-09-11, and what the pool's own restart leaves
+   * behind every time.
+   *
+   * Read once, synchronously, because `has()` is answered synchronously and the
+   * torrent asks it immediately — a piece reported missing while a scan is
+   * still running is a piece fetched again. One directory listing per torrent.
+   *
+   * Correctness is not taken on trust: the torrent hashes every piece it means
+   * to use, so a file here that does not match is refused by the layer above
+   * and downloaded again.
+   *
+   * @returns {void}
+   */
+  #adoptWhatIsAlreadyHere() {
+    let entries = [];
+    try {
+      entries = readdirSync(this.#directory, { withFileTypes: true });
+    } catch {
+      // No directory yet: this torrent is new here, which is the ordinary case.
+      return;
+    }
+    const born = this.#now();
+    for (const entry of entries) {
+      if (!entry.isFile() || !entry.name.endsWith(".piece")) {
+        continue;
+      }
+      const index = Number.parseInt(entry.name.slice(0, -".piece".length), 10);
+      if (!Number.isInteger(index) || index < 0) {
+        continue;
+      }
+      try {
+        const { size } = statSync(path.join(this.#directory, entry.name));
+        if (size <= 0) {
+          continue;
+        }
+        this.#stored.set(index, size);
+        this.#touched.set(index, born);
+        this.#bytes += size;
+      } catch {
+        // Gone between the listing and the reading: not ours to worry about.
+      }
+    }
   }
 
   /** Where this store's pieces live, for logging and cleanup. */
@@ -107,6 +161,15 @@ export class PieceDiskStore {
     return this.#directory;
   }
 
+  /**
+   * Every piece this tier holds, as numbers.
+   *
+   * @returns {number[]}
+   */
+  indexes() {
+    return [...this.#stored.keys()];
+  }
+
   /** How many pieces are on disk. */
   get size() {
     return this.#stored.size;
@@ -213,11 +276,18 @@ export class PieceDiskStore {
   /**
    * Read a piece back into a buffer the caller already owns.
    *
+   * PART of a piece, when the caller asks for one. A peer asks for 16 KB at a
+   * time and a piece here is megabytes, so reading the whole of it to answer
+   * one request is the difference between 16 KB and 4 MB off the disk — field
+   * 2026-09-11: 63 416 reads of which 21.6 % came from memory, 49 696 pieces
+   * revived whole, to serve an upload capped at 512 KB/s.
+   *
    * @param {number} index
    * @param {Uint8Array} target - Destination; its length is what gets read.
+   * @param {number} [at] - Offset within the piece to start at.
    * @returns {Promise<number>} Bytes read.
    */
-  async read(index, target) {
+  async read(index, target, at = 0) {
     if (!this.#stored.has(index)) {
       throw new Error(`Piece ${index} is not on disk.`);
     }
@@ -229,7 +299,7 @@ export class PieceDiskStore {
     let handle = null;
     try {
       handle = await fs.open(this.#pathOf(index), "r");
-      const { bytesRead } = await handle.read(target, 0, target.length, 0);
+      const { bytesRead } = await handle.read(target, 0, target.length, Math.max(0, at));
       this.#touched.set(index, this.#now());
       return bytesRead;
     } finally {