@torrent-tv/proxy 2.83.4 → 2.83.6

package/test/withdraw-piece-claim.test.js

@@ -0,0 +1,212 @@
+/**
+ * @file One owner of the fact "this proxy has piece N".
+ *
+ * The store holds the bytes, so it owns the fact. The library keeps a second
+ * copy of it in its completion bitfield, and until 2026-09-12 nothing
+ * reconciled them: the disk tier dropped a piece once every reader was past it
+ * — correctly, that is what bounds the spill — and the bitfield went on saying
+ * the piece was verified. A read then concluded the piece was had, asked for it,
+ * was told it was absent, and failed; nothing fetched it again either, because
+ * the library does not download what it believes it owns. Field: a film played
+ * 80 seconds and then answered `Piece 0 is verified but absent from the store`
+ * for 92 minutes.
+ *
+ * Pinned here: the announcement and its one rule — said when the piece has gone
+ * from EVERYWHERE, never when one tier alone lost it — and the withdrawal.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import fs from "node:fs/promises";
+import { SharedPieceStore } from "../services/piece-store/shared-piece-store.js";
+import { withdrawClaim } from "../services/download/withdraw-claim.js";
+
+const PIECE = 1024;
+
+/**
+ * A store of four pieces with room for one, so every admission spills the last,
+ * plus the announcements it made.
+ */
+async function storeWithGone(extras = {}) {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "withdraw-test-"));
+  /** @type {number[]} */
+  const gone = [];
+  const store = new SharedPieceStore(PIECE, {
+    length: 4 * PIECE,
+    memoryBytes: PIECE,
+    path: directory,
+    name: "test",
+    files: [{ offset: 0, length: 4 * PIECE, name: "file.bin" }],
+    onPieceGone: ({ index }) => gone.push(index),
+    ...extras
+  });
+  const put = (index) => new Promise((resolve, reject) => {
+    store.put(index, Buffer.alloc(PIECE, index + 1), (error) => (error ? reject(error) : resolve()));
+  });
+  const clean = async () => {
+    store.destroy(() => undefined);
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
+  };
+  return { store, gone, put, clean };
+}
+
+/**
+ * Wait for the CONDITION, with a deadline only as a backstop. A fixed pause
+ * here would measure the machine rather than the store.
+ *
+ * @param {() => boolean} ready
+ * @param {string} what
+ */
+async function until(ready, what) {
+  const deadline = Date.now() + 5_000;
+  while (!ready()) {
+    if (Date.now() > deadline) {
+      throw new Error(`timed out waiting for ${what}`);
+    }
+    await new Promise((resolve) => setImmediate(resolve));
+  }
+}
+
+test("a piece left behind every reader is dropped AND the claim withdrawn", async () => {
+  const { store, gone, put, clean } = await storeWithGone();
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    // WHERE THE READERS STAND, which is the whole trigger: the encoder ran
+    // ahead, so pieces 0-1 are behind every one of them. This is the production
+    // path — `reviseSpillCeiling` asks `forgetBehind(readHeads)` — and it is
+    // what dropped 565 pieces in the field.
+    store.protectRange("reader", 2, 3, 0);
+    const revision = store.reviseSpillCeiling(null);
+
+    assert.ok(revision.behind >= 1, `something should have been dropped, got ${revision.behind}`);
+    assert.ok(gone.includes(0), `piece 0 has gone and should say so, got ${JSON.stringify(gone)}`);
+    assert.ok(
+      gone.every((index) => index < 2),
+      `nothing a reader still wants may be announced, got ${JSON.stringify(gone)}`
+    );
+  } finally {
+    await clean();
+  }
+});
+
+test("a piece dropped as a duplicate of a file held whole is NOT announced", async () => {
+  const { store, gone, put, clean } = await storeWithGone({
+    // Every piece can be had from the assembled file, which is exactly why the
+    // spilled copy is being dropped. Nothing has been lost, so nothing is said.
+    isPieceElsewhere: () => true
+  });
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    // The spill is what puts a piece on disk, and it finishes on its own time;
+    // dropping duplicates deliberately leaves a piece whose spill is still in
+    // flight alone, so the precondition is waited for rather than assumed.
+    await until(() => store.stats().spilled >= 1, "a piece to reach the disk");
+    const dropped = store.dropDuplicatesHeldElsewhere();
+    assert.ok(dropped >= 1, "the spilled duplicate should have been dropped");
+    assert.deepEqual(gone, [], "a piece still readable from a whole file has not gone");
+  } finally {
+    await clean();
+  }
+});
+
+test("a closing store announces nothing, because its torrent is going too", async () => {
+  const { store, gone, put, clean } = await storeWithGone();
+  try {
+    await put(0);
+    await put(1);
+    store.protectRange("reader", 2, 3, 0);
+    store.close(() => undefined);
+    store.reviseSpillCeiling(null);
+    assert.deepEqual(gone, [], "a claim withdrawn against a dying torrent reaches nothing useful");
+  } finally {
+    await clean();
+  }
+});
+
+test("the withdrawal is counted in the store's own figures", async () => {
+  const { store, put, clean } = await storeWithGone();
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    store.protectRange("reader", 2, 3, 0);
+    store.reviseSpillCeiling(null);
+    assert.ok(
+      store.stats().withdrawn >= 1,
+      "the figure that makes the eviction's bargain checkable must move"
+    );
+  } finally {
+    await clean();
+  }
+});
+
+test("a piece the library thinks it has is withdrawn", () => {
+  const asked = [];
+  const torrent = {
+    name: "film.mkv",
+    destroyed: false,
+    bitfield: { get: () => true },
+    _markUnverified: (index) => asked.push(index)
+  };
+  assert.equal(withdrawClaim({ index: 3, files: [{ _torrent: torrent }] }), "withdrawn");
+  assert.deepEqual(asked, [3]);
+});
+
+test("a piece the library already knows is missing is left alone", () => {
+  let touched = 0;
+  const torrent = {
+    destroyed: false,
+    bitfield: { get: () => false },
+    _markUnverified: () => { touched += 1; }
+  };
+  assert.equal(
+    withdrawClaim({ index: 7, torrent }),
+    "nothing-to-withdraw",
+    "re-creating a piece the library is already fetching would discard its blocks in flight"
+  );
+  assert.equal(touched, 0);
+});
+
+test("a destroyed torrent is left alone", () => {
+  let touched = 0;
+  const torrent = {
+    destroyed: true,
+    bitfield: { get: () => true },
+    _markUnverified: () => { touched += 1; }
+  };
+  assert.equal(withdrawClaim({ index: 1, torrent }), "no-torrent");
+  assert.equal(touched, 0);
+});
+
+test("a library that refuses says so instead of failing the eviction", () => {
+  const said = [];
+  const torrent = {
+    name: "film.mkv",
+    destroyed: false,
+    bitfield: { get: () => true },
+    _markUnverified: () => { throw new Error("no such method any more"); }
+  };
+  assert.equal(withdrawClaim({ index: 2, torrent, warn: (line) => said.push(line) }), "refused");
+  assert.equal(said.length, 1);
+  assert.match(said[0], /piece 2 of film\.mkv/);
+});
+
+test("a store with nobody listening evicts exactly as it did before", async () => {
+  const { store, put, clean } = await storeWithGone({ onPieceGone: undefined });
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    store.protectRange("reader", 2, 3, 0);
+    const revision = store.reviseSpillCeiling(null);
+    assert.ok(revision.behind >= 1, "the eviction does not depend on anybody listening");
+  } finally {
+    await clean();
+  }
+});

package/utils/logger.js

@@ -27,11 +27,18 @@ const PREFIX = "[proxy-client]";
 /**
  * When the file is rotated, and how many turns are kept.
  *
- * One turn holds a few hours of a busy session at the current rate; two turns
- * therefore cover a night's worth of restarts, which is the span a morning
- * report asks about. Bounded because the addon's `/data` is the owner's disk.
+ * **Why it is this large.** It was 32 MiB, and that erased the beginning of
+ * the very failure it was needed for. Field 2026-09-12: a session froze at
+ * 17:20 and printed one established fact about 55 times a second, so the file
+ * turned over twice before the session ended — 159 000 lines covering
+ * 17:51-18:29, then 76 385 covering 18:29-18:52. Sixty-one minutes was all
+ * that survived of ninety-two, and the second rotation overwrote the turn that
+ * held the onset. The disk it is bounded for had 91.4 GB free at the time.
+ *
+ * The repetition is a separate fault and is being fixed separately; a log that
+ * cannot hold a session either way is the one that has to go first.
  */
-const MAX_FILE_BYTES = 32 * 1024 * 1024;
+const MAX_FILE_BYTES = 1024 * 1024 * 1024;
 
 /** @type {import("node:fs").WriteStream | null} */
 let fileStream = null;
@@ -159,6 +166,122 @@ export const logger = {
   error: (message) => write("error", message, chalk.red, console.error)
 };
 
+/**
+ * Write a line that has ALREADY been through the repeat rule on another thread.
+ *
+ * The worker holds the same rule and applies it before forwarding, so running
+ * it again here would be one decision taken twice, on two different histories.
+ * It does not lose lines today — a re-printed line carries its held-back count,
+ * which makes the text unique — but that is an accident of the wording, and the
+ * file's own promise is that a line cannot reach the console and miss the file.
+ * It also filled this thread's bounded map with keys that can never repeat.
+ *
+ * @param {string} level
+ * @param {string} message - Already decided; written as it is.
+ * @returns {void}
+ */
+export function writeAlreadyDecided(level, message) {
+  const colour = level === "error"
+    ? chalk.red
+    : level === "warn" ? chalk.yellow : level === "success" ? chalk.green : chalk.cyan;
+  const toConsole = level === "error" ? console.error : level === "warn" ? console.warn : console.log;
+  const line = `${PREFIX} [${ts()}] ${message}`;
+  toConsole(colour(line));
+  toFile(line);
+}
+
+/**
+ * An established fact is said once, then with decreasing frequency.
+ *
+ * **Why.** A failure that establishes itself and does not change is printed by
+ * whatever loop meets it, at that loop's own rate. Field 2026-09-12: one
+ * absent piece produced 235 000 lines in 92 minutes — `Error opening input
+ * file …` 8514 times, `Error opening input files: End of file` 5712, the same
+ * `run-state` transition 2892, the same read failure 2884 — about 55 lines a
+ * second, and it turned the log over twice so the beginning of the failure was
+ * gone before anyone read it. A log is not vitiated by its size alone; it is
+ * vitiated by uniformity, and a bigger file does not fix that.
+ *
+ * **Matched VERBATIM — the whole line, no normalisation of numbers.** Measured
+ * on that log: exact repeats are 52 567 of 76 385 lines, 68.8 %, which is
+ * nearly all of the flood and carries no risk at all of merging two different
+ * statements. Normalising digits would catch a little more and would also merge
+ * the memory series — `rss=327MB`, `rss=726MB` — which exists precisely to
+ * catch a runaway, and suppressing it would be worse than the flood.
+ */
+const REPEAT_FIRST_MS = 1_000;
+const REPEAT_MAX_MS = 60_000;
+/**
+ * How many distinct lines are tracked. Bounded because it is keyed by the full
+ * text: a process that logs unique lines for ever must not grow a map of them.
+ */
+const REPEAT_KEYS = 512;
+/** @type {Map<string, { suppressed: number, printedAt: number, interval: number }>} */
+const recent = new Map();
+
+/**
+ * What to append when a line is said again after repeats were held back.
+ *
+ * One function because it was written twice, in the two branches that decide to
+ * speak — which is how two statements of one rule drift apart.
+ *
+ * @param {number} heldBack
+ * @param {number} overMs
+ * @returns {string}
+ */
+function heldBackSuffix(heldBack, overMs) {
+  return heldBack > 0
+    ? ` [said ${heldBack} more time(s) in the last ${(overMs / 1000).toFixed(1)}s]`
+    : "";
+}
+
+/**
+ * Whether this line is a repeat to hold back, and what to say if it is not.
+ *
+ * @param {string} message
+ * @returns {{ hold: true } | { hold: false, suffix: string }}
+ */
+function repeatCheck(message) {
+  const now = Date.now();
+  const seen = recent.get(message);
+  // Unseen, or not seen for longer than the longest interval — which makes it
+  // news again rather than a continuing fact.
+  if (!seen || now - seen.printedAt > REPEAT_MAX_MS) {
+    // WHAT WAS HELD BACK IS STILL SAID. A stale entry can carry repeats that
+    // were never reported — a line said just under its interval and then not
+    // again for a while — and dropping the count here would be the quiet lie
+    // this whole rule exists to avoid.
+    const heldBack = seen?.suppressed ?? 0;
+    const overMs = seen ? now - seen.printedAt : 0;
+    recent.delete(message);
+    if (recent.size >= REPEAT_KEYS) {
+      // The least recently printed goes: `Map` keeps insertion order and every
+      // print re-inserts, so the first key is the oldest.
+      const oldest = recent.keys().next();
+      if (!oldest.done) {
+        recent.delete(oldest.value);
+      }
+    }
+    recent.set(message, { suppressed: 0, printedAt: now, interval: REPEAT_FIRST_MS });
+    return { hold: false, suffix: heldBackSuffix(heldBack, overMs) };
+  }
+  if (now - seen.printedAt < seen.interval) {
+    seen.suppressed += 1;
+    return { hold: true };
+  }
+  const heldBack = seen.suppressed;
+  const overMs = now - seen.printedAt;
+  recent.delete(message);
+  recent.set(message, {
+    suppressed: 0,
+    printedAt: now,
+    interval: Math.min(REPEAT_MAX_MS, seen.interval * 2)
+  });
+  // SAID, not merely hidden: the rate is the fact here, and a log that quietly
+  // drops repeats reports a healthy proxy where a loop was spinning.
+  return { hold: false, suffix: heldBackSuffix(heldBack, overMs) };
+}
+
 /**
  * One path for every level, so a line cannot reach the console and miss the
  * file depending on which method was called or which thread called it.
@@ -170,15 +293,20 @@ export const logger = {
  * @returns {void}
  */
 function write(level, message, colour, toConsole) {
-  toConsole(colour(`${PREFIX} [${ts()}] ${message}`));
+  const repeat = repeatCheck(message);
+  if (repeat.hold) {
+    return;
+  }
+  const line = `${message}${repeat.suffix}`;
+  toConsole(colour(`${PREFIX} [${ts()}] ${line}`));
   if (forward) {
     try {
-      forward(level, message);
+      forward(level, line);
     } catch {
       // silent-ok: a thread whose channel has closed is shutting down, and a
       // failed log line must not be what ends it.
     }
     return;
   }
-  toFile(`${PREFIX} [${ts()}] ${message}`);
+  toFile(`${PREFIX} [${ts()}] ${line}`);
 }