@torrent-tv/proxy 2.83.3 → 2.83.5

package/test/upload-hurry.test.js

@@ -18,6 +18,40 @@
 import test from "node:test";
 import assert from "node:assert/strict";
 import { decideUploadLimit, torrentsForUploadPolicy } from "../services/torrent-pool.js";
+import { demandFor, forgetTorrent } from "../services/download/registry.js";
+import { Urgency } from "../services/demand/index.js";
+
+/**
+ * A torrent whose metadata has arrived and which nothing is wanted of.
+ *
+ * The files matter: a torrent with no list of files cannot be stated about at
+ * all, and is wanted for that reason alone — it is still being fetched.
+ *
+ * @param {string} name
+ * @returns {object}
+ */
+function quiet(name) {
+  return { name, files: [{ offset: 0, length: 1_000, name }] };
+}
+
+/**
+ * The same, with something stated for it — which is what "somebody wants this"
+ * means now that nothing counts readers.
+ *
+ * @param {string} name
+ * @returns {object}
+ */
+function wanted(name) {
+  const torrent = quiet(name);
+  demandFor(torrent).register.state({
+    claimant: "priority-map:0:0",
+    fileIndex: 0,
+    byteStart: 0,
+    byteEnd: 999,
+    urgency: Urgency.NEAR
+  });
+  return torrent;
+}
 
 const NOW = 1_000_000;
 const healthy = (extra = {}) => ({
@@ -65,34 +99,38 @@ test("the reciprocity boost still works when no hurry is on", () => {
   assert.match(decision.reason, /earn unchoke/);
 });
 
-test("a torrent with no reader still reaches the policy while it is in a hurry", () => {
+test("a torrent nothing is wanted of still reaches the policy while it is in a hurry", () => {
   // The moment that matters: a torrent has just been added and its head and
-  // tail are being fetched for the codec probe. That read goes straight to
-  // `createReadStream`, so no reader is registered — and the selection only
-  // ever kept torrents that had one, which is where the gap was.
-  const hurrying = { name: "film.mkv", hurryUntil: NOW + 20_000 };
-  const idle = { name: "other.mkv" };
-  const usage = new Map();
-
-  assert.deepEqual(
-    torrentsForUploadPolicy([hurrying, idle], usage, NOW),
-    [hurrying],
-    "a torrent with no reader yet was ignored"
-  );
+  // tail are being fetched for the codec probe. Nothing is stated for it yet,
+  // and the selection only ever kept torrents something was — which is where
+  // the gap was.
+  const hurrying = quiet("film.mkv");
+  hurrying.hurryUntil = NOW + 20_000;
+  const idle = quiet("other.mkv");
+  const watched = wanted("watched.mkv");
+  try {
+    assert.deepEqual(
+      torrentsForUploadPolicy([hurrying, idle], NOW),
+      [hurrying],
+      "a torrent nothing is stated for yet was ignored"
+    );
 
-  assert.deepEqual(
-    torrentsForUploadPolicy([{ ...hurrying, hurryUntil: NOW - 1 }, idle], usage, NOW),
-    [],
-    "and stops counting once the rush is over"
-  );
+    assert.deepEqual(
+      torrentsForUploadPolicy([{ ...hurrying, hurryUntil: NOW - 1 }, idle], NOW),
+      [],
+      "and stops counting once the rush is over"
+    );
 
-  const read = { name: "watched.mkv" };
-  usage.set(read, new Set([0]));
-  assert.deepEqual(
-    torrentsForUploadPolicy([read], usage, NOW),
-    [read],
-    "a torrent being read still counts, hurry or not"
-  );
+    assert.deepEqual(
+      torrentsForUploadPolicy([watched], NOW),
+      [watched],
+      "a torrent somebody wants something of still counts, hurry or not"
+    );
+  } finally {
+    forgetTorrent(hurrying);
+    forgetTorrent(idle);
+    forgetTorrent(watched);
+  }
 });
 
 test("a torrent nobody is reading is not treated as starving", () => {
@@ -102,7 +140,7 @@ test("a torrent nobody is reading is not treated as starving", () => {
   // minutes, every one of them reported as `earn unchoke ... down=0KB/s`.
   const idleButChoked = {
     name: "film.mkv",
-    hasActiveReader: false,
+    isWanted: false,
     wires: [
       { amInterested: true, peerChoking: true },
       { amInterested: true, peerChoking: true }
@@ -112,7 +150,7 @@ test("a torrent nobody is reading is not treated as starving", () => {
   };
   assert.equal(decideUploadLimit([idleButChoked], { now: NOW }).bytesPerSec, 50 * 1024);
 
-  const waiting = { ...idleButChoked, hasActiveReader: true };
+  const waiting = { ...idleButChoked, isWanted: true };
   assert.equal(
     decideUploadLimit([waiting], { now: NOW }).bytesPerSec,
     512 * 1024,
@@ -128,7 +166,7 @@ test("a torrent short of nothing is not worth uploading for", () => {
   // buying while somebody is still short of bytes.
   const complete = {
     name: "watched to the end",
-    hasActiveReader: true,
+    isWanted: true,
     hasUnmetDemand: false,
     done: false,
     downloadSpeed: 0,

package/test/withdraw-piece-claim.test.js

@@ -0,0 +1,212 @@
+/**
+ * @file One owner of the fact "this proxy has piece N".
+ *
+ * The store holds the bytes, so it owns the fact. The library keeps a second
+ * copy of it in its completion bitfield, and until 2026-09-12 nothing
+ * reconciled them: the disk tier dropped a piece once every reader was past it
+ * — correctly, that is what bounds the spill — and the bitfield went on saying
+ * the piece was verified. A read then concluded the piece was had, asked for it,
+ * was told it was absent, and failed; nothing fetched it again either, because
+ * the library does not download what it believes it owns. Field: a film played
+ * 80 seconds and then answered `Piece 0 is verified but absent from the store`
+ * for 92 minutes.
+ *
+ * Pinned here: the announcement and its one rule — said when the piece has gone
+ * from EVERYWHERE, never when one tier alone lost it — and the withdrawal.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import fs from "node:fs/promises";
+import { SharedPieceStore } from "../services/piece-store/shared-piece-store.js";
+import { withdrawClaim } from "../services/download/withdraw-claim.js";
+
+const PIECE = 1024;
+
+/**
+ * A store of four pieces with room for one, so every admission spills the last,
+ * plus the announcements it made.
+ */
+async function storeWithGone(extras = {}) {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "withdraw-test-"));
+  /** @type {number[]} */
+  const gone = [];
+  const store = new SharedPieceStore(PIECE, {
+    length: 4 * PIECE,
+    memoryBytes: PIECE,
+    path: directory,
+    name: "test",
+    files: [{ offset: 0, length: 4 * PIECE, name: "file.bin" }],
+    onPieceGone: ({ index }) => gone.push(index),
+    ...extras
+  });
+  const put = (index) => new Promise((resolve, reject) => {
+    store.put(index, Buffer.alloc(PIECE, index + 1), (error) => (error ? reject(error) : resolve()));
+  });
+  const clean = async () => {
+    store.destroy(() => undefined);
+    await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 20 });
+  };
+  return { store, gone, put, clean };
+}
+
+/**
+ * Wait for the CONDITION, with a deadline only as a backstop. A fixed pause
+ * here would measure the machine rather than the store.
+ *
+ * @param {() => boolean} ready
+ * @param {string} what
+ */
+async function until(ready, what) {
+  const deadline = Date.now() + 5_000;
+  while (!ready()) {
+    if (Date.now() > deadline) {
+      throw new Error(`timed out waiting for ${what}`);
+    }
+    await new Promise((resolve) => setImmediate(resolve));
+  }
+}
+
+test("a piece left behind every reader is dropped AND the claim withdrawn", async () => {
+  const { store, gone, put, clean } = await storeWithGone();
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    // WHERE THE READERS STAND, which is the whole trigger: the encoder ran
+    // ahead, so pieces 0-1 are behind every one of them. This is the production
+    // path — `reviseSpillCeiling` asks `forgetBehind(readHeads)` — and it is
+    // what dropped 565 pieces in the field.
+    store.protectRange("reader", 2, 3, 0);
+    const revision = store.reviseSpillCeiling(null);
+
+    assert.ok(revision.behind >= 1, `something should have been dropped, got ${revision.behind}`);
+    assert.ok(gone.includes(0), `piece 0 has gone and should say so, got ${JSON.stringify(gone)}`);
+    assert.ok(
+      gone.every((index) => index < 2),
+      `nothing a reader still wants may be announced, got ${JSON.stringify(gone)}`
+    );
+  } finally {
+    await clean();
+  }
+});
+
+test("a piece dropped as a duplicate of a file held whole is NOT announced", async () => {
+  const { store, gone, put, clean } = await storeWithGone({
+    // Every piece can be had from the assembled file, which is exactly why the
+    // spilled copy is being dropped. Nothing has been lost, so nothing is said.
+    isPieceElsewhere: () => true
+  });
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    // The spill is what puts a piece on disk, and it finishes on its own time;
+    // dropping duplicates deliberately leaves a piece whose spill is still in
+    // flight alone, so the precondition is waited for rather than assumed.
+    await until(() => store.stats().spilled >= 1, "a piece to reach the disk");
+    const dropped = store.dropDuplicatesHeldElsewhere();
+    assert.ok(dropped >= 1, "the spilled duplicate should have been dropped");
+    assert.deepEqual(gone, [], "a piece still readable from a whole file has not gone");
+  } finally {
+    await clean();
+  }
+});
+
+test("a closing store announces nothing, because its torrent is going too", async () => {
+  const { store, gone, put, clean } = await storeWithGone();
+  try {
+    await put(0);
+    await put(1);
+    store.protectRange("reader", 2, 3, 0);
+    store.close(() => undefined);
+    store.reviseSpillCeiling(null);
+    assert.deepEqual(gone, [], "a claim withdrawn against a dying torrent reaches nothing useful");
+  } finally {
+    await clean();
+  }
+});
+
+test("the withdrawal is counted in the store's own figures", async () => {
+  const { store, put, clean } = await storeWithGone();
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    store.protectRange("reader", 2, 3, 0);
+    store.reviseSpillCeiling(null);
+    assert.ok(
+      store.stats().withdrawn >= 1,
+      "the figure that makes the eviction's bargain checkable must move"
+    );
+  } finally {
+    await clean();
+  }
+});
+
+test("a piece the library thinks it has is withdrawn", () => {
+  const asked = [];
+  const torrent = {
+    name: "film.mkv",
+    destroyed: false,
+    bitfield: { get: () => true },
+    _markUnverified: (index) => asked.push(index)
+  };
+  assert.equal(withdrawClaim({ index: 3, files: [{ _torrent: torrent }] }), "withdrawn");
+  assert.deepEqual(asked, [3]);
+});
+
+test("a piece the library already knows is missing is left alone", () => {
+  let touched = 0;
+  const torrent = {
+    destroyed: false,
+    bitfield: { get: () => false },
+    _markUnverified: () => { touched += 1; }
+  };
+  assert.equal(
+    withdrawClaim({ index: 7, torrent }),
+    "nothing-to-withdraw",
+    "re-creating a piece the library is already fetching would discard its blocks in flight"
+  );
+  assert.equal(touched, 0);
+});
+
+test("a destroyed torrent is left alone", () => {
+  let touched = 0;
+  const torrent = {
+    destroyed: true,
+    bitfield: { get: () => true },
+    _markUnverified: () => { touched += 1; }
+  };
+  assert.equal(withdrawClaim({ index: 1, torrent }), "no-torrent");
+  assert.equal(touched, 0);
+});
+
+test("a library that refuses says so instead of failing the eviction", () => {
+  const said = [];
+  const torrent = {
+    name: "film.mkv",
+    destroyed: false,
+    bitfield: { get: () => true },
+    _markUnverified: () => { throw new Error("no such method any more"); }
+  };
+  assert.equal(withdrawClaim({ index: 2, torrent, warn: (line) => said.push(line) }), "refused");
+  assert.equal(said.length, 1);
+  assert.match(said[0], /piece 2 of film\.mkv/);
+});
+
+test("a store with nobody listening evicts exactly as it did before", async () => {
+  const { store, put, clean } = await storeWithGone({ onPieceGone: undefined });
+  try {
+    await put(0);
+    await put(1);
+    await put(2);
+    store.protectRange("reader", 2, 3, 0);
+    const revision = store.reviseSpillCeiling(null);
+    assert.ok(revision.behind >= 1, "the eviction does not depend on anybody listening");
+  } finally {
+    await clean();
+  }
+});

package/utils/logger.js

@@ -27,11 +27,18 @@ const PREFIX = "[proxy-client]";
 /**
  * When the file is rotated, and how many turns are kept.
  *
- * One turn holds a few hours of a busy session at the current rate; two turns
- * therefore cover a night's worth of restarts, which is the span a morning
- * report asks about. Bounded because the addon's `/data` is the owner's disk.
+ * **Why it is this large.** It was 32 MiB, and that erased the beginning of
+ * the very failure it was needed for. Field 2026-09-12: a session froze at
+ * 17:20 and printed one established fact about 55 times a second, so the file
+ * turned over twice before the session ended — 159 000 lines covering
+ * 17:51-18:29, then 76 385 covering 18:29-18:52. Sixty-one minutes was all
+ * that survived of ninety-two, and the second rotation overwrote the turn that
+ * held the onset. The disk it is bounded for had 91.4 GB free at the time.
+ *
+ * The repetition is a separate fault and is being fixed separately; a log that
+ * cannot hold a session either way is the one that has to go first.
  */
-const MAX_FILE_BYTES = 32 * 1024 * 1024;
+const MAX_FILE_BYTES = 1024 * 1024 * 1024;
 
 /** @type {import("node:fs").WriteStream | null} */
 let fileStream = null;
@@ -159,6 +166,92 @@ export const logger = {
   error: (message) => write("error", message, chalk.red, console.error)
 };
 
+/**
+ * An established fact is said once, then with decreasing frequency.
+ *
+ * **Why.** A failure that establishes itself and does not change is printed by
+ * whatever loop meets it, at that loop's own rate. Field 2026-09-12: one
+ * absent piece produced 235 000 lines in 92 minutes — `Error opening input
+ * file …` 8514 times, `Error opening input files: End of file` 5712, the same
+ * `run-state` transition 2892, the same read failure 2884 — about 55 lines a
+ * second, and it turned the log over twice so the beginning of the failure was
+ * gone before anyone read it. A log is not vitiated by its size alone; it is
+ * vitiated by uniformity, and a bigger file does not fix that.
+ *
+ * **Matched VERBATIM — the whole line, no normalisation of numbers.** Measured
+ * on that log: exact repeats are 52 567 of 76 385 lines, 68.8 %, which is
+ * nearly all of the flood and carries no risk at all of merging two different
+ * statements. Normalising digits would catch a little more and would also merge
+ * the memory series — `rss=327MB`, `rss=726MB` — which exists precisely to
+ * catch a runaway, and suppressing it would be worse than the flood.
+ */
+const REPEAT_FIRST_MS = 1_000;
+const REPEAT_MAX_MS = 60_000;
+/**
+ * How many distinct lines are tracked. Bounded because it is keyed by the full
+ * text: a process that logs unique lines for ever must not grow a map of them.
+ */
+const REPEAT_KEYS = 512;
+/** @type {Map<string, { suppressed: number, printedAt: number, interval: number }>} */
+const recent = new Map();
+
+/**
+ * Whether this line is a repeat to hold back, and what to say if it is not.
+ *
+ * @param {string} message
+ * @returns {{ hold: true } | { hold: false, suffix: string }}
+ */
+function repeatCheck(message) {
+  const now = Date.now();
+  const seen = recent.get(message);
+  // Unseen, or not seen for longer than the longest interval — which makes it
+  // news again rather than a continuing fact.
+  if (!seen || now - seen.printedAt > REPEAT_MAX_MS) {
+    // WHAT WAS HELD BACK IS STILL SAID. A stale entry can carry repeats that
+    // were never reported — a line said just under its interval and then not
+    // again for a while — and dropping the count here would be the quiet lie
+    // this whole rule exists to avoid.
+    const heldBack = seen?.suppressed ?? 0;
+    const overMs = seen ? now - seen.printedAt : 0;
+    recent.delete(message);
+    if (recent.size >= REPEAT_KEYS) {
+      // The least recently printed goes: `Map` keeps insertion order and every
+      // print re-inserts, so the first key is the oldest.
+      const oldest = recent.keys().next();
+      if (!oldest.done) {
+        recent.delete(oldest.value);
+      }
+    }
+    recent.set(message, { suppressed: 0, printedAt: now, interval: REPEAT_FIRST_MS });
+    return {
+      hold: false,
+      suffix: heldBack > 0
+        ? ` [said ${heldBack} more time(s) in the last ${(overMs / 1000).toFixed(1)}s]`
+        : ""
+    };
+  }
+  if (now - seen.printedAt < seen.interval) {
+    seen.suppressed += 1;
+    return { hold: true };
+  }
+  const heldBack = seen.suppressed;
+  const overMs = now - seen.printedAt;
+  recent.delete(message);
+  recent.set(message, {
+    suppressed: 0,
+    printedAt: now,
+    interval: Math.min(REPEAT_MAX_MS, seen.interval * 2)
+  });
+  return {
+    hold: false,
+    // SAID, not merely hidden: the rate is the fact here, and a log that quietly
+    // drops repeats reports a healthy proxy where a loop was spinning.
+    suffix: heldBack > 0
+      ? ` [said ${heldBack} more time(s) in the last ${(overMs / 1000).toFixed(1)}s]`
+      : ""
+  };
+}
+
 /**
  * One path for every level, so a line cannot reach the console and miss the
  * file depending on which method was called or which thread called it.
@@ -170,15 +263,20 @@ export const logger = {
  * @returns {void}
  */
 function write(level, message, colour, toConsole) {
-  toConsole(colour(`${PREFIX} [${ts()}] ${message}`));
+  const repeat = repeatCheck(message);
+  if (repeat.hold) {
+    return;
+  }
+  const line = `${message}${repeat.suffix}`;
+  toConsole(colour(`${PREFIX} [${ts()}] ${line}`));
   if (forward) {
     try {
-      forward(level, message);
+      forward(level, line);
     } catch {
       // silent-ok: a thread whose channel has closed is shutting down, and a
       // failed log line must not be what ends it.
     }
     return;
   }
-  toFile(`${PREFIX} [${ts()}] ${message}`);
+  toFile(`${PREFIX} [${ts()}] ${line}`);
 }

package/services/torrent-worker/file-claims.js

@@ -1,91 +0,0 @@
-/**
- * @file Who is holding which file open.
- *
- * A claim keeps a file's data from being removed while something is reading it.
- * Until 2.9.77 claims were keyed by `sourceKey:fileIndex`, which quietly made
- * them **shared**: a second reader of the same file found the key taken and
- * added nothing, so the first reader to finish released the claim out from
- * under the second. The proxy reads one file from several places at once —
- * ffmpeg's input, the keyframe index, the codec probe, a second viewer — so
- * this was the normal case, not an edge one.
- *
- * The fix is not a counter. A counter restores the arithmetic but keeps the
- * ambiguity: a release names a file, not a claim, so a duplicate or late
- * release still decrements someone else's hold and nothing can detect it. Here
- * every claim gets its own identity and a release names exactly that claim —
- * so a stray release matches nothing, is reported, and harms no one.
- */
-
-/**
- * @typedef {object} FileClaims
- * @property {(sourceKey: string, fileIndex: number, release: () => void) => string} open
- * @property {(claimId: string) => boolean} close
- * @property {() => void} closeAll
- * @property {number} size
- */
-
-/**
- * Track file claims by identity.
- *
- * @returns {FileClaims}
- */
-export function createFileClaims() {
-  /** Claim id → the function that releases that one claim. */
-  const claims = new Map();
-  let counter = 0;
-
-  return {
-    /**
-     * Record a new claim and return its identity.
-     *
-     * Each call is a distinct claim even for the same file — that is the whole
-     * point.
-     *
-     * @param {string} sourceKey
-     * @param {number} fileIndex
-     * @param {() => void} release
-     * @returns {string}
-     */
-    open(sourceKey, fileIndex, release) {
-      counter += 1;
-      // The file is in the id purely so a log line reads usefully; matching is
-      // on the whole string.
-      const claimId = `${sourceKey}:${fileIndex}:${counter}`;
-      claims.set(claimId, release);
-      return claimId;
-    },
-
-    /**
-     * Release one claim. Returns false when there was no such claim, which is
-     * worth logging: it means a release arrived twice, or after teardown.
-     *
-     * @param {string} claimId
-     * @returns {boolean}
-     */
-    close(claimId) {
-      const release = claims.get(claimId);
-      if (!release) {
-        return false;
-      }
-      claims.delete(claimId);
-      release();
-      return true;
-    },
-
-    /**
-     * Release everything — the worker is shutting down.
-     *
-     * @returns {void}
-     */
-    closeAll() {
-      for (const [, release] of claims) {
-        release();
-      }
-      claims.clear();
-    },
-
-    get size() {
-      return claims.size;
-    }
-  };
-}

package/test/file-claims.test.js

@@ -1,64 +0,0 @@
-/**
- * @file File claims must be held per reader, not per file.
- *
- * The proxy reads one file from several places at once — ffmpeg's input, the
- * keyframe index, the codec probe, a second viewer. Keying claims by file made
- * them shared, so the first reader to finish released the hold while the others
- * were still reading, and the file's data could then be removed under them.
- */
-
-import test from "node:test";
-import assert from "node:assert/strict";
-import { createFileClaims } from "../services/torrent-worker/file-claims.js";
-
-test("two readers of one file hold two claims", () => {
-  const claims = createFileClaims();
-  let released = 0;
-
-  const first = claims.open("source", 0, () => (released += 1));
-  const second = claims.open("source", 0, () => (released += 1));
-
-  assert.notEqual(first, second, "the second reader reused the first one's claim");
-  assert.equal(claims.size, 2);
-
-  claims.close(first);
-  assert.equal(released, 1, "closing one claim released more than one hold");
-  assert.equal(claims.size, 1, "the second reader's claim went with the first");
-
-  claims.close(second);
-  assert.equal(released, 2);
-  assert.equal(claims.size, 0);
-});
-
-test("a repeated release affects nothing and reports itself", () => {
-  const claims = createFileClaims();
-  let released = 0;
-  const claimId = claims.open("source", 3, () => (released += 1));
-
-  assert.equal(claims.close(claimId), true);
-  assert.equal(claims.close(claimId), false, "a second release was accepted as valid");
-  assert.equal(released, 1, "the hold was released twice");
-});
-
-test("a release naming nothing is rejected rather than guessed at", () => {
-  const claims = createFileClaims();
-  let released = 0;
-  claims.open("source", 0, () => (released += 1));
-
-  assert.equal(claims.close("source:0:999"), false);
-  assert.equal(released, 0, "an unknown claim released a real hold");
-  assert.equal(claims.size, 1);
-});
-
-test("teardown releases every outstanding claim", () => {
-  const claims = createFileClaims();
-  let released = 0;
-  claims.open("a", 0, () => (released += 1));
-  claims.open("a", 1, () => (released += 1));
-  claims.open("b", 0, () => (released += 1));
-
-  claims.closeAll();
-
-  assert.equal(released, 3);
-  assert.equal(claims.size, 0);
-});