@torrent-tv/proxy 2.58.2 → 2.59.0

package/services/torrent-worker/worker.js

@@ -499,7 +499,12 @@ setInterval(() => {
     const reads = stats.fromMemory + stats.fromDisk;
     const fromMemoryShare = reads > 0 ? ((stats.fromMemory / reads) * 100).toFixed(1) : "—";
     log(
+      // In BYTES as well as in pieces. The count alone says nothing without the
+      // piece size, and the piece size differs per torrent: on the film the
+      // proxy was killed under, 2026-08-28, "63" meant 504 MB.
       `piece-store "${stats.name.slice(0, 40)}": resident=${stats.resident}/${stats.capacity} ` +
+      `(${Math.round((stats.residentBytes || 0) / 1048576)}MB of ` +
+      `${Math.round((stats.budgetBytes || 0) / 1048576)}MB allowed) ` +
       `pinned=${stats.pinned} spilled=${stats.spilled} reads=${reads} (${fromMemoryShare}% from memory) ` +
       `spills=${stats.spills} revivals=${stats.revivals}` +
       (stats.blockedByPins > 0 ? ` blocked-by-pins=${stats.blockedByPins}` : "")

package/services/usrsctp-state.js

@@ -0,0 +1,146 @@
+/**
+ * @file Read usrsctp's live association state, without a rebuild.
+ *
+ * Roadmap item 11. `node_datachannel.node` ships unstripped — it carries
+ * usrsctp's own local symbols, including `system_base_info` and
+ * `usrsctp_getsockopt` — so the association's real state (peer receive
+ * window, pending data, retransmission timeout, congestion window) can be
+ * read out of THIS live process with a short gdb attach. No source rebuild,
+ * no SCTP_DEBUG image, no waiting for a packet capture to be read by eye.
+ *
+ * The walk (hash the association table, call `usrsctp_getsockopt` with
+ * `SCTP_STATUS` through the running process) and the healthy baseline it was
+ * checked against are in
+ * `research/session-2026-08-27-28-freeze-onset-and-sessions.md`, section 1.
+ * The script itself is bundled at {@link SCTPSTATE_SCRIPT_PATH} rather than
+ * hand-placed on a host, so it ships with every release instead of surviving
+ * only as long as someone remembers to copy it back after a container is
+ * recreated.
+ *
+ * `gdb` attaching with ptrace pauses every thread of the process for the
+ * duration of the read — one `getsockopt` call, measured at a fraction of a
+ * second in the manual procedure this automates.
+ */
+
+import { spawn } from "node:child_process";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { shouldStartCapture, WITNESS_COOLDOWN_MS } from "./packet-witness.js";
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+
+/** The bundled gdb script that performs the usrsctp state walk. */
+export const SCTPSTATE_SCRIPT_PATH = path.join(HERE, "..", "assets", "diagnostics", "sctpstate.gdb");
+
+/** How long gdb may run before it is killed. Generous: this is a rare, one-shot read. */
+export const GDB_TIMEOUT_MS = 15_000;
+
+/**
+ * Create the reader.
+ *
+ * Single-flight and cooldown-gated the same way the packet witness is
+ * ({@link shouldStartCapture}) — a wedge that stays certain for minutes must
+ * not spawn a fresh gdb attach on every tick, and two readings ten seconds
+ * apart tell the same story a hundred readings would.
+ *
+ * @param {Object} options
+ * @param {(message: string) => void} options.log
+ * @param {typeof spawn} [options.spawnProcess] - Seam for tests.
+ * @param {number} [options.pid] - Defaults to this process's own pid.
+ * @param {string} [options.scriptPath]
+ * @param {number} [options.cooldownMs]
+ * @returns {{ maybeRead: (reasonText: string) => boolean }}
+ */
+export function createUsrsctpStateReader({
+  log,
+  spawnProcess = spawn,
+  pid = process.pid,
+  scriptPath = SCTPSTATE_SCRIPT_PATH,
+  cooldownMs = WITNESS_COOLDOWN_MS
+}) {
+  /** @type {{ running: boolean, lastStartedAt: number }} */
+  const state = { running: false, lastStartedAt: 0 };
+
+  /**
+   * Read the association state now, if the gating rules allow it.
+   *
+   * @param {string} reasonText - What declared the wedge, for the log line.
+   * @returns {boolean} True when a read actually started.
+   */
+  const maybeRead = (reasonText) => {
+    if (!shouldStartCapture({ ...state, cooldownMs })) {
+      return false;
+    }
+    state.running = true;
+    state.lastStartedAt = Date.now();
+    const startedAt = state.lastStartedAt;
+    void (async () => {
+      let out = "";
+      let err = "";
+      try {
+        await new Promise((resolve) => {
+          let settled = false;
+          let child;
+          const finish = () => {
+            if (settled) {
+              return;
+            }
+            settled = true;
+            clearTimeout(killer);
+            resolve();
+          };
+          try {
+            child = spawnProcess(
+              "gdb",
+              ["-q", "-batch", "-p", String(pid), "-x", scriptPath],
+              { stdio: ["ignore", "pipe", "pipe"] }
+            );
+          } catch (error) {
+            err = `could not start gdb: ${error?.message ?? error}`;
+            resolve();
+            return;
+          }
+          child.stdout?.on("data", (chunk) => {
+            out += chunk.toString();
+          });
+          child.stderr?.on("data", (chunk) => {
+            err += chunk.toString();
+          });
+          child.on("error", (error) => {
+            err += `${err ? " " : ""}gdb error: ${error?.message ?? error}`;
+            finish();
+          });
+          child.on("close", finish);
+          const killer = setTimeout(() => {
+            try {
+              child.kill("SIGKILL");
+            } catch {
+              // already gone
+            }
+          }, GDB_TIMEOUT_MS);
+          if (typeof killer.unref === "function") {
+            killer.unref();
+          }
+        });
+      } finally {
+        const lines = out.trim().length > 0 ? out.trim().split("\n") : [];
+        if (lines.length > 0) {
+          log(`usrsctp state (${reasonText}): ${lines.join(" | ")}`);
+        } else {
+          log(`usrsctp state (${reasonText}): no reading — ${err.trim() || "gdb produced no output"}`);
+        }
+        state.running = false;
+        // Same spacing rule as the packet witness: honour the cooldown even
+        // when the read ended quickly.
+        const earliestNext = startedAt + cooldownMs;
+        if (state.lastStartedAt < earliestNext) {
+          state.lastStartedAt = earliestNext;
+        }
+      }
+    })();
+    return true;
+  };
+
+  return { maybeRead };
+}

package/test/memory-budget.test.js

@@ -0,0 +1,69 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { describeMemory } from "../services/memory-report.js";
+import {
+  budgetForNewStore,
+  totalStoreBudgetBytes
+} from "../services/piece-store/shared-piece-store.js";
+
+const MEGABYTE = 1024 * 1024;
+const GIGABYTE = 1024 * MEGABYTE;
+
+test("the whole of the torrent stores is bounded, not each one of them", () => {
+  // The failure this replaces: the budget was per torrent, so two torrents took
+  // two of it. Whatever the machine has, the total is one budget.
+  const available = 8 * GIGABYTE;
+  const alone = budgetForNewStore(available, 1);
+  const withThree = budgetForNewStore(available, 3);
+  assert.equal(alone, totalStoreBudgetBytes(available));
+  assert.ok(withThree < alone, "a third store must not be given a first store's share");
+  assert.ok(withThree * 3 <= totalStoreBudgetBytes(available) + 3);
+});
+
+test("the budget is a share of what the machine can give, capped", () => {
+  // A quarter of two gigabytes is under the ceiling and is what is taken.
+  assert.equal(totalStoreBudgetBytes(2 * GIGABYTE), 512 * MEGABYTE);
+  // A quarter of one gigabyte is 256 MB — below the ceiling, so not capped.
+  assert.equal(totalStoreBudgetBytes(GIGABYTE), 256 * MEGABYTE);
+  // And it never exceeds the ceiling however much is free.
+  assert.equal(totalStoreBudgetBytes(64 * GIGABYTE), 512 * MEGABYTE);
+});
+
+test("a machine with almost nothing left still gets a workable floor", () => {
+  // Refusing to serve is worse than exceeding the share, and the memory line
+  // says plainly what is held either way.
+  const tiny = budgetForNewStore(32 * MEGABYTE, 4);
+  assert.equal(tiny, 64 * MEGABYTE);
+});
+
+test("the memory line says bytes, and names what it could not measure", () => {
+  const usage = {
+    rss: 2422628 * 1024,
+    heapUsed: 180 * MEGABYTE,
+    heapTotal: 220 * MEGABYTE,
+    external: 900 * MEGABYTE,
+    arrayBuffers: 850 * MEGABYTE
+  };
+  const measured = describeMemory({
+    process: usage,
+    availableBytes: 1900 * MEGABYTE,
+    availableMeasured: true,
+    stores: [
+      { name: "a film", residentBytes: 504 * MEGABYTE, budgetBytes: 512 * MEGABYTE }
+    ]
+  });
+  // The figures the kernel's own kill line quoted, so the two can be compared.
+  assert.match(measured, /rss=2366MB/);
+  assert.match(measured, /1 torrent store\(s\) holding 504MB of 512MB allowed/);
+  assert.match(measured, /machine has 1900MB available$/);
+
+  const estimated = describeMemory({
+    process: usage,
+    availableBytes: 1900 * MEGABYTE,
+    availableMeasured: false,
+    stores: []
+  });
+  assert.match(estimated, /no torrent stores/);
+  assert.match(estimated, /estimated — \/proc\/meminfo could not be read/);
+});

package/test/probe-wedge-certainty.test.js

@@ -0,0 +1,60 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { probeWedgeIsCertain, readProbeState, PROBE_INTERVAL_MS } from "../services/delivery-probe.js";
+
+test("a seen-counter bounded lag is not a wedge, however long it lasts", () => {
+  // Session 4dcac61b, field log 2026-08-28: gap held at 6-7 probes for 95+
+  // seconds on a backgrounded tab, but `seen` kept climbing right along with
+  // `sent` — this connection's own history says gaps up to ~3.5 s (7 probes
+  // at 500 ms) are ordinary, so the same 3.5 s stuck must not read as certain.
+  const verdict = probeWedgeIsCertain({
+    stuckForMs: 3400,
+    longestHealthySeenGapMs: 3500
+  });
+  assert.equal(verdict.certain, false);
+});
+
+test("a seen-counter frozen past this connection's own worst legitimate gap is a wedge", () => {
+  // Session d85ae4f5, the same field log: `seen` frozen at one value for over
+  // a minute while `sent` climbed unbounded — this is the shape the detector
+  // exists to catch.
+  const verdict = probeWedgeIsCertain({
+    stuckForMs: 90_000,
+    longestHealthySeenGapMs: 3500
+  });
+  assert.equal(verdict.certain, true);
+});
+
+test("with no healthy history yet, one probe interval is still required", () => {
+  const verdict = probeWedgeIsCertain({
+    stuckForMs: PROBE_INTERVAL_MS - 1,
+    longestHealthySeenGapMs: 0
+  });
+  assert.equal(verdict.certain, false);
+  assert.equal(verdict.needMs, PROBE_INTERVAL_MS);
+});
+
+test("readProbeState calls association-stopped only when the unreliable channel agrees", () => {
+  // Ordered channels behind, but the unordered/no-retransmit one current:
+  // head-of-line blocking in one stream, not the association.
+  const streamStuck = readProbeState({
+    seq: 100,
+    seen: { proxy: 90, "proxy-control": 90, "proxy-fast": 99 },
+    labels: ["proxy", "proxy-control", "proxy-fast"],
+    echoes: 5,
+    echoAgeMs: 100,
+    allowed: { proxy: 2, "proxy-control": 2, "proxy-fast": 2 }
+  });
+  assert.equal(streamStuck.verdict, "stream-stuck");
+
+  const associationStopped = readProbeState({
+    seq: 100,
+    seen: { proxy: 90, "proxy-control": 90, "proxy-fast": 90 },
+    labels: ["proxy", "proxy-control", "proxy-fast"],
+    echoes: 5,
+    echoAgeMs: 100,
+    allowed: { proxy: 2, "proxy-control": 2, "proxy-fast": 2 }
+  });
+  assert.equal(associationStopped.verdict, "association-stopped");
+});

package/test/usrsctp-state.test.js

@@ -0,0 +1,115 @@
+/**
+ * @file The usrsctp state reader's gating and command construction.
+ *
+ * Roadmap item 11: reads usrsctp's live association state via gdb the moment
+ * a wedge is declared. Everything here is the part that decides WHEN and WITH
+ * WHAT ARGUMENTS — the spawning itself is thin glue around these, the same
+ * shape as the packet witness (test/packet-witness.test.js).
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+import { EventEmitter } from "node:events";
+
+import { createUsrsctpStateReader, SCTPSTATE_SCRIPT_PATH } from "../services/usrsctp-state.js";
+
+class FakeChild extends EventEmitter {
+  constructor(command, args) {
+    super();
+    this.command = command;
+    this.args = args;
+    this.stdout = new EventEmitter();
+    this.stderr = new EventEmitter();
+  }
+
+  kill() {
+    // Not exercised by these tests — every fake run finishes on its own.
+    return true;
+  }
+}
+
+/**
+ * @param {string} output - What the fake gdb writes to stdout before closing.
+ * @returns {{ spawnProcess: Function, calls: Array<{ command: string, args: string[] }> }}
+ */
+function makeSpawn(output) {
+  const calls = [];
+  const spawnProcess = (command, args) => {
+    calls.push({ command, args });
+    const child = new FakeChild(command, args);
+    setImmediate(() => {
+      if (output) {
+        child.stdout.emit("data", Buffer.from(output));
+      }
+      child.emit("close", 0);
+    });
+    return child;
+  };
+  return { spawnProcess, calls };
+}
+
+/** Wait until `check` holds, rather than for a chosen interval. */
+async function waitFor(check) {
+  const deadline = Date.now() + 5_000;
+  for (;;) {
+    if (check()) {
+      return;
+    }
+    if (Date.now() > deadline) {
+      throw new Error("timed out waiting for the reading to be logged");
+    }
+    await new Promise((resolve) => setTimeout(resolve, 5));
+  }
+}
+
+test("gdb is invoked attached to this process with the bundled script", async () => {
+  const { spawnProcess, calls } = makeSpawn("state=8 rwnd=95890\n");
+  const lines = [];
+  const reader = createUsrsctpStateReader({
+    log: (message) => lines.push(message),
+    spawnProcess,
+    pid: 4242
+  });
+  const started = reader.maybeRead("test wedge");
+  assert.equal(started, true);
+  await waitFor(() => lines.length > 0);
+  assert.equal(calls.length, 1);
+  assert.deepEqual(calls[0].args, ["-q", "-batch", "-p", "4242", "-x", SCTPSTATE_SCRIPT_PATH]);
+  assert.match(lines[0], /state=8 rwnd=95890/);
+  assert.match(lines[0], /test wedge/);
+});
+
+test("no output is reported as no reading, not silence", async () => {
+  const { spawnProcess } = makeSpawn("");
+  const lines = [];
+  const reader = createUsrsctpStateReader({ log: (message) => lines.push(message), spawnProcess, pid: 1 });
+  reader.maybeRead("empty case");
+  await waitFor(() => lines.length > 0);
+  assert.match(lines[0], /no reading/);
+});
+
+test("a second read is refused within the cooldown, like the packet witness", async () => {
+  const { spawnProcess, calls } = makeSpawn("state=8\n");
+  const lines = [];
+  const reader = createUsrsctpStateReader({
+    log: (message) => lines.push(message),
+    spawnProcess,
+    pid: 1,
+    cooldownMs: 60_000
+  });
+  assert.equal(reader.maybeRead("first"), true);
+  await waitFor(() => lines.length > 0);
+  assert.equal(reader.maybeRead("second, too soon"), false);
+  assert.equal(calls.length, 1);
+});
+
+test("a missing gdb is reported, not thrown", async () => {
+  const lines = [];
+  const spawnProcess = () => {
+    throw new Error("spawn gdb ENOENT");
+  };
+  const reader = createUsrsctpStateReader({ log: (message) => lines.push(message), spawnProcess, pid: 1 });
+  reader.maybeRead("no gdb on this host");
+  await waitFor(() => lines.length > 0);
+  assert.match(lines[0], /could not start gdb/);
+});