@torrent-tv/proxy 2.64.7 → 2.64.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.64.7",
+  "version": "2.64.9",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "publishConfig": {

package/services/hls-session-manager.js

@@ -468,6 +468,10 @@ const BEHIND_HEAD_REPAIR_MS = 400;
 // Generous against that measurement, and far short of the hundreds of segments
 // a scan reaches.
 const BEHIND_HEAD_REPAIR_MAX_SEGMENTS = 60;
+// How far the accounting of a backward restart looks for work about to be done
+// twice. It runs on the restart path and a session an hour in has thousands of
+// segments; the figure is for a comparison, not an inventory.
+const BACKWARD_RESTART_SCAN_SEGMENTS = 300;
 // Hard cap on the total settle wait, measured from the first request of a
 // burst, so a still-moving scrubber cannot delay a genuine seek forever.
 const SEEK_SETTLE_MAX_MS = 1_000;
@@ -4699,6 +4703,10 @@ export class HlsSessionManager {
     // 0.54-1.47 s — does not account for it. Before rebuilding the hottest path
     // in the proxy on a guess, make each stage state its own cost.
     const restartEnteredAt = Date.now();
+    // Reads where the old run began BEFORE the new one overwrites it, and does
+    // not await: everything below is the restart path, which is measured in
+    // milliseconds and has been worked on twice to keep it that way.
+    this.#accountBackwardRestart(session, startIndex);
     // One directory per run. Two runs writing the same segment name at once
     // produce a file that is neither, which is the only reason a restart ever
     // had to wait for its predecessor to die.
@@ -9330,6 +9338,85 @@ export class HlsSessionManager {
     );
   }
 
+  /**
+   * What moving the encoder BACKWARDS costs, said out loud when it happens.
+   *
+   * Nothing already written is lost — every run keeps its own directory and
+   * {@link HlsSessionManager##findProducedFile} serves the union of all of them
+   * — so the price of a restart is not the files. It is two other things, and
+   * neither was ever counted:
+   *
+   *  - **work done twice.** The new run begins at the target and encodes
+   *    forward through segments the old run had already finished. ffmpeg cannot
+   *    know they exist, so it makes them again.
+   *  - **the viewer in front.** While the run walks back up to where it already
+   *    was, nothing new is being made ahead of them, and their cushion drains.
+   *
+   * Both are what decides whether a session should be allowed a SECOND
+   * concurrent run instead — roadmap item 64. That question cannot be answered
+   * from taste, and this is the reading it needs: how often it happens at all,
+   * how far back, and how much of the walk is a repeat.
+   *
+   * Nothing here is awaited by the caller. Everything below the call site is
+   * the restart path, which is measured in milliseconds and has been worked on
+   * twice to keep it that way; a reading that delays the thing it is reading
+   * about is not a reading. The figures that MUST be taken before the new run
+   * exists are taken synchronously, and only the file counting is left to run
+   * on its own — against the directories that existed at this instant, so what
+   * the new run is about to write cannot be counted as already there.
+   *
+   * @param {HlsSession} session
+   * @param {number} startIndex - Where the new run will begin.
+   * @returns {void}
+   */
+  #accountBackwardRestart(session, startIndex) {
+    const previousStart = session.encodeStartIndex;
+    if (!Number.isInteger(previousStart) || !Number.isInteger(startIndex) || startIndex >= previousStart) {
+      // A first run, or one moving forward. Neither costs anything here: a
+      // forward restart skips material it never made.
+      return;
+    }
+    const processed = Number(session.progress?.processedSeconds);
+    const head = Number.isFinite(processed)
+      ? Math.max(previousStart, this.#segmentIndexForTime(session, processed))
+      : previousStart;
+    // Bounded: a session an hour in has thousands of segments, and the count is
+    // for a comparison, not an inventory.
+    const last = Math.min(head, startIndex + BACKWARD_RESTART_SCAN_SEGMENTS);
+    const dirsBefore = this.#runDirs(session);
+    const accounting = session.backwardRestarts ?? { count: 0, segmentsBack: 0, worstBack: 0, remade: 0 };
+    accounting.count += 1;
+    accounting.segmentsBack += previousStart - startIndex;
+    accounting.worstBack = Math.max(accounting.worstBack, previousStart - startIndex);
+    session.backwardRestarts = accounting;
+
+    void (async () => {
+      let alreadyOnDisk = 0;
+      for (let index = startIndex; index <= last; index += 1) {
+        const fileName = session.segmentFormat.segmentFileName(index);
+        for (const dir of dirsBefore) {
+          try {
+            await access(path.join(dir, fileName));
+            alreadyOnDisk += 1;
+            break;
+          } catch {
+            // Not this run's; try an older one.
+          }
+        }
+      }
+      accounting.remade += alreadyOnDisk;
+      logger.info(
+        `transcode ${session.id} moving the encoder BACK from #${previousStart} to #${startIndex} ` +
+        `(head was #${head}): ${alreadyOnDisk} of the ${last - startIndex + 1} segment(s) it will walk through ` +
+        `are already on disk and will be made again, and nothing is produced ahead of #${head} until it gets ` +
+        `back there — ${accounting.count} backward restart(s) this session, worst ${accounting.worstBack} ` +
+        `segment(s) back, ${accounting.remade} segment(s) remade in total (roadmap 64)`
+      );
+    })().catch(() => {
+      // silent-ok: a reading that fails is not worth ending a restart over.
+    });
+  }
+
   /**
    * The directories runs have written into, newest first.
    *

package/services/memory-report.js

@@ -101,6 +101,98 @@ export async function readAvailableMemory() {
   return null;
 }
 
+/**
+ * Anonymous memory grouped by the SHAPE of the mappings holding it.
+ *
+ * The rollup says how much there is; this says what it looks like, and the
+ * three shapes it can take are three different diagnoses of the same number:
+ *
+ *  - **one growing `[heap]`** — the allocator's break-managed arena. Freed
+ *    blocks stay in it, and on musl there is no `malloc_trim` to ask for them
+ *    back. Nothing above the allocator is holding anything.
+ *  - **many large anonymous mappings** — one per big allocation, which is what
+ *    a 4 MiB piece buffer is. If their count tracks the pieces the store says
+ *    it holds, the memory is accounted for; if it keeps climbing while the
+ *    store's count does not, the buffers are being kept alive by somebody.
+ *  - **many medium ones** — the allocator's own per-thread arenas, taken and
+ *    not returned.
+ *
+ * The field failure of 2026-08-31 is 700 MB that is none of the JavaScript
+ * heaps, none of the piece store, and none of ffmpeg. Which of the three
+ * shapes it has decides what to change, and no reading so far can tell them
+ * apart (roadmap item 2, step 4).
+ *
+ * @param {string} text - The contents of `/proc/self/smaps`.
+ * @returns {{ heapBytes: number, largeBytes: number, largeCount: number,
+ *   largestBytes: number, smallBytes: number, smallCount: number,
+ *   fileBytes: number }}
+ */
+export function summariseMappings(text) {
+  const summary = {
+    heapBytes: 0,
+    largeBytes: 0,
+    largeCount: 0,
+    largestBytes: 0,
+    smallBytes: 0,
+    smallCount: 0,
+    fileBytes: 0
+  };
+  // A mapping is a header line followed by its fields; only `Rss` is wanted,
+  // because a mapping that is reserved and untouched costs no memory.
+  let pathName = null;
+  for (const line of String(text ?? "").split("\n")) {
+    const header = /^[0-9a-f]+-[0-9a-f]+ \S{4} [0-9a-f]+ \S+ \d+\s*(.*)$/.exec(line);
+    if (header) {
+      pathName = header[1].trim();
+      continue;
+    }
+    const rss = /^Rss:\s+(\d+)\s+kB$/.exec(line);
+    if (!rss || pathName === null) {
+      continue;
+    }
+    const bytes = Number(rss[1]) * 1024;
+    if (bytes === 0) {
+      continue;
+    }
+    if (pathName === "[heap]") {
+      summary.heapBytes += bytes;
+    } else if (pathName !== "" && !pathName.startsWith("[")) {
+      // Backed by a file: the executable, the libraries, anything mapped in.
+      // Counted so the anonymous figures can be checked against `rss`.
+      summary.fileBytes += bytes;
+    } else if (bytes >= LARGE_MAPPING_BYTES) {
+      summary.largeBytes += bytes;
+      summary.largeCount += 1;
+      summary.largestBytes = Math.max(summary.largestBytes, bytes);
+    } else {
+      summary.smallBytes += bytes;
+      summary.smallCount += 1;
+    }
+  }
+  return summary;
+}
+
+/**
+ * Where "large" begins. Two megabytes, so a 4 MiB piece buffer is always large
+ * and an allocator's ordinary arena is not.
+ */
+const LARGE_MAPPING_BYTES = 2 * 1024 * 1024;
+
+/**
+ * The mapping summary for this process, or null where /proc is not there.
+ *
+ * @returns {Promise<ReturnType<typeof summariseMappings> | null>}
+ */
+export async function readMappingSummary() {
+  try {
+    return summariseMappings(await readFile("/proc/self/smaps", "utf8"));
+  } catch {
+    // silent-ok: not Linux, or the kernel does not publish it. The line leaves
+    // the term out rather than printing a worse one.
+  }
+  return null;
+}
+
 /**
  * Available memory, falling back to what the runtime can offer.
  *
@@ -196,6 +288,7 @@ function megabytes(bytes) {
  * @param {number} [reading.availableBytes]
  * @param {boolean} [reading.availableMeasured]
  * @param {number | null} [reading.anonymousBytes]
+ * @param {ReturnType<typeof summariseMappings> | null} [reading.mappings]
  * @param {number | null} [reading.diskFreeBytes]
  * @param {{ name: string, residentBytes: number, committedBytes: number, spilledBytes: number, budgetBytes: number }[]} [reading.stores]
  * @returns {string}
@@ -207,6 +300,7 @@ export function describeMemory({
   availableBytes,
   availableMeasured,
   anonymousBytes = null,
+  mappings = null,
   diskFreeBytes = null,
   stores = []
 }) {
@@ -230,9 +324,16 @@ export function describeMemory({
   if (scope === "thread") {
     return `memory (${label || "thread"}): ${isolate}; ${storesPart}`;
   }
+  const shape = mappings === null
+    ? ""
+    : ` mappings=[heap ${megabytes(mappings.heapBytes)}, ` +
+      `${mappings.largeCount} anon ≥2MB = ${megabytes(mappings.largeBytes)} ` +
+      `(largest ${megabytes(mappings.largestBytes)}), ` +
+      `${mappings.smallCount} anon <2MB = ${megabytes(mappings.smallBytes)}, ` +
+      `files ${megabytes(mappings.fileBytes)}]`;
   return (
     `memory: rss=${megabytes(usage.rss)} ${isolate}` +
-    `${anonymousBytes === null ? "" : ` anon=${megabytes(anonymousBytes)}`}; ` +
+    `${anonymousBytes === null ? "" : ` anon=${megabytes(anonymousBytes)}`}${shape}; ` +
     `${storesPart}; ` +
     `machine has ${megabytes(availableBytes ?? 0)} available` +
     `${availableMeasured ? "" : " (estimated — /proc/meminfo could not be read)"}` +
@@ -397,6 +498,10 @@ export function startMemoryReport({
             availableBytes: bytes,
             availableMeasured: measured,
             anonymousBytes,
+            // Read only when the line is written: `smaps` is one entry per
+            // mapping and a busy process has thousands, which is a different
+            // cost from the rollup's single line.
+            mappings: await readMappingSummary(),
             diskFreeBytes,
             stores
           }));

package/services/piece-store/shared-piece-store.js

@@ -107,6 +107,38 @@ function availableMemorySync() {
   return os.freemem();
 }
 
+/**
+ * How many piece buffers this thread has let go of, and how many the collector
+ * has actually taken back.
+ *
+ * The one reading that separates the two explanations of the 700 MB nobody can
+ * account for (roadmap item 2, field 2026-08-31): if the two numbers track each
+ * other, this code is holding nothing and whatever grows is below us, in the
+ * allocator — on musl there is no `malloc_trim` and no way to ask. If the gap
+ * widens, a reference of ours outlives the piece, and then a heap snapshot can
+ * name the holder.
+ *
+ * What it does NOT prove: a `SharedArrayBuffer`'s memory is shared between the
+ * isolates, so this thread's handle going means only that THIS thread let go.
+ * The main thread counts its own (`torrent-worker/client.js`), and the pair is
+ * what answers the question.
+ */
+const released = { count: 0, collected: 0 };
+const collector = typeof FinalizationRegistry === "function"
+  ? new FinalizationRegistry(() => {
+    released.collected += 1;
+  })
+  : null;
+
+/**
+ * What the collector has taken back against what was let go.
+ *
+ * @returns {{ released: number, collected: number }}
+ */
+export function pieceBufferCollection() {
+  return { released: released.count, collected: released.collected };
+}
+
 const MIN_BUDGET_BYTES = 64 * 1024 * 1024;
 const MIN_RESIDENT_PIECES = 2;
 /**
@@ -501,7 +533,7 @@ export class SharedPieceStore {
         this.#counters.fromMemory += 1;
         return already;
       }
-      const target = new SharedArrayBuffer(this.#lengthOf(index));
+      const target = this.#watchForCollection(new SharedArrayBuffer(this.#lengthOf(index)));
       await this.#disk.read(index, Buffer.from(target));
       this.#registerPiece(index, target);
       this.#counters.fromDisk += 1;
@@ -512,6 +544,19 @@ export class SharedPieceStore {
     }
   }
 
+  /**
+   * Count this buffer as one this thread will have to let go of, and notice
+   * when the collector takes it. See {@link pieceBufferCollection}.
+   *
+   * @param {SharedArrayBuffer} buffer
+   * @returns {SharedArrayBuffer} The same buffer.
+   */
+  #watchForCollection(buffer) {
+    released.count += 1;
+    collector?.register(buffer, null);
+    return buffer;
+  }
+
   /**
    * A fresh buffer holding this piece's bytes.
    *
@@ -521,7 +566,7 @@ export class SharedPieceStore {
    */
   #copyIntoNewBuffer(index, bytes) {
     const length = this.#lengthOf(index);
-    const sab = new SharedArrayBuffer(length);
+    const sab = this.#watchForCollection(new SharedArrayBuffer(length));
     const view = Buffer.from(sab);
     if (bytes.copy) {
       bytes.copy(view, 0, 0, length);

package/services/torrent-worker/client.js

@@ -41,6 +41,37 @@ const WORKER_EXIT_GRACE_MS = 5_000;
  * queued behind that work, so reading a finished 10 MB file took 12-23 s where
  * handing it to the channel took 125 ms.
  */
+/**
+ * Piece buffers this thread has been handed, and how many the collector has
+ * taken back.
+ *
+ * A `SharedArrayBuffer`'s memory belongs to neither isolate: it lives until
+ * BOTH have let go. So the worker's own count answers only half the question,
+ * and this is the other half — 700 MB grew in the field on 2026-08-31 with the
+ * store's own memory falling, and the two candidates are "a reference of ours
+ * outlives the piece" and "the allocator keeps what we free". These two
+ * counters separate them (roadmap item 2).
+ *
+ * A fragment arrives as a fresh handle onto the same shared memory every time,
+ * so what is counted is handles, not pieces. That is the right quantity anyway:
+ * one handle retained keeps the whole piece alive.
+ */
+const fragmentBuffers = { seen: 0, collected: 0 };
+const fragmentCollector = typeof FinalizationRegistry === "function"
+  ? new FinalizationRegistry(() => {
+    fragmentBuffers.collected += 1;
+  })
+  : null;
+
+/**
+ * What the collector has taken back against what this thread was handed.
+ *
+ * @returns {{ seen: number, collected: number }}
+ */
+export function fragmentBufferCollection() {
+  return { seen: fragmentBuffers.seen, collected: fragmentBuffers.collected };
+}
+
 export class TorrentWorkerClient {
   #worker;
   /**
@@ -138,6 +169,8 @@ export class TorrentWorkerClient {
             );
           }
           this.#lastPieceByRead.set(message.id, message.pieceIndex);
+          fragmentBuffers.seen += 1;
+          fragmentCollector?.register(buffer, null);
           const view = new Uint8Array(buffer, message.offset, message.length);
 
           const reader = this.#fragmentReaders.get(message.id);