@torrent-tv/proxy 2.100.0 → 2.100.2

package/bin/cli.js

@@ -89,6 +89,7 @@ program
   .option("--no-port-mapping", "Disable automatic UPnP/NAT-PMP port mapping")
   .option("--delivery-sink", "Serve /api/delivery-sink, a torrent-free byte stream for delivery testing")
   .option("--pin-preset <name>", "Encode every software picture at this libx264 preset only, for runs that must be compared at one quality (off by default)")
+  .option("--heap-snapshots", "Write a heap snapshot of a thread whose memory reaches a new high-water; each one stops that thread while it is written (off by default)")
   .option("--profile-start <seconds>", "Profile the CPU of the main thread and the torrent thread for this many seconds from the first source opened, written beside the state (off by default)")
   .option(
     "--send-chunk-bytes <bytes>",
@@ -395,6 +396,7 @@ try {
     logFile: options.logFile ?? "",
     deliverySink: options.deliverySink === true,
     profileStartSeconds: Number(options.profileStart) || 0,
+    heapSnapshots: options.heapSnapshots === true,
     pinnedPreset: typeof options.pinPreset === "string" && options.pinPreset.trim() ? options.pinPreset.trim() : null,
     // Late-bound the same way `webRtcManager` is below: the torrent pool is
     // built inside `startProxyServer`, before `dataChannelHandler` — the
@@ -524,6 +526,7 @@ try {
     // 2026-09-07 and every one was gone by the time the process that wrote them
     // died of a heap it had never been possible to look at.
     snapshotDir: options.stateDir || undefined,
+    heapSnapshots: options.heapSnapshots === true,
     snapshotFloorBytes: 400 * 1024 * 1024,
     snapshotGrowthBytes: 400 * 1024 * 1024,
     keepSnapshots: 3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.100.0",
+  "version": "2.100.2",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -127,6 +127,8 @@ function buildPortCandidates(startPort, maxAttempts = 51) {
 export async function startProxyServer({
   host, port, transcodeAudio, ffmpegBin, memoryBytes, segmentFormat, stateDir, onSubtitleCues, diagnostics, budgetPolicy,
   deliverySink = false,
+  // Whether a memory high-water earns a heap snapshot (`--heap-snapshots`).
+  heapSnapshots = false,
   // Seconds of main-thread CPU profile from the first source opened; 0 is off.
   profileStartSeconds = 0,
   // The one software preset every picture is encoded at, for runs compared at
@@ -178,6 +180,7 @@ export async function startProxyServer({
   const torrentPool = new WorkerTorrentPool({
     memoryBytes,
     stateDir,
+    heapSnapshots,
     onHoldingsChanged: () => onStateChanged(),
     onSourceForgotten: ({ sourceKey }) => {
       sourcePreparation?.forget(sourceKey);
@@ -889,6 +892,7 @@ export async function startProxyServer({
         const params = await containerOver({ sourceKey: output.file.sourceKey, fileIndex });
         if (!params) return { kind: "needs-source" };
         if (timing) {
+          params.walkAccount = timing.walk ??= { lockMs: 0, reads: 0, bytes: 0 };
           params.onTimed = (_statement, waitedMs, readMs) => {
             timing.reads += 1;
             timing.queuedMs += waitedMs;
@@ -942,6 +946,41 @@ export async function startProxyServer({
       }
       return { kind: "result", sources };
     },
+    // THE SOUND'S PACKETS FROM THE PICTURE'S COPY. A Matroska soundtrack is
+    // fed to its encoder packet by packet, and finding its packets walks the
+    // clusters the Cues name for the piece — picture and sound together,
+    // about 20 MB for 0.3 MB of sound on the field's 1080p film. The picture's
+    // input copies those same clusters through the torrent thread first, and
+    // walked a second time there each soundtrack piece took 6-11 s, slower
+    // than the film plays (r61, torrent-tv/meta#178). Walked from the copy
+    // while it is held, the sound reads only its own packets later.
+    stretchCopied: (output, from, to, input) => {
+      for (const source of input.sources ?? []) {
+        const container = containerOrchestrator.known(source.sourceKey, source.fileIndex);
+        const tracks = containerOrchestrator.tracks.get(`${source.sourceKey}:${source.fileIndex}`);
+        if (typeof container?.learnPackets !== "function" || !Array.isArray(tracks)) continue;
+        const carried = new Set((source.input.selections ?? []).map(selection => selection.track?.trackNumber));
+        const sounds = new Set();
+        for (const other of outputParts.outputs.values()) {
+          const sound = other.spec?.audio;
+          if (other.spec?.video || !sound || other.file?.sourceKey !== source.sourceKey || sound.fileIndex !== source.fileIndex) continue;
+          const track = tracks.filter(candidate => candidate.type === "audio")[sound.trackIndex];
+          if (track && !carried.has(track.trackNumber)) sounds.add(track.trackNumber);
+        }
+        if (!sounds.size || !Number.isFinite(source.input.from) || !(source.input.to > source.input.from)) continue;
+        const read = (start, end) => {
+          const answer = input.read(source.fileIndex, start, end);
+          return answer && typeof answer.then !== "function" && answer.end === end ? answer.bytes : null;
+        };
+        const startedAt = Date.now();
+        void container.learnPackets({ from: source.input.from, to: source.input.to, trackIds: [...sounds] }, read).then(learned =>
+          logger.info(`encode input #${from}..#${to} of ${output.outputKey}: soundtrack packets of track(s) ${[...sounds].join(",")} ` +
+            `over ${source.input.from.toFixed(3)}-${source.input.to.toFixed(3)}s ` +
+            (learned === "walked" ? "walked" : `not all walked (${learned === "memory" ? "memory refused" : "the copy does not hold the rest"})`) +
+            ` from its copy in ${Date.now() - startedAt}ms`),
+        error => logger.warn(`encode input #${from}..#${to} of ${output.outputKey}: walking the soundtrack from its copy failed: ${error?.message ?? error}`));
+      }
+    },
     readEncodeRanges: async (source, ranges, maxBytes) => {
       const torrent = torrentPool.knownTorrent(source.sourceKey);
       return torrent ? torrentPool.readHeldRangesOf(torrent, source.fileIndex, ranges, maxBytes) : null;

package/services/encode/EncodeInputs.js

@@ -55,9 +55,17 @@ export class EncodeInputs {
   #rankAt;
   #log;
   #now;
+  /**
+   * Told when an original input's stretch is copied whole, while its bytes are
+   * held, so what else they answer can be read from them (`learnPackets`).
+   *
+   * @type {(output: object, from: number, to: number, input: object) => void}
+   */
+  #stretchCopied;
 
   constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
-    rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now }) {
+    rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {} }) {
+    this.#stretchCopied = stretchCopied;
     this.#openOf = openOf;
     this.#resolve = resolve;
     this.#read = readRanges;
@@ -583,6 +591,10 @@ export class EncodeInputs {
       const copied = result.whole ?? Promise.resolve(true);
       void copied.then(whole => {
         if (whole && result.copyMs > 0) keep(this.#copyRates, result.bytes / result.copyMs);
+        if (whole) {
+          try { this.#stretchCopied(output, from, end, result); }
+          catch (error) { this.#log(`encode input #${from}..#${end} of ${addressOf(output)}: what was told of its copy failed: ${error?.message ?? error}`); }
+        }
         this.#log(`encode input #${from}..#${end} of ${addressOf(output)}: ` + (whole
           ? `stretch copied, ${result.bytes} bytes in ${result.copyMs}ms of copying since it was admitted`
           : result.copyFailed?.() ? "the stretch could not be copied: a piece it needs is no longer held"
@@ -636,6 +648,9 @@ function describeTiming(timing, now) {
     `${timing.resolutions} resolution(s) ${timing.resolveMs}ms ` +
     `(${timing.reads} file read(s): queued ${timing.queuedMs}ms, reading ${timing.readMs}ms), ` +
     `held ranges ${timing.heldMs}ms, budget ${timing.budgetMs}ms` +
+    // Where a packet input's walk spent its read: waiting for the same
+    // track's earlier walks, or reading the file (r62, torrent-tv/meta#178).
+    (timing.walk ? `; packet walk waited ${timing.walk.lockMs}ms for its track and read ${timing.walk.reads} time(s), ${timing.walk.bytes} bytes of the file` : "") +
     (timing.openApproximate ? "; sized by the host's startup open, this output has not opened yet" : "");
 }
 

package/services/media/ContainerOrchestrator.js

@@ -215,7 +215,7 @@ export class ContainerOrchestrator {
         value = await container.readSourceRanges(params.packetInterval);
         if (value.kind !== "result") return { ...value, requestId };
       } else if (statement === "packets") {
-        value = await container.readPacketIndex(params.packetInterval, { isCurrent: params.isCurrent });
+        value = await container.readPacketIndex(params.packetInterval, { isCurrent: params.isCurrent, account: params.walkAccount });
         if (value === null) return { kind: "terminal", reason: "request-obsolete", requestId };
         await value?.prepareAudioDependencies?.(params.packetInterval, container.readRange);
       } else {

package/services/media/container/MatroskaContainer.js

@@ -47,7 +47,8 @@ import { firstFrameOffset, readBlockHeader, walkHeldClusters } from "./matroska-
 import { buildMatroskaIndex, matroskaIntervalCovered, walkMatroskaPackets } from "./matroska-packets.js";
 import { PacketRecordsChain } from "./PacketRecordsChain.js";
 import { RetainedReads } from "./RetainedReads.js";
-import { isUnavailable } from "./unavailable.js";
+import { isUnavailable, strictReader } from "./unavailable.js";
+import { IndexMemoryUnavailable } from "./memory-unavailable.js";
 import {
   ID_ATTACHED_FILE, ID_ATTACHMENTS, ID_CHAPTERS, ID_FILE_DATA, ID_FILE_MEDIA_TYPE, ID_FILE_NAME, ID_INFO_TITLE, ID_TAGS,
   chapterTitlesOf, coverIndexOf, trackRatesFromTags, uidKeyOf, workFromTags
@@ -784,10 +785,14 @@ export class MatroskaContainer extends Container {
 
   /**
    * @param {object} [interval]
-   * @param {{ isCurrent?: () => boolean }} [options] - A walk whose request is
-   *   no longer current stops between clusters and answers null.
+   * @param {{ isCurrent?: () => boolean, readRange?: (start: number, end: number) => Promise<Buffer | null> | Buffer | null }} [options]
+   *   A walk whose request is no longer current stops between clusters and
+   *   answers null. `readRange`, when given, is where this one walk reads
+   *   instead of the file's own reader; what it does not hold is unavailable.
+   *   `account`, when given, is told how long each track's walk waited for
+   *   that track's earlier walks and what it read (`lockMs`, `reads`, `bytes`).
    */
-  async readPacketIndex(interval, { isCurrent } = {}) {
+  async readPacketIndex(interval, { isCurrent, readRange, account } = {}) {
     if (this.#packets) return this.#packets;
     const layout = await this.#segmentLayout();
     if (!layout) throw new Error("Matroska Segment is absent.");
@@ -806,7 +811,8 @@ export class MatroskaContainer extends Container {
     const complete = new Set();
     let whole = true;
     for (const track of requested) {
-      const chain = await this.#readTrackPackets(layout, tracks, track, start, interval, isCurrent);
+      const chain = await this.#readTrackPackets(layout, tracks, track, start, interval, isCurrent,
+        readRange ? strictReader(readRange, this.fileSize) : this.readRange, account);
       if (!chain) return null;
       streams.set(track.trackNumber, chain.records);
       if (chain.reachesEnd) complete.add(track.trackNumber);
@@ -818,6 +824,34 @@ export class MatroskaContainer extends Container {
     return index;
   }
 
+  /**
+   * Record these tracks' packets over `interval` from bytes another reader
+   * already holds, reading nothing of the file itself.
+   *
+   * A soundtrack's piece is walked over the clusters the Cues name for its
+   * interval, the picture's bytes with it: about 20 MB for 0.3 MB of sound on
+   * the field's 1080p film, read through the torrent thread a second time
+   * after the picture's input had copied the same clusters. Walked from that
+   * copy while it is held, the sound's input later reads only its own packets
+   * (r61, torrent-tv/meta#178).
+   *
+   * @param {{ from: number, to: number, trackIds: number[] }} interval
+   * @param {(start: number, end: number) => Buffer | null} readRange - Null where it does not hold the bytes.
+   * @returns {Promise<"walked" | "missing" | "memory">} Whether the
+   *   interval's packets are recorded, or what stopped the walk: bytes the
+   *   copy does not hold, or memory refused.
+   */
+  async learnPackets(interval, readRange) {
+    try {
+      return await this.readPacketIndex(interval, { readRange }) !== null ? "walked" : "missing";
+    } catch (error) {
+      // The sound's own walk reads the rest later, from where this one stopped.
+      if (isUnavailable(error)) return "missing";
+      if (error instanceof IndexMemoryUnavailable) return "memory";
+      throw error;
+    }
+  }
+
   /**
    * One track's packets from `start` on, read as far as the interval needs.
    *
@@ -831,13 +865,21 @@ export class MatroskaContainer extends Container {
    *
    * @returns {Promise<{ records: PacketRecordsChain, startAt: number, reachesEnd: boolean }>}
    */
-  async #readTrackPackets(layout, tracks, track, start, interval, isCurrent) {
+  async #readTrackPackets(layout, tracks, track, start, interval, isCurrent, reader, account = null) {
     const id = track.trackNumber;
     const previous = this.#packetWalks.get(id) ?? Promise.resolve();
     let finished;
     const walk = previous.then(() => new Promise(resolve => { finished = resolve; }));
     this.#packetWalks.set(id, walk);
+    const waitedFrom = Date.now();
     await previous;
+    const read = account ? async (start, end) => {
+      const bytes = await reader(start, end);
+      account.reads += 1;
+      account.bytes += bytes.length;
+      return bytes;
+    } : reader;
+    if (account) account.lockMs += Date.now() - waitedFrom;
     try {
       const readings = this.#packetReadings.get(id) ?? [];
       this.#packetReadings.set(id, readings);
@@ -886,7 +928,7 @@ export class MatroskaContainer extends Container {
         current = chainOf().at(-1);
         if (current.reachedEnd) break;
         const following = readings.find(reading => reading.startAt > current.startAt);
-        const outcome = await walkMatroskaPackets({ readRange: this.readRange, fileSize: this.fileSize,
+        const outcome = await walkMatroskaPackets({ readRange: read, fileSize: this.fileSize,
           portionBytes: this.portionBytes, layout, tracks, trackIds: [id], interval, packetMemory: this.packetMemory,
           state: current, until: following?.startAt ?? null, isCovered: covered, readsThrough, isCurrent });
         if (outcome === "obsolete") return null;

package/services/server/wire-outputs.js

@@ -226,6 +226,7 @@ export function wireOutputs({
     readMetadataActivity = null,
     sourceInputsFor = null,
     readEncodeRanges = null,
+    stretchCopied = () => {},
     readEncodeHeldRanges = null,
     encodeInputs = null,
     indexMemory = null,
@@ -1069,6 +1070,7 @@ export function wireOutputs({
     resolve: resolveEncodeInput,
     readRanges: readEncodeRanges,
     heldRanges: readEncodeHeldRanges,
+    stretchCopied,
     reviseBudget: () => parts.machineBudget.revise("memory", "admitted encode inputs"),
     capacity: () => parts.machineBudget.capacityOf("memory"),
     // How long this output's input takes to open, from the one owner of start

package/services/storage/memory-report.js

@@ -438,6 +438,12 @@ export function unaccountedBufferBytes(arrayBuffers, stores = []) {
  * @param {number} [options.snapshotFloorBytes]
  * @param {number} [options.snapshotGrowthBytes]
  * @param {number} [options.keepSnapshots] - Newest to keep; zero keeps all.
+ * @param {boolean} [options.heapSnapshots] - Whether a high-water earns a heap
+ *   snapshot at all. Off unless the operator asks: a snapshot stops the thread
+ *   for as long as it is written, and in a healthy session on Home Assistant
+ *   the torrent thread's array buffers reach the floor below — one snapshot
+ *   of 524 MB stopped that thread 4.4 s, 20 s after a seek the viewer was
+ *   waiting on (r61, torrent-tv/meta#178).
  * @param {number} [options.bufferSnapshotFloorBytes] - Array buffers above what
  *   the stores hold that earn a snapshot of their own; zero never takes one.
  * @param {number} [options.bufferSnapshotGrowthBytes]
@@ -460,6 +466,7 @@ export function startMemoryReport({
   snapshotFloorBytes = 500 * 1024 * 1024,
   snapshotGrowthBytes = 100 * 1024 * 1024,
   keepSnapshots = 0,
+  heapSnapshots = false,
   bufferSnapshotFloorBytes = 0,
   bufferSnapshotGrowthBytes = 500 * 1024 * 1024
 }) {
@@ -601,7 +608,7 @@ export function startMemoryReport({
       // and when V8 did kill the process at 1.99 GB of old space, no snapshot
       // had ever been taken of the growth that did it.
       const heapWatched = processMemory.heapTotal;
-      if (heapWatched > highWater + snapshotGrowthBytes && heapWatched > snapshotFloorBytes) {
+      if (heapSnapshots && heapWatched > highWater + snapshotGrowthBytes && heapWatched > snapshotFloorBytes) {
         highWater = heapWatched;
         await takeSnapshot(heapWatched, "a new high-water of the heap");
       }
@@ -609,6 +616,7 @@ export function startMemoryReport({
       // the heap: each one is in it with its size and its holder.
       const unaccounted = unaccountedBufferBytes(processMemory.arrayBuffers, stores);
       if (
+        heapSnapshots &&
         bufferSnapshotFloorBytes > 0 &&
         unaccounted > bufferSnapshotFloorBytes &&
         unaccounted > bufferHighWater + bufferSnapshotGrowthBytes

package/services/storage/piece-store/piece-disk-store.js

@@ -283,7 +283,23 @@ export class PieceDiskStore {
     await this.#removing.get(index);
     await this.#ensureDirectory();
     await this.#makeRoomFor(bytes.length, index);
-    await fs.writeFile(this.#pathOf(index), bytes);
+    // ONE WRITE FOR THE PIECE, not `fs.writeFile`: that writes 512 KiB at a
+    // time, sixteen requests for an 8 MiB piece, and each waits for a turn of
+    // this thread's loop. On Home Assistant the torrent thread's loop ran about
+    // a second late while downloading, a piece took that many seconds to reach
+    // the disk, and pieces arriving at 20 MB/s waited for their writes: 1.1 GB
+    // of them in that thread's memory at once (r61, torrent-tv/meta#178).
+    const handle = await fs.open(this.#pathOf(index), "w");
+    try {
+      let written = 0;
+      while (written < bytes.length) {
+        const { bytesWritten } = await handle.write(bytes, written, bytes.length - written, written);
+        if (!(bytesWritten > 0)) throw new Error(`Piece ${index} could not be written: the disk took nothing.`);
+        written += bytesWritten;
+      }
+    } finally {
+      await handle.close();
+    }
     if (!this.#stored.has(index)) {
       this.#bytes += bytes.length;
     } else {

package/services/torrent/worker/client.js

@@ -117,13 +117,13 @@ export class TorrentWorkerClient {
   /**
    * @param {{ memoryBytes?: number, stateDir?: string, onPiecesArrived?: (event: object) => void }} [options]
    */
-  constructor({ memoryBytes, stateDir, onPiecesArrived, onSourceForgotten, onHoldingsChanged } = {}) {
+  constructor({ memoryBytes, stateDir, heapSnapshots = false, onPiecesArrived, onSourceForgotten, onHoldingsChanged } = {}) {
     this.#worker = new Worker(fileURLToPath(WORKER_URL), {
       // `stateDir` travels because the worker writes heap snapshots of its own
       // isolate there. It cannot choose a directory any other way: a worker may
       // not change the process's working directory, and the isolate that has
       // died three times is the one no snapshot has ever been taken of.
-      workerData: { memoryBytes, stateDir }
+      workerData: { memoryBytes, stateDir, heapSnapshots }
     });
     this.#caller = createCaller(this.#worker);
     this.#onPiecesArrived = onPiecesArrived ?? (() => undefined);

package/services/torrent/worker/worker.js

@@ -692,6 +692,7 @@ startMemoryReport({
   quietMs: 60_000,
   changeBytes: 25 * 1024 * 1024,
   snapshotDir: workerData?.stateDir || undefined,
+  heapSnapshots: workerData?.heapSnapshots === true,
   snapshotFloorBytes: 400 * 1024 * 1024,
   snapshotGrowthBytes: 400 * 1024 * 1024,
   // CHOSEN, not derived: no healthy session has recorded this figure yet, so