@torrent-tv/proxy 2.77.0 → 2.79.0

package/services/encode/EncodeRun.js

@@ -96,6 +96,28 @@ export class EncodeRun {
   /** @type {number} */
   #startedAt = 0;
 
+  /** Half a name left over from the last chunk of the encoder's own channel. */
+  #closedTail = "";
+
+  /**
+   * When this run was told to stop, so the death itself can be priced.
+   *
+   * The plan weighs moving an encoder against letting it drive on, and dying is
+   * one of the terms. It was measured in the field at 430-729 ms — larger than
+   * the start it is added to — by a log line that lived in the one place that
+   * killed a run. That place is gone, so the run times its own death.
+   */
+  #stopOrderedAt = 0;
+
+  /**
+   * When the first thing this run ever produced appeared.
+   *
+   * The other term the plan needs: a run started where nothing is downloaded
+   * waits for the swarm before it can encode a frame, and that wait is the
+   * largest part of what a restart costs. Nothing measured it before.
+   */
+  #firstOutputAt = 0;
+
   /** @type {number} */
   #speedX = 0;
 
@@ -132,6 +154,9 @@ export class EncodeRun {
    * @param {{ info: (line: string) => void, warn: (line: string) => void, error?: (line: string) => void }} params.logger
    * @param {() => number} [params.now]
    * @param {(ended: RunEnded) => void} [params.onEnded]
+   * @param {(name: string) => void} [params.onClosed] - Called with the file name
+   *   of every piece the encoder has FINISHED writing, as the encoder itself
+   *   names it on its own channel.
    * @param {(progress: { processedSeconds: number | null, speed: string | null }) => void} [params.onProgress]
    *   Called for every `-progress` report. Seconds count from the START OF THIS
    *   RUN on both branches — neither `-output_ts_offset` nor `-copyts` changes
@@ -158,6 +183,7 @@ export class EncodeRun {
     logger,
     now,
     onEnded,
+    onClosed,
     onProgress,
     lastSegmentIndex,
     inputUnavailable,
@@ -174,6 +200,9 @@ export class EncodeRun {
     this.now = typeof now === "function" ? now : Date.now;
     this.onEnded = typeof onEnded === "function" ? onEnded : () => {};
     this.onProgress = typeof onProgress === "function" ? onProgress : () => {};
+    // Told the NAME of every piece the encoder has closed. The name is the
+    // proof it is whole; nothing else here can prove that.
+    this.onClosed = typeof onClosed === "function" ? onClosed : () => {};
     this.lastSegmentIndex = typeof lastSegmentIndex === "function" ? lastSegmentIndex : () => null;
     this.inputUnavailable = typeof inputUnavailable === "function" ? inputUnavailable : () => false;
     this.argsDescribed = argsDescribed;
@@ -304,6 +333,16 @@ export class EncodeRun {
    */
   #wire(process) {
     process.stdout?.on("data", (chunk) => this.#readProgress(String(chunk)));
+    // THE CHANNEL THE ENCODER NAMES ITS FINISHED PIECES ON.
+    //
+    // A name arrives here when ffmpeg CLOSES the file, so the name is proof the
+    // piece is whole — measured on the addon host 2026-09-05. Nothing else can
+    // prove it: a file on disk may still be being written, and the only other
+    // evidence available was the existence of the NEXT file, which never comes
+    // for the last piece of a run.
+    //
+    // Lines can arrive split, so what is left over is kept for the next chunk.
+    process.stdio?.[3]?.on("data", (chunk) => this.#readClosedPieces(String(chunk)));
     process.stderr?.on("data", (chunk) => {
       const line = String(chunk).trim();
       if (line.length > 0) {
@@ -358,6 +397,9 @@ export class EncodeRun {
   noteProduced(index) {
     if (Number.isInteger(index) && index >= this.from) {
       this.#produced.add(index);
+      if (this.#firstOutputAt === 0) {
+        this.#firstOutputAt = this.now();
+      }
       if (this.#state === ENCODE_RUN_STATE.STARTING) {
         this.#transition(ENCODE_RUN_EVENT.FIRST_SEGMENT);
       }
@@ -369,6 +411,25 @@ export class EncodeRun {
    *
    * @param {number} speedX - Times realtime.
    */
+  /**
+   * Names of finished pieces, as the encoder writes them.
+   *
+   * @param {string} text
+   */
+  #readClosedPieces(text) {
+    this.#closedTail += text;
+    const lines = this.#closedTail.split(/\r?\n/);
+    // The last piece of the chunk may be half a name; it waits for the rest.
+    this.#closedTail = lines.pop() ?? "";
+    for (const line of lines) {
+      const name = line.trim();
+      if (name.length === 0) {
+        continue;
+      }
+      this.onClosed(name);
+    }
+  }
+
   noteSpeed(speedX) {
     if (Number.isFinite(speedX) && speedX > 0) {
       this.#speedX = speedX;
@@ -389,6 +450,7 @@ export class EncodeRun {
     }
     this.#stopping = true;
     this.#stopReason = because;
+    this.#stopOrderedAt = this.now();
     this.#transition(ENCODE_RUN_EVENT.STOP_ORDERED);
     // A suspended process does not act on SIGTERM until it is continued, so the
     // wait for its exit would never end. Let it run before asking it to stop.
@@ -554,6 +616,14 @@ export class EncodeRun {
       to: this.to,
       reached: this.reached,
       livedMs,
+      // How long dying took, and how long the first output took to appear.
+      // Null where the run was never told to stop, or never produced anything:
+      // an absent measurement says so rather than reading as zero.
+      dyingMs: this.#stopOrderedAt > 0 ? this.now() - this.#stopOrderedAt : null,
+      firstOutputMs:
+        this.#firstOutputAt > 0 && this.#startedAt > 0
+          ? this.#firstOutputAt - this.#startedAt
+          : null,
       normal: ending === ENCODE_EXIT.COMPLETE,
       lastError: this.lastError
     };

package/services/encode/SegmentStore.js

@@ -39,6 +39,8 @@ import { rmSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
+import { discardOpenPiece } from "./open-piece.js";
+
 /** Where every output's segments live. One root for the process. */
 export const DEFAULT_STORE_ROOT = path.join(os.tmpdir(), "torrent-tv-hls");
 
@@ -256,6 +258,30 @@ export class SegmentStore {
     return proven.sort((left, right) => left - right);
   }
 
+  /**
+   * Whether this piece is finished, and may therefore be served.
+   *
+   * Two proofs, and the first is the good one:
+   *
+   * 1. **the encoder said so** — it names each file on a channel of its own the
+   *    moment it closes it, so the name is the writer's own statement that the
+   *    piece is whole;
+   * 2. **the next file exists** — which only proves it for pieces this process
+   *    did not watch being written, left by an earlier life of it. It is not
+   *    true of the last piece of any run, and that is what used to hold the
+   *    first segment of every run from the viewer.
+   *
+   * @param {string} key
+   * @param {number} index
+   * @returns {boolean}
+   */
+  isClosed(key, index) {
+    if (this.#closed.get(key)?.has(index)) {
+      return true;
+    }
+    return this.refresh(key).byNumber.has(index + 1);
+  }
+
   /**
    * Say that a segment is closed for a reason the disk cannot show.
    *
@@ -277,6 +303,39 @@ export class SegmentStore {
     known.add(index);
   }
 
+  /**
+   * Throw away the piece a run had open when it ended, if it is unusable.
+   *
+   * The store owns this output's directory and knows how its files are named,
+   * so it is the one place that can answer which file a run left open. The
+   * judging of a NON-EMPTY file — does it carry every track it should — needs
+   * the output's init bytes and belongs to whoever holds them; passed in, and
+   * absent it only an empty file is removed, which is the case that caused this
+   * to be written (a run stopped 548 ms after starting left a zero-byte file
+   * whose name then read as a segment made).
+   *
+   * @param {string} key
+   * @param {{ from: number, to: number } | null} within - The run's own
+   *   numbers: several runs write into one directory, so the piece to discard
+   *   has to be looked for inside the stretch the ended run was given.
+   * @param {((raw: Buffer) => boolean) | null} [judgeUsable]
+   * @returns {Promise<number | null>} The segment number removed, or null.
+   */
+  async discardOpenPieceOf(key, within, judgeUsable = null) {
+    const format = this.#formats.get(key);
+    if (!format) {
+      return null;
+    }
+    const removed = await discardOpenPiece(this.directoryFor(key), format, within, judgeUsable);
+    if (removed !== null) {
+      this.#held.delete(key);
+      this.#logger?.info?.(
+        `segment store: discarded the open piece #${removed} of ${key.slice(0, 60)}`
+      );
+    }
+    return removed;
+  }
+
   /**
    * The one number in this output whose closure nothing on disk proves.
    *

package/services/encode/open-piece.js

@@ -0,0 +1,88 @@
+/**
+ * @file The piece a run had open when it ended.
+ *
+ * A fact of a run's output directory, and therefore of the encoding layer. It
+ * lived in the eleven-thousand-line file that is being taken apart, where it was
+ * called by the one place that killed a run; stopping is decided in one place
+ * now and carried out in another, so the cleanup belongs to the layer that owns
+ * the directories rather than to whoever happened to do the killing.
+ */
+
+import { readdir, readFile, stat, unlink } from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Remove the piece a run had open when it ended, if that piece is unusable.
+ *
+ * The `segment` muxer creates its output file when it OPENS it and writes into
+ * it until the next cut, so at any instant exactly one file in a run's
+ * directory is unfinished: the highest-numbered one. A run that reaches the end
+ * of its work closes that file and it is a good piece; a run killed for a seek
+ * does not — measured 2026-09-03, ffmpeg exited 19 ms after SIGTERM and left
+ * `segment-00025.mp4` at zero bytes, which then closed the only hole in the
+ * numbering and convinced the look-ahead to keep the encoder stopped for having
+ * "produced" it.
+ *
+ * Only an unusable piece goes. A run stopped between two cuts leaves a finished
+ * file behind, and deleting good output would mean making it a second time.
+ *
+ * @param {string | null | undefined} runDirPath
+ * @param {{ isSegmentFileName: (name: string) => boolean, segmentIndexFromName: (name: string) => number }} segmentFormat
+ * @param {((raw: Buffer) => boolean) | null} judgeUsable - Whether a non-empty
+ *   piece carries what it should. Null where nothing can say, and then only an
+ *   empty file is removed.
+ * @returns {Promise<number | null>} The segment number removed, or null.
+ */
+export async function discardOpenPiece(runDirPath, segmentFormat, within, judgeUsable) {
+  if (!runDirPath || typeof segmentFormat?.isSegmentFileName !== "function") {
+    return null;
+  }
+  // Only inside the stretch the ended run was given. Every run of an output
+  // writes into one directory now — they are kept apart by their intervals
+  // rather than by a directory each — so the highest-numbered file in there may
+  // belong to a run that is still going, and removing it would take away a
+  // piece somebody is producing.
+  const from = Number.isInteger(within?.from) ? within.from : 0;
+  const to = Number.isInteger(within?.to) && within.to >= from ? within.to : Number.MAX_SAFE_INTEGER;
+  let highest = null;
+  try {
+    for (const name of await readdir(runDirPath)) {
+      if (!segmentFormat.isSegmentFileName(name)) {
+        continue;
+      }
+      const index = segmentFormat.segmentIndexFromName(name);
+      if (index < from || index > to) {
+        continue;
+      }
+      if (index >= 0 && (highest === null || index > highest.index)) {
+        highest = { index, name };
+      }
+    }
+  } catch {
+    return null; // The run wrote nothing, or its directory is already gone.
+  }
+  if (highest === null) {
+    return null;
+  }
+  const filePath = path.join(runDirPath, highest.name);
+  let unusable = false;
+  try {
+    const info = await stat(filePath);
+    if (info.size === 0) {
+      unusable = true;
+    } else if (typeof judgeUsable === "function") {
+      unusable = !judgeUsable(await readFile(filePath));
+    }
+  } catch {
+    return null; // Gone between the listing and the question.
+  }
+  if (!unusable) {
+    return null;
+  }
+  try {
+    await unlink(filePath);
+    return highest.index;
+  } catch {
+    return null; // Already removed.
+  }
+}

package/services/encode/run-budget.js

@@ -0,0 +1,89 @@
+/**
+ * @file How many encoders this machine can afford on one output.
+ *
+ * Not how many are wanted — that is the demand map's answer — but how many can
+ * run at once without making things worse. It is the SMALLEST of several
+ * limits, and every one of them is measured:
+ *
+ * 1. **the processor.** A second encoder slows the first: measured on the addon
+ *    host at 1.70x for two at 854x480 and 1.98x at 1920x1080. The ladder is
+ *    walked while the measured penalty still leaves every encoder above
+ *    realtime, and it stops where the measurements stop rather than continuing
+ *    a curve two points cannot describe;
+ * 2. **the swarm.** Every encoder reads the same torrent, so together they
+ *    cannot consume faster than it is delivered. On the field file of
+ *    2026-09-05 a second of film weighs 842 KB and the swarm gave 2119-3347
+ *    KB/s, so the encoders' speeds together may reach 2.5-4x — which is why a
+ *    copy the processor would run at 8x runs at four, and why a third encoder
+ *    makes all three slower rather than adding anything;
+ * 3. **memory for the torrent's pieces.** Encoders placed far apart hold
+ *    windows that do not overlap, so the store must hold their SUM. On the
+ *    addon host the store was already full on the readers of one viewer — "6
+ *    readers want 14 pieces of 14 the store may hold" — so this is the limit
+ *    that binds first there, not the processor.
+ *
+ * Which of them bound the answer is returned beside it, because "why is there
+ * only one encoder" is otherwise a question no log can answer.
+ */
+
+/**
+ * @typedef {object} RunBudget
+ * @property {number} runs - How many encoders may run on this output.
+ * @property {string} because - Which limit decided it.
+ */
+
+/**
+ * @param {object} params
+ * @param {number} params.byProcessor - What the processor allows, from the
+ *   measured speed and the measured penalty for concurrency. At least one.
+ * @param {number} [params.speedX] - The measured speed of one encoder here.
+ * @param {number} [params.refetchSecPerFilmSecond] - Seconds of swarm time per
+ *   second of film: the film's own byte rate over what the swarm delivers. One
+ *   encoder at speed `s` therefore consumes `s * refetch` of the swarm, and
+ *   what they may consume together is all of it. Absent where neither rate has
+ *   been measured, and then the swarm does not bound the answer.
+ * @param {number} [params.storeBytes] - What the piece store may hold.
+ * @param {number} [params.readerWindowBytes] - What one encoder's reader keeps.
+ * @returns {RunBudget}
+ */
+export function affordableRuns({
+  byProcessor,
+  speedX,
+  refetchSecPerFilmSecond,
+  storeBytes,
+  readerWindowBytes
+}) {
+  let runs = Number.isFinite(byProcessor) && byProcessor > 0 ? Math.floor(byProcessor) : 1;
+  let because = "the processor";
+
+  // The swarm. One encoder at speed `s` takes `s * refetch` of what is
+  // delivered, and everything running takes the sum, which cannot pass one.
+  if (
+    Number.isFinite(refetchSecPerFilmSecond) &&
+    refetchSecPerFilmSecond > 0 &&
+    Number.isFinite(speedX) &&
+    speedX > 0
+  ) {
+    const bySwarm = Math.max(1, Math.floor(1 / (speedX * refetchSecPerFilmSecond)));
+    if (bySwarm < runs) {
+      runs = bySwarm;
+      because = "what the swarm delivers";
+    }
+  }
+
+  // Memory: encoders far apart hold windows that do not overlap.
+  if (
+    Number.isFinite(storeBytes) &&
+    storeBytes > 0 &&
+    Number.isFinite(readerWindowBytes) &&
+    readerWindowBytes > 0
+  ) {
+    const byMemory = Math.max(1, Math.floor(storeBytes / readerWindowBytes));
+    if (byMemory < runs) {
+      runs = byMemory;
+      because = "memory for the torrent's pieces";
+    }
+  }
+
+  return { runs: Math.max(1, runs), because };
+}

package/services/encode/run-command.js

@@ -620,6 +620,21 @@ export function buildRunCommand({
       "0.05",
       "-segment_start_number",
       String(safeIndex),
+      // THE ENCODER SAYS WHEN A PIECE IS FINISHED, on a channel of its own.
+      //
+      // Measured on the addon host 2026-09-05: a name appears in this list when
+      // the file is CLOSED, not when it is created — at the third sample
+      // `seg-000.mp4` was on disk and absent from the list, and it appeared at
+      // the fourth, in the same moment `seg-001.mp4` came into being. So a name
+      // here is the writer's own statement that the piece is whole.
+      //
+      // Without it, a finished file is indistinguishable from one still being
+      // written, and the only proof available was the existence of the NEXT
+      // one — which never comes for the last piece of every run.
+      "-segment_list",
+      "pipe:3",
+      "-segment_list_flags",
+      "+live",
       ...explicitTimes,
       segmentFormat.segmentFileNameTemplate()
     );

package/services/encode/run-costs.js

@@ -0,0 +1,94 @@
+/**
+ * @file What starting and stopping an encoder costs on THIS host.
+ *
+ * Two figures, and both are terms in the one decision anybody makes about a
+ * running encoder: let it drive on through material that already exists, or
+ * stop it and start another where the material is missing.
+ *
+ * Before this file neither was measured here. The start was a single reading
+ * taken once on one machine and written into the code as a constant; the stop
+ * and the wait for the first output were not counted at all, so the comparison
+ * priced only one side of itself and always answered the same way.
+ *
+ * Readings are kept and read as their median: one starved start must not decide
+ * the rule, and a host that has since become busy must be able to change the
+ * answer.
+ */
+
+/**
+ * How many recent readings a figure is taken from. The same reasoning as the
+ * other learned figures in this proxy: long enough that one reading does not
+ * move the answer, short enough that the answer still follows the host.
+ */
+const RECENT_READINGS = 20;
+
+/**
+ * The middle of a set of readings, or null when there are none.
+ *
+ * Written here rather than borrowed from the general helper one level up: this
+ * layer states facts and imports nothing above itself, and four lines of
+ * arithmetic are not worth breaking that for.
+ *
+ * @param {number[]} values
+ * @returns {number | null}
+ */
+function middleOf(values) {
+  if (values.length === 0) {
+    return null;
+  }
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
+}
+
+export class RunCosts {
+  /** How long dying took, in milliseconds. @type {number[]} */
+  #dying = [];
+
+  /** How long the first output took to appear, in milliseconds. @type {number[]} */
+  #firstOutput = [];
+
+  /**
+   * Take the two readings a finished run carries. Either may be absent — a run
+   * that was never told to stop did not die on command, and one that produced
+   * nothing has no first output — and an absent reading is not a zero.
+   *
+   * @param {{ dyingMs?: number | null, firstOutputMs?: number | null }} ended
+   */
+  note(ended) {
+    if (Number.isFinite(ended?.dyingMs)) {
+      RunCosts.#keep(this.#dying, /** @type {number} */ (ended.dyingMs));
+    }
+    if (Number.isFinite(ended?.firstOutputMs)) {
+      RunCosts.#keep(this.#firstOutput, /** @type {number} */ (ended.firstOutputMs));
+    }
+  }
+
+  /**
+   * @param {number[]} readings
+   * @param {number} value
+   */
+  static #keep(readings, value) {
+    readings.push(value);
+    while (readings.length > RECENT_READINGS) {
+      readings.shift();
+    }
+  }
+
+  /**
+   * The two costs in seconds, from this host's own readings.
+   *
+   * Zero where nothing has been measured yet. Zero understates both, so a plan
+   * that has no readings prices moving an encoder as cheaper than it is — which
+   * is why the plan keeps a run it cannot compare rather than moving it.
+   *
+   * @returns {{ killCostSec: number, firstByteWaitSec: number, samples: number }}
+   */
+  seconds() {
+    return {
+      killCostSec: (middleOf(this.#dying) ?? 0) / 1000,
+      firstByteWaitSec: (middleOf(this.#firstOutput) ?? 0) / 1000,
+      samples: Math.min(this.#dying.length, this.#firstOutput.length)
+    };
+  }
+}