@torrent-tv/proxy 2.96.14 → 2.97.1

package/bin/cli.js

@@ -88,6 +88,7 @@ program
   .option("--no-transcode-audio", "Disable optional HLS AAC audio transcoding")
   .option("--no-port-mapping", "Disable automatic UPnP/NAT-PMP port mapping")
   .option("--delivery-sink", "Serve /api/delivery-sink, a torrent-free byte stream for delivery testing")
+  .option("--profile-start <seconds>", "Profile the main thread's CPU for this many seconds from the first source opened, written beside the state (off by default)")
   .option(
     "--send-chunk-bytes <bytes>",
     "Size of one data-channel message when sending a response body (0 = whatever the body read hands over, ~64KB). For measuring whether the delivery wedge depends on that size."
@@ -392,6 +393,7 @@ try {
     budgetPolicy,
     logFile: options.logFile ?? "",
     deliverySink: options.deliverySink === true,
+    profileStartSeconds: Number(options.profileStart) || 0,
     // Late-bound the same way `webRtcManager` is below: the torrent pool is
     // built inside `startProxyServer`, before `dataChannelHandler` — the
     // thing that actually owns a channel to push down — exists.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.96.14",
+  "version": "2.97.1",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/routes/encode-input/get.js

@@ -36,6 +36,19 @@ export function handleEncodeInputGet(req, reply, { inputOf, refused = () => {} }
     refused({ token, fileIndex, start: range.start, runTag: input.runTag ?? null });
     return reply.code(503).send({ error: "Original input bytes are outside the admitted ranges." });
   }
+  // What was handed to the system for this answer before its connection
+  // closed, and whether all of it was: an answer runs to the end of its
+  // admitted range and FFmpeg closes the connection once it has what it wants,
+  // so the range answered says nothing about the bytes taken. What FFmpeg read
+  // is at most what was handed over; the kernel's buffer is the difference,
+  // and it cannot be seen from here (torrent-tv/meta#177).
+  const socket = req.raw?.socket;
+  const writtenBefore = socket?.bytesWritten ?? 0;
+  reply.raw?.once?.("close", () => {
+    input.noteAnswer?.({ fileIndex, start: range.start, end: held.end,
+      written: Math.max(0, (socket?.bytesWritten ?? writtenBefore) - writtenBefore),
+      handedOver: reply.raw?.writableFinished === true });
+  });
   return reply.code(206).header("Content-Length", String(held.bytes.length))
     .header("Content-Range", `bytes ${range.start}-${held.end}/${length}`).send(held.bytes);
 }

package/server.js

@@ -80,6 +80,8 @@ import { measureStartAndStop } from "./services/encode/start-stop-cost.js";
 import { benchmarkAudio } from "./services/encode/audio-calibration.js";
 import { logger } from "./utils/logger.js";
 import { completedFilesRoot } from "./services/storage/files/CompletedFiles.js";
+import { startupCpuProfile } from "./services/storage/cpu-profile.js";
+import { startProcessorReport } from "./services/storage/processor-report.js";
 
 const require = createRequire(import.meta.url);
 const { version } = require("./package.json");
@@ -119,6 +121,8 @@ function buildPortCandidates(startPort, maxAttempts = 51) {
 export async function startProxyServer({
   host, port, transcodeAudio, ffmpegBin, memoryBytes, segmentFormat, stateDir, onSubtitleCues, diagnostics, budgetPolicy,
   deliverySink = false,
+  // Seconds of main-thread CPU profile from the first source opened; 0 is off.
+  profileStartSeconds = 0,
   // What viewers' connections carry beyond the film, per byte of film; the
   // transport counts it and is built after this server, so it is late-bound.
   serviceShare = () => null,
@@ -875,6 +879,22 @@ export async function startProxyServer({
       await torrentPool.fillFileInBackground?.(torrent, fileIndex);
     }
   });
+  // Where the processor goes, and at a cold start what the main thread does
+  // with it, when the operator asks (torrent-tv/meta#177).
+  const startupProfile = startupCpuProfile({
+    seconds: profileStartSeconds,
+    dir: stateDir || "",
+    log: (line) => logger.info(line),
+    mayKeep: (bytes) => diagnostics?.mayKeep?.({ what: "a CPU profile", bytes }) ?? true
+  });
+  startProcessorReport({
+    log: (line) => logger.info(line),
+    ffmpegPids: () => [...outputParts.outputs.values()]
+      .flatMap((output) => outputParts.encodeRuns.liveRunsOf(output))
+      .map((run) => run.process?.pid)
+      .filter((pid) => Number.isInteger(pid)),
+    detailed: () => startupProfile.active()
+  });
   // What the last life of this process left on the disk. The kernel kills this
   // one often enough for that to be an ordinary state rather than an odd one —
   // twice in a single viewing on 2026-09-02 — and when it does, no exit handler
@@ -1106,9 +1126,11 @@ export async function startProxyServer({
   );
   // Bytes a browser can time its link with before any film has been chosen.
   app.get("/api/link-probe", async (req, reply) => handleApiLinkProbeGet(req, reply));
-  app.post("/api/sources", async (req, reply) =>
-    handleApiSourcesPost(req, reply, { sourceRegistry, viewers: outputParts.viewers })
-  );
+  app.post("/api/sources", async (req, reply) => {
+    // A cold start begins with the first source opened (torrent-tv/meta#177).
+    startupProfile.begin();
+    return handleApiSourcesPost(req, reply, { sourceRegistry, viewers: outputParts.viewers });
+  });
   app.get("/api/sources/:sourceKey/stats", async (req, reply) =>
     handleApiSourceStatsGet(req, reply, { sourceRegistry, torrentPool })
   );

package/services/encode/EncodeInputs.js

@@ -33,8 +33,13 @@ export class EncodeInputs {
   #wanted = new Map();
   /** Bytes per millisecond of recent original-input copies, read where the copy is made. */
   #copyRates = [];
-  /** Seconds an encoder took to open its original input, per output. */
-  #opens = new Map();
+  /**
+   * How long an encoder takes to open an output's original input: asked of
+   * the one owner of start readings (`RunCosts`, torrent-tv/meta#184).
+   *
+   * @type {(output: object) => { seconds: number, measured: boolean } | null}
+   */
+  #openOf;
   #resolve;
   #read;
   #heldRanges;
@@ -46,7 +51,9 @@ export class EncodeInputs {
   #log;
   #now;
 
-  constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true, log = () => {}, now = Date.now }) {
+  constructor({ resolve, readRanges, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
+    openOf = () => null, log = () => {}, now = Date.now }) {
+    this.#openOf = openOf;
     this.#resolve = resolve;
     this.#read = readRanges;
     this.#heldRanges = heldRanges;
@@ -130,10 +137,26 @@ export class EncodeInputs {
     return null;
   }
 
-  retain(output, windows) {
+  /**
+   * Withdraw what this output no longer needs: a request no wanted window
+   * covers, and — when `starts` is given — a copy already held whose start is
+   * neither where a live run ends nor where one is being started. A next
+   * stretch is copied for the run working before it (torrent-tv/meta#158);
+   * once that run has gone and the plan starts elsewhere, after a seek, the
+   * copy held memory nobody would use (torrent-tv/meta#184).
+   *
+   * @param {object} output
+   * @param {{ from: number, to: number }[]} windows
+   * @param {Set<number> | null} [starts]
+   */
+  retain(output, windows, starts = null) {
     for (const [key, request] of this.#requests) {
-      if (request.output !== output || windows.some(window => request.from <= window.to && request.to >= window.from)) continue;
-      this.#note(request, "withdrawn: no wanted window covers it");
+      if (request.output !== output) continue;
+      const covered = windows.some(window => request.from <= window.to && request.to >= window.from);
+      const unused = starts !== null && request.result?.kind === "result" && !starts.has(request.from);
+      if (covered && !unused) continue;
+      this.#note(request, covered ? "withdrawn: no live run ends before it and no start is placed there"
+        : "withdrawn: no wanted window covers it");
       this.#requests.delete(key);
       this.#wanted.delete(key);
       request.result?.release?.();
@@ -154,16 +177,13 @@ export class EncodeInputs {
     let existing = this.#requests.get(key);
     if (existing && existing.to !== to) {
       // The plan's bound for this start moved. A copy already made beyond it
-      // would be a run over somebody else's stretch, so it is given back; one
-      // being made is checked when it is done, and one waiting for bytes or
-      // memory takes the new bound when its own event prepares it again.
+      // holds the shorter stretch whole, so the run is commanded over its
+      // prefix and the copy is kept until the run ends: copying it again cost
+      // as long as the first copy (torrent-tv/meta#184). One being made is
+      // narrowed when it is done, and one waiting for bytes or memory takes the
+      // new bound when its own event prepares it again.
       existing.to = to;
-      if (existing.result?.kind === "result" && existing.result.to > to) {
-        this.#requests.delete(key);
-        this.#wanted.delete(key);
-        existing.result.release();
-        existing = undefined;
-      }
+      if (existing.result?.kind === "result" && existing.result.to > to) existing.result.to = Math.max(existing.from, to);
     }
     if (existing?.result?.kind === "result") {
       this.#requests.delete(key);
@@ -200,44 +220,38 @@ export class EncodeInputs {
       this.#wanted.delete(key);
       request.result?.release?.();
     }
-    this.#opens.delete(addressOf(output));
   }
 
   /**
-   * How long an encoder took to open this output's original input: from its
-   * spawn to the name of its first closed piece, less what encoding that piece
-   * costs. Measured by whoever ran it, kept here where a stretch is sized.
+   * How many bytes a stretch may hold so that copying it takes no longer than
+   * opening the input it replaces: the recent copy rate times this output's
+   * recent open. Beyond it the open's share of each piece falls below the
+   * copy's, which is the same for every piece; a longer stretch buys less and
+   * delays the run's start more. Null until a copy rate is measured and an
+   * open is known, and the run then gets one piece. An output that has not
+   * opened yet is sized by the host's startup reading, which is said: at a
+   * cold start the second run was one piece too for want of it, and cost
+   * another open (6.34 s, torrent-tv/meta#166 run 12).
    *
    * @param {object} output
-   * @param {number} seconds
+   * @returns {{ bytes: number, openMeasured: boolean } | null}
    */
-  noteOpen(output, seconds) {
-    if (!Number.isFinite(seconds) || seconds < 0) return;
-    const readings = this.#opens.get(addressOf(output)) ?? [];
-    keep(readings, seconds);
-    this.#opens.set(addressOf(output), readings);
+  stretchFor(output) {
+    const rate = middleOf(this.#copyRates);
+    const open = this.#openOf(output);
+    return rate === null || !open ? null : { bytes: Math.floor(rate * open.seconds * 1000), openMeasured: open.measured };
   }
 
   /**
-   * How many bytes a stretch may hold so that copying it takes no longer than
-   * opening the input it replaces: the recent copy rate times this output's
-   * recent open. Beyond it the open's share of each piece falls below the
-   * copy's, which is the same for every piece; a longer stretch buys less and
-   * delays the run's start more. Null until both have been measured, and the
-   * run then gets one piece, as it did before either reading existed.
-   *
    * @param {object} output
    * @returns {number | null}
    */
   stretchBytesFor(output) {
-    const rate = middleOf(this.#copyRates);
-    const open = middleOf(this.#opens.get(addressOf(output)) ?? []);
-    return rate === null || open === null ? null : Math.floor(rate * open * 1000);
+    return this.stretchFor(output)?.bytes ?? null;
   }
 
   #prepare(request) {
     request.pending = true;
-    request.stale = false;
     request.revision = this.#revision;
     request.memoryRevision = this.#memoryRevision;
     // Where the time from the request to its input goes, across every attempt:
@@ -260,13 +274,9 @@ export class EncodeInputs {
         if (result.kind === "result") Object.assign(result, { from: request.from, to: request.from });
       }
       if (this.#requests.get(request.key) !== request) { result.release?.(); return; }
-      if (result.kind === "result" && result.to > request.to) {
-        // The plan narrowed this start while the copy was being made.
-        result.release();
-        request.result = null;
-        request.stale = true;
-        return;
-      }
+      // The plan narrowed this start while the copy was being made: the run
+      // is commanded over its prefix.
+      if (result.kind === "result" && result.to > request.to) result.to = Math.max(request.from, request.to);
       request.result = result;
       this.#note(request, result.kind === "result" ? `ready #${result.from}..#${result.to}, ${result.bytes ?? "?"} bytes` +
         (Number.isFinite(result.copyMs) ? `, copied in ${result.copyMs}ms` : "")
@@ -281,8 +291,8 @@ export class EncodeInputs {
     }).finally(() => {
       request.pending = false;
       if (request.result?.kind !== "needs-memory") this.#wanted.delete(request.key);
-      const changed = request.stale || (request.result?.kind === "needs-memory"
-        ? request.memoryRevision !== this.#memoryRevision : request.revision !== this.#revision);
+      const changed = request.result?.kind === "needs-memory"
+        ? request.memoryRevision !== this.#memoryRevision : request.revision !== this.#revision;
       if (this.#requests.get(request.key) === request && changed &&
         request.result?.kind !== "result" && request.result?.kind !== "terminal") this.#prepare(request);
     });
@@ -360,7 +370,9 @@ export class EncodeInputs {
       }
       return good;
     };
-    const target = this.stretchBytesFor(output);
+    const sizing = this.stretchFor(output);
+    const target = sizing?.bytes ?? null;
+    if (sizing && !sizing.openMeasured) request.timing.openApproximate = true;
     let end = target === null || this.#heldRanges === null ? from : await longest(target);
     if (this.#requests.get(request.key) !== request) return undefined;
     this.#wanted.set(request.key, { wanted: stretches.get(end).bytes, required: minimum });
@@ -429,7 +441,8 @@ function describeTiming(timing, now) {
   return `; asked ${Math.max(0, now - timing.askedAt)}ms ago over ${timing.attempts} attempt(s): ` +
     `${timing.resolutions} resolution(s) ${timing.resolveMs}ms ` +
     `(${timing.reads} file read(s): queued ${timing.queuedMs}ms, reading ${timing.readMs}ms), ` +
-    `held ranges ${timing.heldMs}ms, budget ${timing.budgetMs}ms`;
+    `held ranges ${timing.heldMs}ms, budget ${timing.budgetMs}ms` +
+    (timing.openApproximate ? "; sized by the host's startup open, this output has not opened yet" : "");
 }
 
 function addressOf(output) { return output.outputKey ?? output.id; }

package/services/encode/EncodeOrchestrator.js

@@ -145,6 +145,30 @@ export class EncodeOrchestrator {
     this.#costs.noteStartup(measured);
   }
 
+  /**
+   * One output's original-input open, and what sizes its stretch: the one
+   * owner of start readings is this class's `RunCosts` (torrent-tv/meta#184).
+   *
+   * @param {string} address
+   * @param {number} seconds
+   */
+  noteOpen(address, seconds) {
+    this.#costs.noteOpen(address, seconds);
+  }
+
+  /**
+   * @param {string} address
+   * @returns {{ seconds: number, measured: boolean } | null}
+   */
+  openOf(address) {
+    return this.#costs.openOf(address);
+  }
+
+  /** @param {string} address */
+  forgetOpens(address) {
+    this.#costs.forgetOpens(address);
+  }
+
   /**
    * What a stop and a start cost on this host, in seconds, as the plan reads
    * them — for anyone else pricing a fresh encoder by the same figures.
@@ -289,41 +313,6 @@ export class EncodeOrchestrator {
     return coverage;
   }
 
-  /**
-   * Where a run started here must stop: the free stretch in front of it.
-   *
-   * Asked of the one map, brought up to date first. It used to be worked out by
-   * the session manager, which reached into this layer for the map and into the
-   * store for what was on the disk and put the two together itself — one fact
-   * with two owners and a third party carrying it between them, which is how the
-   * two came to disagree.
-   *
-   * @param {object} params
-   * @param {string} params.address
-   * @param {number} params.from - Where the run will start.
-   * @param {object | null} [params.exceptRun] - The run being replaced, whose
-   *   own claim is not somebody else's material.
-   * @param {number} [params.segmentCount] - The output's length, when known.
-   * @returns {number} The last number to work through, or `-1` for the end of
-   *   the film.
-   */
-  freeStretchEnd({ address, from, exceptRun = null, segmentCount = 0 }) {
-    if (!address) {
-      return -1;
-    }
-    if (segmentCount > 0) {
-      this.coverageOf(address).setSegmentCount(segmentCount);
-    }
-    const coverage = this.#upToDateCoverage(address);
-    const start = Math.max(0, from);
-    const free = coverage.freeRunFrom(start, exceptRun);
-    if (!Number.isFinite(free)) {
-      return -1;
-    }
-    const end = start + Math.max(1, free) - 1;
-    return segmentCount > 0 && end >= segmentCount - 1 ? -1 : end;
-  }
-
   /**
    * @param {string} address
    * @returns {import("../encode/EncodeRun.js").EncodeRun[]}
@@ -974,9 +963,11 @@ export class EncodeOrchestrator {
     // were two steps, and two owners each performed the second one.
     //
     // A run whose input is one copied stretch can make nothing past it, so it
-    // claims that stretch and not the plan's longer bound: the rest stays free
-    // for the next run (torrent-tv/meta#158). Every other run claims as before.
-    const through = Number.isInteger(run.inputThrough) ? run.inputThrough : to;
+    // claims that stretch — its own `to` — and not the plan's longer bound:
+    // the rest stays free for the next run (torrent-tv/meta#158). A run whose
+    // input grows a segment at a time claims the plan's bound, as it may reach
+    // it (torrent-tv/meta#184).
+    const through = run.originalInput ? run.to : to;
     this.coverageOf(address).claim(run, from, endOfRun({ from, to: through }));
   }
 

package/services/encode/EncodeRun.js

@@ -64,8 +64,14 @@ const MICROSECONDS_PER_SECOND = 1_000_000;
  *   it finished none.
  * @property {number} livedMs
  * @property {number | null} [firstNamedMs] - Spawn to the moment the encoder
- *   named its first closed piece: the open of its input plus that one piece's
- *   encoding, without the wait for the following piece that publication adds.
+ *   named its first closed piece, less the time it was paused before then: the
+ *   open of its input plus that one piece's encoding, without the wait for the
+ *   following piece that publication adds.
+ * @property {number | null} [firstNamedWallMs] - The same on the wall clock,
+ *   pauses included: what the viewer waited.
+ * @property {number | null} [firstOutputWallMs] - Spawn to the first output on
+ *   the wall clock; `firstOutputMs` leaves the pauses out.
+ * @property {number} [stoppedMs] - How long the run was paused in all.
  * @property {boolean} normal - Whether this ending is the expected one.
  * @property {string} lastError - The last thing ffmpeg said on stderr.
  * @property {string | null} provenName - The last piece this run named while it
@@ -166,6 +172,10 @@ export class EncodeRun {
    */
   #firstNamedAt = 0;
 
+  /** How long the run had been paused when each of the two above was taken. */
+  #stoppedBeforeFirstOutput = 0;
+  #stoppedBeforeFirstNamed = 0;
+
   /** The film made and the run's own working time, as of the last progress report. */
   #workSample = null;
 
@@ -569,6 +579,7 @@ export class EncodeRun {
       this.#produced.add(index);
       if (this.#firstOutputAt === 0) {
         this.#firstOutputAt = this.now();
+        this.#stoppedBeforeFirstOutput = this.clock.stoppedMs(this.#firstOutputAt);
       }
       if (this.#state === ENCODE_RUN_STATE.STARTING) {
         this.#transition(ENCODE_RUN_EVENT.FIRST_SEGMENT);
@@ -607,7 +618,10 @@ export class EncodeRun {
       }
       if (this.#publicationError) break;
       const index = this.indexOfName(name);
-      if (this.#firstNamedAt === 0 && Number.isInteger(index) && index >= this.from) this.#firstNamedAt = this.now();
+      if (this.#firstNamedAt === 0 && Number.isInteger(index) && index >= this.from) {
+        this.#firstNamedAt = this.now();
+        this.#stoppedBeforeFirstNamed = this.clock.stoppedMs(this.#firstNamedAt);
+      }
       const endOfWork = Number.isFinite(endOfRun(this)) ? this.to : this.lastSegmentIndex();
       // A packet crossing the final cut can flush a following file. It may
       // complete its predecessor, but no unadmitted interval can be published.
@@ -920,11 +934,18 @@ export class EncodeRun {
       // Null where the run was never told to stop, or never produced anything:
       // an absent measurement says so rather than reading as zero.
       dyingMs: this.#stopOrderedAt > 0 ? this.now() - this.#stopOrderedAt : null,
+      // Without the time this run was paused before then: a pause is our own
+      // decision, not what starting cost (torrent-tv/meta#184). The wall-clock
+      // figures beside them are what the viewer waited.
       firstOutputMs:
         this.#firstOutputAt > 0 && this.#startedAt > 0
-          ? this.#firstOutputAt - this.#startedAt
+          ? this.#firstOutputAt - this.#startedAt - this.#stoppedBeforeFirstOutput
           : null,
-      firstNamedMs: this.#firstNamedAt > 0 && this.#startedAt > 0 ? this.#firstNamedAt - this.#startedAt : null,
+      firstOutputWallMs: this.#firstOutputAt > 0 && this.#startedAt > 0 ? this.#firstOutputAt - this.#startedAt : null,
+      firstNamedMs: this.#firstNamedAt > 0 && this.#startedAt > 0
+        ? this.#firstNamedAt - this.#startedAt - this.#stoppedBeforeFirstNamed : null,
+      firstNamedWallMs: this.#firstNamedAt > 0 && this.#startedAt > 0 ? this.#firstNamedAt - this.#startedAt : null,
+      stoppedMs: this.clock.stoppedMs(),
       normal: ending === ENCODE_EXIT.COMPLETE,
       lastError: this.lastError
     };