@torrent-tv/proxy 2.97.11 → 2.97.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.97.11",
+  "version": "2.97.12",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/routes/encode-input/get.js

@@ -1,7 +1,7 @@
 import { parseRange } from "../../utils/parse-range.js";
 
 /** FFmpeg reads only original byte ranges already owned by its live run. */
-export function handleEncodeInputGet(req, reply, { inputOf, refused = () => {} }) {
+export async function handleEncodeInputGet(req, reply, { inputOf, refused = () => {} }) {
   const address = req.raw?.socket?.remoteAddress;
   if (!["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(address)) return reply.code(403).send();
   const token = Number(req.params.token), fileIndex = Number(req.params.fileIndex);
@@ -29,7 +29,15 @@ export function handleEncodeInputGet(req, reply, { inputOf, refused = () => {} }
   // and a byte past it is refused, which from 7.1 on ends the run: every read
   // FFmpeg makes, its stream search at open included, must be named by the
   // container (torrent-tv/meta#165).
-  const held = input.read(fileIndex, range.start, range.end, true);
+  // A read inside the stretch of film waits for its copy, which goes on while
+  // FFmpeg opens (`OriginalInput.js`); the answer is then whole, as any other.
+  const held = await input.read(fileIndex, range.start, range.end, true);
+  if (!held && input.copyFailed?.()) {
+    // The copy of the stretch failed: a piece left the store between the
+    // check that it was there and the copy. The run ends on this refusal, and
+    // the failure is said where the copy is accounted for (`EncodeInputs.js`).
+    return reply.code(503).send({ error: "Original input bytes could not be copied." });
+  }
   if (!held) {
     // FFmpeg asked for a byte the run was not given: the container named too
     // little. Said, because FFmpeg itself reports only a failed seek or read.

package/server.js

@@ -716,7 +716,10 @@ export async function startProxyServer({
         if (original) {
           const ranges = await containerOrchestrator.inspect(params, "source-ranges");
           if (ranges.kind !== "result") return ranges;
+          // The torrent's piece is the unit its stretch is copied in, so a copy
+          // nobody needs any more stops at the next piece (`OriginalInput.js`).
           sources.push({ sourceKey: params.sourceKey, fileIndex, timeShiftSeconds,
+            pieceLength: torrentPool.knownTorrent(params.sourceKey)?.pieceLength ?? null,
             input: { ...ranges.value, original: true, selections: wanted.map(track => ({ track,
               index: track.declaredIndex ?? choices.find(choice => choice.type === track.type)?.index ?? 0 })) } });
           continue;

package/services/encode/EncodeInputs.js

@@ -187,6 +187,33 @@ export class EncodeInputs {
     this.#retryMemory();
   }
 
+  /**
+   * Withdraw what is being prepared or held for a start the plan no longer
+   * places: `starts` holds every start the last plan of this output placed or
+   * held back, and the number after each run it left alive. `retain` withdrew
+   * such a copy only once it was held, and only when a run was started; while
+   * it was being made it shared the torrent thread with the copy the viewer
+   * was waiting for. Field 2026-10-12: after a seek back, the next stretch of
+   * the run the seek had stopped went on being copied beside the one for the
+   * new position, 127.5 and 126.5 MB in 16 s each, and was withdrawn only once
+   * it was done (torrent-tv/meta#178).
+   *
+   * @param {object} output
+   * @param {Set<number>} starts
+   */
+  withdrawUnplaced(output, starts) {
+    for (const [key, request] of this.#requests) {
+      if (request.output !== output || request.result?.kind === "terminal" || starts.has(request.from)) continue;
+      this.#note(request, "withdrawn: the plan places no run there");
+      this.#requests.delete(key);
+      this.#wanted.delete(key);
+      // A copy already answered may still be copying its stretch; released,
+      // it stops at the next piece.
+      request.result?.release?.();
+    }
+    this.#retryMemory();
+  }
+
   /**
    * The input of a run starting at `from`, once it is prepared.
    *
@@ -306,7 +333,9 @@ export class EncodeInputs {
       if (result.kind === "result" && result.to > request.to) result.to = Math.max(request.from, request.to);
       request.result = result;
       this.#note(request, result.kind === "result" ? `ready #${result.from}..#${result.to}, ${result.bytes ?? "?"} bytes` +
-        (Number.isFinite(result.copyMs) ? `, copied in ${result.copyMs}ms` : "")
+        (Number.isFinite(result.copyMs) ? (result.deferredBytes > 0
+          ? `, opening part copied in ${result.copyMs}ms, ${result.deferredBytes} bytes of the stretch copied while the run opens`
+          : `, copied in ${result.copyMs}ms`) : "")
         : `${result.kind}${result.reason ? ` ${result.reason}` : ""}${Number.isFinite(result.bytes) ? ` ${result.bytes} bytes` : ""}`,
       describeTiming(request.timing, this.#now()));
       if (result.kind === "result" || result.kind === "terminal") this.#changed(request.output, result);
@@ -444,7 +473,16 @@ export class EncodeInputs {
     if (result.kind === "needs-memory") this.#wanted.set(request.key, { wanted: minimum, required: minimum });
     if (result.kind === "result") {
       Object.assign(result, { from, to: end });
-      if (result.copyMs > 0) keep(this.#copyRates, result.bytes / result.copyMs);
+      // The stretch is copied after the answer, while the run opens; the rate
+      // is the whole copy's, learned once it is done.
+      const copied = result.whole ?? Promise.resolve(true);
+      void copied.then(whole => {
+        if (whole && result.copyMs > 0) keep(this.#copyRates, result.bytes / result.copyMs);
+        this.#log(`encode input #${from}..#${end} of ${addressOf(output)}: ` + (whole
+          ? `stretch copied, ${result.bytes} bytes in ${result.copyMs}ms of copying since it was admitted`
+          : result.copyFailed?.() ? "the stretch could not be copied: a piece it needs is no longer held"
+            : "released before its stretch was copied"));
+      });
     }
     return result;
   }

package/services/encode/EncodeOrchestrator.js

@@ -228,6 +228,7 @@ export class EncodeOrchestrator {
     segmentCoverage = null,
     planSoon = null,
     inputDemandChanged = null,
+    inputsUnplaced = null,
     describeWaits = null,
     admission = null,
     logger,
@@ -264,6 +265,9 @@ export class EncodeOrchestrator {
     // loop that calls it. Absent in a test, where the clock is the test's own.
     this.planSoon = typeof planSoon === "function" ? planSoon : () => undefined;
     this.inputDemandChanged = typeof inputDemandChanged === "function" ? inputDemandChanged : () => undefined;
+    // Told, after each plan of an output, where that plan places runs and
+    // where the runs it kept end, so inputs prepared for anywhere else go back.
+    this.inputsUnplaced = typeof inputsUnplaced === "function" ? inputsUnplaced : () => undefined;
   }
 
   /**
@@ -882,6 +886,20 @@ export class EncodeOrchestrator {
       coverage.claim(action.run, action.from, endOfRun({ from: action.from, to: action.to }));
     }
 
+    // AN INPUT IS PREPARED ONLY FOR A START THIS PLAN PLACES: one it starts or
+    // holds back (a copied picture's input may begin one number earlier), or
+    // the number after a run it left alive, whose next stretch is copied while
+    // it works. Any other copy shares the torrent thread with the one a viewer
+    // waits for (torrent-tv/meta#178).
+    const starts = new Set();
+    for (const action of actions) {
+      if (action.type !== "start" && action.type !== "move") continue;
+      starts.add(action.from);
+      starts.add(action.from - 1);
+    }
+    for (const run of this.runsOn(address)) if (run.isAlive) starts.add(run.to + 1);
+    this.inputsUnplaced(address, starts);
+
     // NOBODY IS MAKING WHAT SOMEBODY IS WAITING FOR. Said here, with the numbers
     // the decision was taken from, because it is the one state in which a viewer
     // waits for ever and every line above it reads as a healthy proxy.

package/services/encode/EncodeRuns.js

@@ -1178,7 +1178,16 @@ export class EncodeRuns {
       // 48.9 s after run #1..#32 ended, then the start of #33..#892, and the
       // viewer stood 69 s (torrent-tv/meta#166). Asked now, it is copied while
       // this run works and is ready when the plan places the next run there.
-      if (inputEnd < mayReach) this.#host.encodeInputs?.take(session, inputEnd + 1, mayReach);
+      // It waits for this run's own stretch to be copied: that copy goes on
+      // while the run opens and FFmpeg's reads wait for it, and a second copy
+      // beside it on the same thread would slow it (torrent-tv/meta#178).
+      if (inputEnd < mayReach) {
+        const askNext = () => {
+          if (this.isLive(session) && run.isAlive) this.#host.encodeInputs?.take(session, inputEnd + 1, mayReach);
+        };
+        if (admittedInput.whole) void admittedInput.whole.then(whole => { if (whole) askNext(); });
+        else askNext();
+      }
     } else if (admittedInput) {
       run.admittedInputKeys = new Map([[safeIndex, inputKey]]);
       let nextIndex = startIndex + 1;
@@ -1252,10 +1261,14 @@ export class EncodeRuns {
     const pieceSeconds = Number(grid?.[run.from + 1]) - Number(grid?.[run.from]);
     const speed = run.speedX > 0 ? run.speedX : this.#host.encodeOrchestrator.speedOn?.(session.outputKey) ?? 0;
     if (!Number.isFinite(ended.firstNamedMs) || !(pieceSeconds > 0) || !(speed > 0)) return;
-    const openSeconds = Math.max(0, ended.firstNamedMs / 1000 - pieceSeconds / speed);
+    // FFmpeg's reads may have waited for the stretch being copied; that wait
+    // is the copy's, and the stretch is sized from the open (torrent-tv/meta#178).
+    const waitedMs = Number.isFinite(ended.firstNamedWallMs) ? run.originalInput?.waitedBefore?.(ended.firstNamedWallMs) ?? 0 : 0;
+    const openSeconds = Math.max(0, (ended.firstNamedMs - waitedMs) / 1000 - pieceSeconds / speed);
     this.#host.encodeOrchestrator.noteOpen?.(session.outputKey ?? "", openSeconds);
     this.#host.logger.info(`encode input of run #${run.from}..#${run.to} on ${session.outputKey}: opened in ` +
-      `${openSeconds.toFixed(2)}s (first piece named at ${ended.firstNamedMs}ms, ${pieceSeconds.toFixed(3)}s of film at ${speed.toFixed(2)}x)`);
+      `${openSeconds.toFixed(2)}s (first piece named at ${ended.firstNamedMs}ms, ${pieceSeconds.toFixed(3)}s of film at ${speed.toFixed(2)}x` +
+      `${waitedMs > 0 ? `, ${waitedMs}ms of it waiting for the stretch to be copied, not counted` : ""})`);
   }
 
   /**

package/services/encode/OriginalInput.js

@@ -3,6 +3,15 @@ import { createHash } from "node:crypto";
 /** Retain complete original byte ranges under the encoder's input allowance.
  * The available-only reader acquires complete pieces atomically before copying;
  * owned buffers then outlive torrent eviction without keeping duplicate pins.
+ *
+ * THE RUN STARTS ONCE WHAT FFMPEG OPENS WITH IS COPIED. Each source's largest
+ * range — the stretch of film — is copied after the rest, and the answer comes
+ * back before it: FFmpeg's start and its reads of the header and the index then
+ * overlap that copy, and a read inside the stretch waits for it. Field
+ * 2026-10-12, after a seek back: 126.5 MB copied in 16.3 s before the encoder
+ * was started, then 10.1 s of its own open, and the viewer stood 33 s
+ * (torrent-tv/meta#178). `whole` settles when everything is copied — true — or
+ * when the copy failed or the input was released first — false.
  */
 export async function admitOriginalInput({ sources, reserve, readRanges, now = Date.now }) {
   const bytes = originalInputBytes(sources);
@@ -15,27 +24,84 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
   // How long the copy itself took, apart from any wait for memory or bytes
   // before it: the figure a run's stretch is sized from, read where the copy is.
   let copyMs = 0;
+  // Per source, the range copied after the answer: its position in the
+  // source's ranges, and the copy's promise once it has begun.
+  const later = [];
+  const read = async (source, ranges) => {
+    const startedAt = now();
+    const buffers = await readRanges(source, ranges, bytes);
+    copyMs += Math.max(0, now() - startedAt);
+    if (buffers === null) return null;
+    if (!Array.isArray(buffers) || buffers.length !== ranges.length || buffers.some((buffer, index) =>
+      !Buffer.isBuffer(buffer) || buffer.length !== ranges[index][1] - ranges[index][0] + 1)) {
+      throw new Error("Original input reader returned incomplete ranges.");
+    }
+    return buffers;
+  };
   try {
     for (const source of sources) {
-      const startedAt = now();
-      const buffers = await readRanges(source, source.input.ranges, bytes);
-      copyMs += Math.max(0, now() - startedAt);
-      if (buffers === null) return { kind: "needs-bytes", sourceKey: source.sourceKey, fileIndex: source.fileIndex, ranges: source.input.ranges };
-      if (!Array.isArray(buffers) || buffers.length !== source.input.ranges.length || buffers.some((buffer, index) =>
-        !Buffer.isBuffer(buffer) || buffer.length !== source.input.ranges[index][1] - source.input.ranges[index][0] + 1)) {
-        throw new Error("Original input reader returned incomplete ranges.");
-      }
+      const { ranges } = source.input;
+      const largest = ranges.reduce((best, range, index) =>
+        range[1] - range[0] > ranges[best][1] - ranges[best][0] ? index : best, 0);
+      const deferred = ranges.length > 1 ? largest : -1;
+      const first = ranges.filter((_, index) => index !== deferred);
+      const buffers = await read(source, first);
+      if (buffers === null) return { kind: "needs-bytes", sourceKey: source.sourceKey, fileIndex: source.fileIndex, ranges };
       // THE INPUT IS NAMED BY WHAT IT IS, NOT BY ITS BYTES. Every source is a
       // torrent whose pieces are verified before they are stored, and a read
       // gives every byte or none, so the same source, file, ranges and
       // selections are the same bytes. Hashing the bytes as well took 13.45 s
       // of 180 s of the main thread on a Matroska film whose picture and sound
       // copy the same ~30 MB a piece (torrent-tv/meta#178).
-      hash.update(JSON.stringify([source.sourceKey, source.fileIndex, source.input.ranges, source.input.selections]));
-      held.set(source.fileIndex, { source, buffers });
+      hash.update(JSON.stringify([source.sourceKey, source.fileIndex, ranges, source.input.selections]));
+      const entry = { source, buffers: ranges.map((_, index) => index === deferred ? null : buffers[index - (deferred >= 0 && index > deferred ? 1 : 0)]) };
+      held.set(source.fileIndex, entry);
+      if (deferred >= 0) later.push({ entry, position: deferred });
     }
     let released = false;
+    let failed = false;
     retained = true;
+    // A stretch is copied one piece of the torrent at a time, so a release
+    // stops it at the next piece: the thread it is copied on serves every other
+    // read too, and a copy nobody will use went on to its end (torrent-tv/meta#178).
+    const copyLater = async job => {
+      const { source } = job.entry;
+      const [first, last] = source.input.ranges[job.position];
+      const unit = Number(source.pieceLength) > 0 ? Number(source.pieceLength) : last - first + 1;
+      const target = Buffer.allocUnsafeSlow(last - first + 1);
+      try {
+        for (let at = first; at <= last; at += unit) {
+          if (released) return false;
+          const buffers = await read(source, [[at, Math.min(last, at + unit - 1)]]);
+          if (buffers === null) { failed = true; return false; }
+          buffers[0].copy(target, at - first);
+        }
+      } catch {
+        failed = true;
+        return false;
+      }
+      if (released) return false;
+      job.entry.buffers[job.position] = target;
+      return true;
+    };
+    // The stretches, one source after another: they share one reader, and the
+    // first source's stretch is the one FFmpeg reaches first.
+    const whole = (async () => {
+      for (const job of later) {
+        if (released) return false;
+        job.copy = job.copy ?? copyLater(job);
+        if (!await job.copy) return false;
+      }
+      return !released;
+    })();
+    // A read of a stretch waits for its copy, which begins at once for the
+    // stretch asked for even when another is still being copied before it.
+    const copied = (entry, position) => {
+      const job = later.find(candidate => candidate.entry === entry && candidate.position === position);
+      if (!job) return Promise.resolve(entry.buffers[position] !== null);
+      job.copy = job.copy ?? copyLater(job);
+      return job.copy;
+    };
     // What FFmpeg asked of this input, for the run's own account of where its
     // time went: field 2026-10-08, a run took 6-9 s for a piece FFmpeg makes in
     // 1.6 s from a file, and nothing said whether it was waiting on its input.
@@ -45,12 +111,38 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
     // whether all of it was (torrent-tv/meta#177).
     /** @type {{ fileIndex: number, start: number, end: number, written: number, handedOver: boolean, at: number }[]} */
     const answers = [];
+    // When FFmpeg's reads waited for a stretch still being copied.
+    /** @type {{ from: number, to: number | null }[]} */
+    const waits = [];
+    const answer = (entry, position, start, stop) => {
+      const now = Date.now();
+      reads.count += 1;
+      reads.bytes += stop - start + 1;
+      reads.firstAt ||= now;
+      reads.lastAt = now;
+      const [from] = entry.source.input.ranges[position];
+      return { length: entry.source.input.fileLength, end: stop, bytes: entry.buffers[position].subarray(start - from, stop - from + 1) };
+    };
     return {
-      kind: "result", original: true, sources, bytes, copyMs, fingerprint: hash.digest("hex"),
+      kind: "result", original: true, sources, bytes, fingerprint: hash.digest("hex"),
+      // What the copy took so far; final once `whole` has settled.
+      get copyMs() { return copyMs; },
+      whole,
+      // Bytes of the stretches still to be copied when the answer was given.
+      deferredBytes: later.reduce((sum, job) => {
+        const [first, last] = job.entry.source.input.ranges[job.position];
+        return sum + last - first + 1;
+      }, 0),
+      copyFailed: () => failed,
       // No track's own end: the container names the bytes an interval needs,
       // not where each track stops. The final piece is judged by the input
       // reaching the end of the file instead (`piece-completeness.js`).
       tracks: sources.flatMap(source => (source.input.selections ?? []).map(selection => ({ track: selection.track }))),
+      /**
+       * The answer to a read from `start`, or null when it lies outside what
+       * the run holds. Inside a stretch still being copied the answer is a
+       * promise of the same, settled once the stretch is copied.
+       */
       read(fileIndex, start, end, partial = false) {
         if (released) return null;
         const entry = held.get(fileIndex);
@@ -61,12 +153,22 @@ export async function admitOriginalInput({ sources, reserve, readRanges, now = D
         const position = ranges.findIndex(([from, to]) => from <= start && (partial ? start : stop) <= to);
         if (position < 0) return null;
         stop = Math.min(stop, ranges[position][1]);
-        const now = Date.now();
-        reads.count += 1;
-        reads.bytes += stop - start + 1;
-        reads.firstAt ||= now;
-        reads.lastAt = now;
-        return { length: fileLength, end: stop, bytes: entry.buffers[position].subarray(start - ranges[position][0], stop - ranges[position][0] + 1) };
+        if (entry.buffers[position] !== null) return answer(entry, position, start, stop);
+        const wait = { from: Date.now(), to: null };
+        waits.push(wait);
+        return copied(entry, position).then(ok => {
+          wait.to = Date.now();
+          return ok && !released ? answer(entry, position, start, stop) : null;
+        });
+      },
+      /**
+       * Wall-clock milliseconds FFmpeg's reads spent waiting for a stretch to
+       * be copied before `until`. A run's open is measured from its first piece,
+       * and this wait is the copy's, not the open's: counted in, the stretch
+       * sized from the open would grow by its own copy time.
+       */
+      waitedBefore(until) {
+        return waits.reduce((sum, wait) => sum + Math.max(0, Math.min(wait.to ?? until, until) - wait.from), 0);
       },
       lengthOf: fileIndex => held.get(fileIndex)?.source.input.fileLength ?? null,
       reads: () => ({ ...reads, answers: describeAnswers(answers) }),

package/services/server/wire-outputs.js

@@ -959,6 +959,9 @@ export function wireOutputs({
     inputDemandChanged: (address, windows) => {
       for (const output of parts.outputs.outputsOn(address)) parts.encodeInputs?.retain(output, windows);
     },
+    inputsUnplaced: (address, starts) => {
+      for (const output of parts.outputs.outputsOn(address)) parts.encodeInputs?.withdrawUnplaced(output, starts);
+    },
     admission: parts.admission,
     // What the viewers actually waited for, by band. The ledger is the
     // priority layer's; the encoding is handed a way to ask it.