@torrent-tv/proxy 2.91.0 → 2.92.0

package/services/encode/AdmittedInput.js

@@ -0,0 +1,136 @@
+import { createHash } from "node:crypto";
+import { matroskaInput, matroskaTrackIdentity } from "./MatroskaInput.js";
+import { pcmNormalization, normalizePcmSlices } from "./pcm-normalization.js";
+
+/** Acquire complete source ranges before any process can consume the input. */
+export async function admitInput({ sources, reserve, readRanges }) {
+  if (!Array.isArray(sources) || sources.length === 0 || typeof reserve !== "function" || typeof readRanges !== "function") {
+    throw new TypeError("Admission requires resolved source inputs, memory reservation and an available-only reader.");
+  }
+  const bytes = sources.reduce((total, source) => total + source.input.ranges.reduce((sum, [start, end]) => sum + end - start + 1, 0) +
+    source.input.tracks.reduce((sum, input) => sum + (pcmNormalization(input.track)
+      ? input.packets.reduce((size, packet) => size + packet.ranges.reduce((n, [start, end]) => n + end - start + 1, 0), 0) : 0), 0), 0);
+  if (!Number.isSafeInteger(bytes) || bytes <= 0) throw new TypeError("Input byte count must be a positive safe integer.");
+  const release = await reserve(bytes);
+  if (typeof release !== "function") return release?.kind === "terminal" ? release : { kind: "needs-memory", bytes };
+  let retained = false;
+  try {
+    const hash = createHash("sha256");
+    const tracks = [];
+    const held = new Map();
+    const normalized = new Map();
+    for (const source of sources) {
+      const buffers = await readRanges(source, source.input.ranges, bytes);
+      if (buffers === null) return { kind: "needs-bytes", sourceKey: source.sourceKey, fileIndex: source.fileIndex, ranges: source.input.ranges };
+      if (!Array.isArray(buffers) || buffers.length !== source.input.ranges.length || buffers.some((buffer, index) =>
+        !Buffer.isBuffer(buffer) || buffer.length !== source.input.ranges[index][1] - source.input.ranges[index][0] + 1)) {
+        throw new Error("Available-only input reader returned incomplete ranges.");
+      }
+      hash.update(JSON.stringify(source.input.ranges));
+      for (const buffer of buffers) {
+        hash.update(String(buffer.length));
+        hash.update(":");
+        hash.update(buffer);
+      }
+      for (const input of source.input.tracks) {
+        const shifted = { ...input,
+          ...(Number.isFinite(input.sourceEndSeconds) ? { sourceEndSeconds: input.sourceEndSeconds - (source.timeShiftSeconds ?? 0) } : {}),
+          packets: input.packets.map(packet => ({ ...packet,
+          pts: packet.pts - (source.timeShiftSeconds ?? 0),
+          ...(packet.dts === undefined ? {} : { dts: packet.dts - (source.timeShiftSeconds ?? 0) }) })) };
+        tracks.push(shifted);
+        hash.update(matroskaTrackIdentity(input.track));
+        hash.update(JSON.stringify(shifted.packets.map(packet => [packet.pts, packet.dts ?? null,
+          packet.duration, packet.keyframe === true, packet.ranges, packet.discardPaddingSeconds ?? 0,
+          packet.bitOffset ?? null, packet.bitLength ?? null])));
+        held.set(shifted, { ranges: source.input.ranges, buffers });
+      }
+    }
+    let released = false;
+    const originSeconds = tracks.reduce((origin, input) => input.packets.reduce((value, packet) => Math.min(value, packet.pts), origin), Infinity);
+    if (!Number.isFinite(originSeconds)) throw new Error("Admitted input contains no packets.");
+    const packetSlices = (input, packet) => {
+      if (released) throw new Error("Admitted input has been released.");
+      const source = held.get(input);
+      if (!source) throw new Error("Packet does not belong to this admitted input.");
+      return packet.ranges.map(([start, end]) => {
+        const position = source.ranges.findIndex(([from, to]) => from <= start && end <= to);
+        if (position < 0) throw new Error("Packet is outside its admitted ranges.");
+        return source.buffers[position].subarray(start - source.ranges[position][0], end - source.ranges[position][0] + 1);
+      });
+    };
+    const readPacket = (input, packet) => {
+      const slices = packetSlices(input, packet);
+      return slices.length === 1 ? slices[0] : Buffer.concat(slices);
+    };
+    for (const input of tracks) {
+      for (const packet of input.packets) {
+        const slices = packetSlices(input, packet);
+        if (packet.expectedHash !== undefined) {
+          const payloadHash = createHash("sha256");
+          for (const slice of slices) payloadHash.update(slice);
+          if (payloadHash.digest("hex") !== packet.expectedHash) {
+            return { kind: "terminal", reason: "packet-address-does-not-match-payload", trackId: input.track.trackNumber,
+              ranges: packet.ranges };
+          }
+        }
+        const format = pcmNormalization(input.track);
+        if (format) normalized.set(packet, normalizePcmSlices(slices, format));
+      }
+    }
+    retained = true;
+    return {
+      kind: "result", bytes, fingerprint: hash.digest("hex"), originSeconds, tracks, readPacket,
+      stream: options => matroskaInput({ tracks,
+        readPacket: (input, packet) => normalized.has(packet) ? [normalized.get(packet)] : packetSlices(input, packet),
+        originSeconds, ...options }),
+      release: () => {
+        if (released) return;
+        released = true;
+        held.clear();
+        normalized.clear();
+        release();
+      }
+    };
+  } finally {
+    if (!retained) release();
+  }
+}
+
+/** Respect stdin backpressure and release input on every completion path. */
+export async function writeAdmittedInput(input, stdin, { next = null } = {}) {
+  if (input?.kind !== "result") throw new TypeError("Only complete input may be written to an encoder.");
+  const originSeconds = input.originSeconds;
+  const declarations = JSON.stringify(input.tracks.map(input => matroskaTrackIdentity(input.track)));
+  const seen = new Map();
+  let current = input, includeHeader = true;
+  try {
+    while (current) {
+    if (JSON.stringify(current.tracks.map(input => matroskaTrackIdentity(input.track))) !== declarations) {
+      throw new Error("Consecutive encoder input changed its declared tracks.");
+    }
+    for (let index = 0; index < current.tracks.length; index++) {
+      const end = current.tracks[index].sourceEndSeconds;
+      if (Number.isFinite(end)) input.tracks[index].sourceEndSeconds = end;
+    }
+    for await (const chunk of current.stream({ originSeconds, includeHeader, seen })) {
+      await new Promise((resolve, reject) => {
+        const failed = error => { stdin.off("error", failed); reject(error); };
+        stdin.once("error", failed);
+        stdin.write(chunk, error => {
+          stdin.off("error", failed);
+          if (error) reject(error);
+          else resolve();
+        });
+      });
+    }
+    current.release();
+    current = next ? await next() : null;
+    includeHeader = false;
+    }
+    stdin.end();
+  } finally {
+    current?.release();
+    input.release();
+  }
+}

package/services/encode/CushionReport.js

@@ -42,13 +42,11 @@ export class CushionReport {
   /** What this reads and asks of the rest of the proxy, and nothing else. @type {object} */
   #host;
 
-  /** Files whose spare soundtracks have been fetched whole, once each. @type {Set<string>} */
-  #spareSoundtracksFetched = new Set();
   #lastSaidAt = new WeakMap();
   #lookAheadDisagreementSince = new WeakMap();
 
   /**
-   * @param {object} host - `viewerSecondsOn`, `viewersOf`, `SourceFiles`, `logger`, `producedNumbers`, `encodeRuns`, `fetchWholeFile`, `getCachedAudioTracks`, `hostLoad`, `lookaheadSeconds`, `outputTimes`, `outputs`
+   * @param {object} host - `viewerSecondsOn`, `viewersOf`, `SourceFiles`, `logger`, `producedNumbers`, `encodeRuns`, `hostLoad`, `lookaheadSeconds`, `outputTimes`, `outputs`
    */
   constructor(host) {
     this.#host = host;
@@ -201,79 +199,6 @@ export class CushionReport {
         `${viewers} viewer(s) holding up to ` +
         `${deepestBuffer === null ? "?" : deepestBuffer.toFixed(1)}s`
     );
-    this.#fetchSpareSoundtracks(session, aheadOfPicture);
-  }
-
-  /**
-   * Fetch the soundtracks that ship beside this picture, whole, while the swarm
-   * has capacity to spare.
-   *
-   * WHY IT WAITS FOR THE CUSHION. A soundtrack nobody has chosen is worth having
-   * on disk — it is a twentieth of the picture (30 MB against 566 MB on the
-   * field torrent) and having it makes every later switch instant instead of
-   * paying for its first pieces. But fetching it takes swarm capacity from the
-   * picture, and there is exactly one moment when that capacity is demonstrably
-   * spare: when the encoder is already as far ahead of the viewer as it is
-   * allowed to get. That is not a guess about the swarm — it is the measurement
-   * the line above just printed.
-   *
-   * WHY IT IS A READ AND NOT A SELECTION. `file.select()` claims every piece of
-   * a file at once, and `#syncSelections` in `torrent/torrent-pool.js` records what that
-   * cost when it was done alongside the readers' own windows: a claim covering
-   * everything always outranked the window, and a seek to 89.1% of a 4.7 GB film
-   * waited 93 s while the swarm fetched 2.47 GB in file order. So this goes
-   * through the same bounded read the edge warm-up uses, which claims a moving
-   * window like any other reader and gives it back when it ends.
-   *
-   * Once per file, and only for a soundtrack in a file of its own — the
-   * picture's own tracks are already in the bytes being played.
-   *
-   * @param {HlsSession} session
-   * @param {number} aheadOfPicture - Seconds of film ready ahead of the viewer.
-   * @returns {void}
-   */
-  #fetchSpareSoundtracks(session, aheadOfPicture) {
-    if (typeof this.#host.fetchWholeFile !== "function") {
-      return;
-    }
-    // The encoder is held at this distance and no further, so reaching it is the
-    // signal that nothing more is being asked of the swarm on the picture's
-    // behalf.
-    if (!(aheadOfPicture >= this.#host.lookaheadSeconds)) {
-      return;
-    }
-    const inventory = this.#host.getCachedAudioTracks?.({
-      sourceKey: session.file.sourceKey,
-      fileIndex: session.file.fileIndex
-    }) ?? [];
-    const wanted = new Set(
-      inventory
-        .filter((entry) => entry?.kind === "sidecar" && Number.isInteger(entry.fileIndex))
-        .map((entry) => entry.fileIndex)
-    );
-    for (const fileIndex of wanted) {
-      const key = this.#host.SourceFiles.keyFor(session.file.sourceKey, fileIndex);
-      if (this.#spareSoundtracksFetched.has(key)) {
-        continue;
-      }
-      this.#spareSoundtracksFetched.add(key);
-      this.#host.logger.info(
-        `transcode ${session.id.slice(0, 8)} the picture is ${Math.round(aheadOfPicture)}s ahead of ` +
-          `the viewer, so file ${fileIndex} — a soundtrack beside it — is fetched whole now; ` +
-          "a switch to it will not wait for the swarm"
-      );
-      // Not awaited: nothing depends on it finishing, and a failure costs only
-      // that the switch pays for its own pieces, as it did before this existed.
-      Promise.resolve(this.#host.fetchWholeFile({ sourceKey: session.file.sourceKey, fileIndex })).catch(
-        (error) => {
-          this.#host.logger.info(
-            `transcode: fetching soundtrack file ${fileIndex} whole failed ` +
-              `(${error instanceof Error ? error.message : String(error)}) — ` +
-              "it will be read when it is played"
-          );
-        }
-      );
-    }
   }
 
   /**

package/services/encode/EncodeInputs.js

@@ -0,0 +1,142 @@
+import { admitInput } from "./AdmittedInput.js";
+
+/** Event-driven preparation; a synchronous run request can take only ready input. */
+export class EncodeInputs {
+  #requests = new Map();
+  #revision = 0;
+  #held = 0;
+  #allowed = 0;
+  #wanted = new Map();
+  #resolve;
+  #read;
+  #revise;
+  #changed;
+  #failed;
+  #capacity;
+  #urgent;
+
+  constructor({ resolve, readRanges, reviseBudget, changed, failed, capacity = () => null, urgent = () => true }) {
+    this.#resolve = resolve;
+    this.#read = readRanges;
+    this.#revise = reviseBudget;
+    this.#changed = changed;
+    this.#failed = failed;
+    this.#capacity = capacity;
+    this.#urgent = urgent;
+  }
+
+  held() { return this.#held; }
+  wanted() { return this.#held + [...this.#wanted.values()].reduce((sum, bytes) => sum + bytes, 0); }
+  allow(bytes) {
+    const changed = this.#allowed !== Math.max(0, bytes);
+    this.#allowed = Math.max(0, bytes);
+    if (changed) this.#retryMemory();
+  }
+
+  #retryMemory() {
+    for (const request of this.#requests.values()) {
+      if (!request.pending && request.result?.kind === "needs-memory") this.#prepare(request);
+    }
+  }
+
+  failureOf(output) {
+    for (const request of this.#requests.values()) {
+      if (request.output === output && request.result?.kind === "terminal") return request.result;
+    }
+    return null;
+  }
+
+  retain(output, windows) {
+    for (const [key, request] of this.#requests) {
+      if (request.output !== output || windows.some(window => request.from <= window.to && request.to >= window.from)) continue;
+      this.#requests.delete(key);
+      this.#wanted.delete(key);
+      request.result?.release?.();
+    }
+    this.#retryMemory();
+  }
+
+  take(output, from, to) {
+    const key = `${output.outputKey ?? output.id}:${from}:${to}`;
+    const existing = this.#requests.get(key);
+    if (existing?.result?.kind === "result") {
+      this.#requests.delete(key);
+      return existing.result;
+    }
+    if (existing?.pending || existing?.revision === this.#revision || existing?.result?.kind === "terminal") return null;
+    const request = existing ?? { output, from, to, key };
+    this.#requests.set(key, request);
+    this.#prepare(request);
+    return null;
+  }
+
+  async acquire(output, from, to) {
+    const key = `${output.outputKey ?? output.id}:${from}:${to}`;
+    const ready = this.take(output, from, to);
+    if (ready) return ready;
+    await this.#requests.get(key)?.promise;
+    const request = this.#requests.get(key);
+    if (request?.result?.kind !== "result") return null;
+    return this.take(output, from, to);
+  }
+
+  bytesChanged() {
+    this.#revision++;
+    for (const request of this.#requests.values()) {
+      if (!request.pending && request.result?.kind !== "result" && request.result?.kind !== "terminal") this.#prepare(request);
+    }
+  }
+
+  forget(output) {
+    for (const [key, request] of this.#requests) {
+      if (request.output !== output) continue;
+      this.#requests.delete(key);
+      this.#wanted.delete(key);
+      request.result?.release?.();
+    }
+  }
+
+  #prepare(request) {
+    request.pending = true;
+    request.revision = this.#revision;
+    request.promise = (async () => {
+      const resolved = await this.#resolve(request.output, request.from, request.to);
+      if (this.#requests.get(request.key) !== request) return;
+      const result = resolved.kind === "result" ? await admitInput({
+        sources: resolved.sources,
+        readRanges: this.#read,
+        reserve: async bytes => {
+          this.#wanted.set(request.key, bytes);
+          await this.#revise();
+          const capacity = this.#capacity();
+          if (capacity !== null && bytes > capacity && this.#urgent(request.output, request.from)) return {
+            kind: "terminal", reason: "source-input-exceeds-memory-capacity", bytes, capacity
+          };
+          if (this.#requests.get(request.key) !== request || this.#allowed - this.#held < bytes) return null;
+          this.#wanted.delete(request.key);
+          this.#held += bytes;
+          let released = false;
+          return () => {
+            if (!released) {
+              released = true; this.#held -= bytes;
+              queueMicrotask(() => this.#retryMemory());
+            }
+          };
+        }
+      }) : resolved;
+      if (this.#requests.get(request.key) !== request) { result.release?.(); return; }
+      request.result = result;
+      if (result.kind === "result" || result.kind === "terminal") this.#changed(request.output, result);
+    })().catch(error => {
+      if (this.#requests.get(request.key) === request) {
+        request.result = { kind: "terminal", reason: "input-preparation-failed", message: error.message };
+        this.#failed(request.output, error);
+      }
+    }).finally(() => {
+      request.pending = false;
+      if (request.result?.kind !== "needs-memory") this.#wanted.delete(request.key);
+      if (this.#requests.get(request.key) === request && request.revision !== this.#revision &&
+        request.result?.kind !== "result" && request.result?.kind !== "terminal") this.#prepare(request);
+    });
+  }
+}

package/services/encode/EncodeOrchestrator.js

@@ -167,6 +167,7 @@ export class EncodeOrchestrator {
     startingSpeedFor = () => 0,
     segmentCoverage = null,
     planSoon = null,
+    inputDemandChanged = null,
     describeWaits = null,
     admission = null,
     logger,
@@ -202,6 +203,7 @@ export class EncodeOrchestrator {
     // Asked for, not commanded: this class decides, and something else owns the
     // loop that calls it. Absent in a test, where the clock is the test's own.
     this.planSoon = typeof planSoon === "function" ? planSoon : () => undefined;
+    this.inputDemandChanged = typeof inputDemandChanged === "function" ? inputDemandChanged : () => undefined;
   }
 
   /**
@@ -536,6 +538,7 @@ export class EncodeOrchestrator {
    */
   notePriorityMap(address, zones) {
     this.#demand.state(address, zones);
+    this.inputDemandChanged(address, this.#demand.mapOn(address));
     // NOBODY IS COMING HERE, so what was remembered about this output's input
     // is about nobody. Kept, those three entries would stay for the life of the
     // process, and the wait they describe would greet whoever opens this output
@@ -994,7 +997,7 @@ export class EncodeOrchestrator {
     // is a name match: no stretch to search and no bytes to judge. Done for
     // every ending, the normal one included, since a run that finished cleanly
     // has nothing under a working name and the sweep then removes nothing.
-    this.#segmentCoverage?.clearUpAfter(ended.address, ended.from);
+    this.#segmentCoverage?.clearUpAfter(ended.address, ended.run?.makingTag ?? ended.from);
     this.coverageOf(ended.address).release(ended.run);
     const remaining = this.runsOn(ended.address).filter((run) => run !== ended.run);
     if (remaining.length === 0) {

package/services/encode/EncodeRun.js

@@ -199,7 +199,7 @@ export class EncodeRun {
    * @param {(reading: { speed: number, at: number }) => void} [params.onSpeedMeasured] -
    *   Told each time this run has measured its processing speed, once per
    *   reading.
-   * @param {(name: string) => string | null} [params.onClosed] - Called with the
+   * @param {(name: string, following: string | null) => string | null} [params.onClosed] - Called with the
    *   WORKING name of every piece the encoder has finished writing, as the
    *   encoder itself names it on its own channel, and answers with the name that
    *   piece is served under — because making it servable is a rename, and only
@@ -230,6 +230,7 @@ export class EncodeRun {
     encoder,
     from,
     to,
+    makingTag = String(from),
     buildArgs,
     spawn,
     logger,
@@ -251,6 +252,7 @@ export class EncodeRun {
     this.encoder = encoder;
     this.from = from;
     this.to = to;
+    this.makingTag = makingTag;
     this.buildArgs = buildArgs;
     this.spawnProcess = spawn;
     this.logger = logger;
@@ -452,6 +454,12 @@ export class EncodeRun {
    * @param {import("node:child_process").ChildProcess} process
    */
   #wire(process) {
+    // The process owns this listener even between admitted input writes.
+    process.stdin?.on("error", (error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      this.lastError ||= message;
+      this.logger.warn(`ffmpeg input #${this.from}..#${this.to} of ${this.address}: ${message}`);
+    });
     process.stdout?.on("data", (chunk) => this.#readProgress(String(chunk)));
     // THE CHANNEL THE ENCODER NAMES ITS FINISHED PIECES ON.
     //
@@ -567,14 +575,14 @@ export class EncodeRun {
       // A subsequent closed file proves the previous one reached a cut.
       // The last file can instead have been flushed by an input failure.
       if (this.#pendingClosed !== null) {
-        this.#publishClosed(this.#pendingClosed);
+        this.#publishClosed(this.#pendingClosed, name);
       }
       if (this.#publicationError) break;
       this.#pendingClosed = name;
     }
   }
 
-  #publishClosed(name) {
+  #publishClosed(name, following = null) {
       this.#measureSpeed();
       // ITS SERVED NAME, which is what whoever owns the disk gives it in answer.
       // ffmpeg writes a piece under a working name and reports that; the piece
@@ -582,7 +590,7 @@ export class EncodeRun {
       // name a request can actually ask for.
       let served;
       try {
-        served = this.onClosed(name, this.#closedTimings.get(name) ?? null);
+        served = this.onClosed(name, following, this.#closedTimings.get(name) ?? null);
       } catch (error) {
         this.#publicationError = error instanceof Error ? error.message : String(error);
         this.lastError = this.#publicationError;
@@ -594,6 +602,7 @@ export class EncodeRun {
       } finally {
         this.#closedTimings.delete(name);
       }
+
       if (!served) return false;
       if (!this.#stopping) {
         this.#provenName = served;