@torrent-tv/proxy 2.88.5 → 2.89.0

package/scripts/build-audio-samples.mjs

@@ -1,31 +1,31 @@
-/** Build-time only. No sample generation runs when the proxy starts. */
-import { spawnSync } from "node:child_process";
-import { mkdirSync, writeFileSync } from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const directory = fileURLToPath(new URL("../assets/calibration/audio/", import.meta.url));
-const formats = [
-  ["aac", "aac", "m4a"], ["ac3", "ac3", "ac3"], ["eac3", "eac3", "eac3"],
-  ["dts", "dca", "dts"], ["flac", "flac", "flac"], ["opus", "libopus", "opus"],
-  ["vorbis", "libvorbis", "ogg"], ["truehd", "truehd", "thd"], ["mp3", "libmp3lame", "mp3"],
-  ["mp2", "mp2", "mp2"], ["alac", "alac", "m4a"], ["pcm", "pcm_s16le", "wav"]
-];
-mkdirSync(directory, { recursive: true });
-const manifest = [];
-for (const [codec, encoder, extension] of formats) {
-  const file = `${codec}.${extension}`;
-  const result = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y",
-    "-f", "lavfi", "-i", "anoisesrc=color=pink:seed=1:sample_rate=48000",
-    "-t", "4", "-ac", "2", "-strict", "-2", "-c:a", encoder, path.join(directory, file)],
-  { encoding: "utf8", windowsHide: true });
-  if (result.status !== 0) throw new Error(`${codec}: ${result.stderr}`);
-  const probe = spawnSync("ffprobe", ["-v", "error", "-show_streams", "-show_format", "-of", "json",
-    path.join(directory, file)], { encoding: "utf8", windowsHide: true });
-  if (probe.status !== 0) throw new Error(`${codec}: ${probe.stderr}`);
-  const { streams, format } = JSON.parse(probe.stdout);
-  const stream = streams.find(({ codec_type }) => codec_type === "audio");
-  manifest.push({ codec, file, durationSeconds: Number(format.duration), channels: stream.channels,
-    samplingFrequency: Number(stream.sample_rate), bytes: Number(format.size) });
-}
-writeFileSync(path.join(directory, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
+/** Build-time only. No sample generation runs when the proxy starts. */
+import { spawnSync } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const directory = fileURLToPath(new URL("../assets/calibration/audio/", import.meta.url));
+const formats = [
+  ["aac", "aac", "m4a"], ["ac3", "ac3", "ac3"], ["eac3", "eac3", "eac3"],
+  ["dts", "dca", "dts"], ["flac", "flac", "flac"], ["opus", "libopus", "opus"],
+  ["vorbis", "libvorbis", "ogg"], ["truehd", "truehd", "thd"], ["mp3", "libmp3lame", "mp3"],
+  ["mp2", "mp2", "mp2"], ["alac", "alac", "m4a"], ["pcm", "pcm_s16le", "wav"]
+];
+mkdirSync(directory, { recursive: true });
+const manifest = [];
+for (const [codec, encoder, extension] of formats) {
+  const file = `${codec}.${extension}`;
+  const result = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y",
+    "-f", "lavfi", "-i", "anoisesrc=color=pink:seed=1:sample_rate=48000",
+    "-t", "4", "-ac", "2", "-strict", "-2", "-c:a", encoder, path.join(directory, file)],
+  { encoding: "utf8", windowsHide: true });
+  if (result.status !== 0) throw new Error(`${codec}: ${result.stderr}`);
+  const probe = spawnSync("ffprobe", ["-v", "error", "-show_streams", "-show_format", "-of", "json",
+    path.join(directory, file)], { encoding: "utf8", windowsHide: true });
+  if (probe.status !== 0) throw new Error(`${codec}: ${probe.stderr}`);
+  const { streams, format } = JSON.parse(probe.stdout);
+  const stream = streams.find(({ codec_type }) => codec_type === "audio");
+  manifest.push({ codec, file, durationSeconds: Number(format.duration), channels: stream.channels,
+    samplingFrequency: Number(stream.sample_rate), bytes: Number(format.size) });
+}
+writeFileSync(path.join(directory, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);

package/server.js

@@ -47,6 +47,8 @@ import { KeyframeTables } from "./services/media/KeyframeTables.js";
 import { contentsOf } from "./services/torrent/Contents.js";
 import { SubtitleOrchestrator } from "./services/media/SubtitleOrchestrator.js";
 import { containerOrchestrator, CONTAINER_HEAD_BYTES } from "./services/media/ContainerOrchestrator.js";
+import { viewerStartsOn } from "./services/viewer/PriorityMap.js";
+import { coalescing } from "./utils/coalesce.js";
 import {
   warmSubtitleCues,
   cuesHeldFor,
@@ -152,7 +154,10 @@ export async function startProxyServer({
     // after the declarations below have run.
     onPiecesArrived: ({ sourceKey, fileIndexes }) => {
       for (const fileIndex of fileIndexes) {
-        void pushFreshCues(sourceKey, fileIndex);
+        void pushFreshCues(`${sourceKey}:${fileIndex}`, sourceKey, fileIndex);
+        // A keyframe table somebody asked for while its bytes had not arrived
+        // is read again now; nothing was recorded for that wait.
+        keyframeTables.readAgainIfUnanswered({ sourceKey, fileIndex, logName: String(fileIndex) });
       }
     }
   });
@@ -263,9 +268,10 @@ export async function startProxyServer({
   // rules. The media layer used to import the second directly.
   /**
    * One file of one torrent, reduced to what the subtitle walk may know: a
-   * name, a length, which ranges are downloaded whole, and how to read one of
-   * those without asking the swarm. Piece length, file offsets and the bitfield
-   * stay in the torrent's thread.
+   * name, a length, its one container, which ranges are downloaded whole, how
+   * to read one of those without asking the swarm, the portion one read may
+   * span, and where the viewers stand. File offsets and the bitfield stay in
+   * the torrent's thread; the piece length crosses as a plain number.
    *
    * @param {string} sourceKey
    * @param {number} fileIndex
@@ -290,30 +296,39 @@ export async function startProxyServer({
       fileIndex,
       name: String(file.name ?? ""),
       length: file.length,
+      portionBytes: Number(torrent.pieceLength) > 0 ? Number(torrent.pieceLength) : undefined,
+      // The file's ONE container — the one the track table, the media info and
+      // the keyframe table are read from. Built here only when nobody has built
+      // it yet, so a pass of the walk does not fetch the file's edges again.
+      container: async () => {
+        const known = containerOrchestrator.known(sourceKey, fileIndex);
+        if (known !== undefined) {
+          return known;
+        }
+        const params = await containerOver({ sourceKey, fileIndex, tailBytes: CONTAINER_HEAD_BYTES });
+        return params ? containerOrchestrator.containerFor(params) : null;
+      },
       heldRanges: () => torrentPool.heldRangesOf(torrent, fileIndex),
-      readHeld: (start, end) => torrentPool.readHeldOf(torrent, fileIndex, start, end)
+      readHeld: (start, end) => torrentPool.readHeldOf(torrent, fileIndex, start, end),
+      // Where the viewers of this file stand: the start of every urgent stretch
+      // of its priority map, which is where each viewer's own need begins.
+      wantedSeconds: () => viewerStartsOn(outputParts.priority?.mapFor(sourceKey, fileIndex)),
+      // A pass left readable clusters for the next one: run it.
+      askAgain: () => {
+        void pushFreshCues(`${sourceKey}:${fileIndex}`, sourceKey, fileIndex);
+      }
     };
   };
-  /** @type {Set<string>} */
-  const walksInFlight = new Set();
   /**
    * Walk whatever new cues a file now holds and push them to the viewers.
    *
-   * @param {string} sourceKey
-   * @param {number} fileIndex
-   * @returns {Promise<void>}
+   * One walk of a file at a time, and one more after it whenever pieces
+   * arrived meanwhile (`coalescing`): `verified` fires per piece, so a queue of
+   * identical passes would only postpone the one with something new to find,
+   * and dropping them — as this used to — lost the pass that would have found
+   * the last pieces of a file.
    */
-  const pushFreshCues = async (sourceKey, fileIndex) => {
-    // A pass that arrives while the previous one is still walking is dropped
-    // rather than queued: `verified` fires per piece, so on a fast download
-    // these arrive many times a second, and a queue of identical passes would
-    // only postpone the one that has something new to find. The walk is
-    // serialized per file inside `SubtitleCues` anyway.
-    const key = `${sourceKey}:${fileIndex}`;
-    if (walksInFlight.has(key)) {
-      return;
-    }
-    walksInFlight.add(key);
+  const pushFreshCues = coalescing(async (sourceKey, fileIndex) => {
     try {
       const file = await heldFileFor(sourceKey, fileIndex);
       if (!file) {
@@ -325,17 +340,16 @@ export async function startProxyServer({
           : `${entry.spanStartSeconds.toFixed(1)}-${entry.spanEndSeconds.toFixed(1)}s`;
         logger.info(
           `subtitle push ${sourceKey.slice(0, 8)}:${fileIndex} track ${entry.trackIndex}: ` +
-          `${entry.cues.length} new cue(s) covering ${span}, ` +
-          `clusters walked ${entry.walkedClusters}/${entry.indexedClusters}, cursor ${entry.cursor}`
+          `${entry.cues.length} new cue(s) covering ${span}` +
+          (entry.withdrawn.length > 0 ? `, ${entry.withdrawn.length} taken back` : "") +
+          `, clusters walked ${entry.walkedClusters}/${entry.indexedClusters}, cursor ${entry.cursor}`
         );
         onSubtitleCues?.({ sourceKey, fileIndex, ...entry });
       }
     } catch (error) {
       logger.warn(`subtitle push ${sourceKey.slice(0, 8)}:${fileIndex} failed: ${error?.message ?? error}`);
-    } finally {
-      walksInFlight.delete(key);
     }
-  };
+  });
   const subtitles = new SubtitleOrchestrator(containerOrchestrator, {
     warm: async (torrent, fileIndex, sourceKey) => {
       const file = await heldFileFor(sourceKey, fileIndex, torrent);
@@ -396,7 +410,8 @@ export async function startProxyServer({
       readRange: (start, end) =>
         torrentPool.readRangeOf(torrent, fileIndex, start, Math.min(end, file.length - 1)),
       fileSize: file.length,
-      label: String(file.name ?? "")
+      label: String(file.name ?? ""),
+      portionBytes: Number(torrent.pieceLength) > 0 ? Number(torrent.pieceLength) : undefined
     };
   };
   const keyframeTables = new KeyframeTables({

package/services/encode/EncodeRuns.js

@@ -15,6 +15,7 @@ import { ENCODE_EXIT } from "./encode-exit.js";
 import { EncodeRun } from "./EncodeRun.js";
 import { computeOutputDimensions } from "./args.js";
 import { buildRunCommand, trueStartOf } from "./run-command.js";
+import { SEGMENT_CUT_TIME_DELTA_SECONDS } from "./output/index.js";
 
 /**
  * @typedef {Object} SegmentFiles
@@ -837,7 +838,8 @@ export class EncodeRuns {
           session.segmentFormat.servedNameOf?.(name) ?? name);
         if (Number.isInteger(index) && index < safeIndex) return null;
         return this.#host.segmentFiles.publish(session.outputKey ?? "", name, session.segmentFormat, {
-          endSeconds: index < session.timeline.segmentCount - 1 ? session.timeline.publishedStartOf(index + 1) : undefined
+          endSeconds: index < session.timeline.segmentCount - 1 ?
+            session.timeline.publishedStartOf(index + 1) - SEGMENT_CUT_TIME_DELTA_SECONDS : undefined
         });
       },
       onEnded: (ended) => this.noteRunEnded(session, run, ended)

package/services/encode/OutputOpening.js

@@ -449,15 +449,16 @@ export class OutputOpening {
       const { arrived } = await this.#host.keyframeTables.within({ sourceKey, fileIndex, logName });
       keyframeMs = Date.now() - keyframeStartMs;
       if (!arrived) {
-        // A read that ran out of its budget is still running, and the table is
-        // still unanswered — which is not the same as a file with no keyframes,
+        // A read that ran out of its budget is still running, and one that
+        // found its bytes not downloaded yet is made again when they arrive;
+        // either way the table is still unanswered — which is not the same as a file with no keyframes,
         // and the distinction is the table's own (`answered` against
         // `readable`). Recorded as an absence it would make a passing shortage
         // of bytes look like a property of the bytes, and every later session
         // of the file would re-encode a picture that can be copied.
         this.#host.logger.warn(
-          `transcode: the keyframe table for "${logName}" has not arrived in ` +
-            `${Math.round(this.#host.keyframeTables.budgetMs / 1000)}s, so this session re-encodes the picture ` +
+          `transcode: the keyframe table for "${logName}" has not arrived after ${keyframeMs}ms ` +
+            `(budget ${Math.round(this.#host.keyframeTables.budgetMs / 1000)}s), so this session re-encodes the picture ` +
             "instead of copying it; the read goes on and the next session of this file gets the copy"
         );
       }

package/services/encode/audio-calibration.js

@@ -1,67 +1,67 @@
-/** Short startup measurements of packaged audio samples. */
-import { spawn } from "node:child_process";
-import { readFile } from "node:fs/promises";
-import { fileURLToPath } from "node:url";
-import path from "node:path";
-import { AUDIO_TRANSCODE_KBPS } from "./args.js";
-import { ENCODE_BENCHMARK_TIMEOUT_MS } from "./hwaccel.js";
-
-const SAMPLE_DIRECTORY = fileURLToPath(new URL("../../assets/calibration/audio/", import.meta.url));
-
-function measure(ffmpegBin, file, operation, signal) {
-  return new Promise((resolve) => {
-    const startedAt = performance.now();
-    let mediaSeconds = 0;
-    let text = "";
-    let diagnostics = "";
-    const child = spawn(ffmpegBin, ["-hide_banner", "-loglevel", "info", "-nostats", "-benchmark",
-      "-i", file, "-map", "0:a:0", "-vn", "-c:a", operation,
-      ...(operation === "aac" ? ["-b:a", `${AUDIO_TRANSCODE_KBPS}k`, "-ac", "2"] : []),
-      "-f", "null", "-", "-progress", "pipe:1"],
-    { stdio: ["ignore", "pipe", "pipe"], windowsHide: true,
-      signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(ENCODE_BENCHMARK_TIMEOUT_MS)]) :
-        AbortSignal.timeout(ENCODE_BENCHMARK_TIMEOUT_MS) });
-    child.stdout.on("data", (data) => {
-      text += String(data);
-      const rows = text.split("\n");
-      text = rows.pop();
-      for (const row of rows) {
-        if (row.startsWith("out_time_us=")) {
-          const value = Number(row.slice("out_time_us=".length)) / 1e6;
-          if (Number.isFinite(value) && value > 0) mediaSeconds = value;
-        }
-      }
-    });
-    child.stderr.on("data", (data) => { diagnostics += String(data); });
-    child.on("error", () => resolve(null));
-    child.on("close", (code) => {
-      const elapsed = (performance.now() - startedAt) / 1000;
-      const runtime = diagnostics.match(/bench:.*rtime=(\d+\.(\d+))s/);
-      // The reported decimal precision bounds timing uncertainty, including a
-      // runtime printed as zero. This is a clock-resolution bound, not a weight.
-      const processingSeconds = runtime ? Number(runtime[1]) + 10 ** -runtime[2].length : null;
-      resolve(code === 0 && mediaSeconds > 0 && processingSeconds > 0 ?
-        { speed: mediaSeconds / processingSeconds, work: mediaSeconds, elapsed, processingSeconds,
-          startupSeconds: Math.max(0, elapsed - processingSeconds) } : null);
-    });
-  });
-}
-
-export async function benchmarkAudio({ ffmpegBin, logger, signal, sampleDirectory = SAMPLE_DIRECTORY }) {
-  const samples = JSON.parse(await readFile(path.join(sampleDirectory, "manifest.json"), "utf8"));
-  const readings = [];
-  const startedAt = performance.now();
-  for (const sample of samples) {
-    for (const operation of ["copy", "aac"]) {
-      if (signal?.aborted) return readings;
-      const reading = await measure(ffmpegBin, path.join(sampleDirectory, sample.file), operation, signal);
-      if (reading) readings.push({ ...sample, operation, ...reading });
-      else logger?.warn(`audio calibration ${sample.codec}->${operation} could not be measured`);
-    }
-  }
-  logger?.info(`audio calibration: ${readings.length} short packaged-sample measurements in ` +
-    `${((performance.now() - startedAt) / 1000).toFixed(2)}s`);
-  return readings;
-}
-
-export { audioReadingFor } from "./audio-work-rate.js";
+/** Short startup measurements of packaged audio samples. */
+import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+import { AUDIO_TRANSCODE_KBPS } from "./args.js";
+import { ENCODE_BENCHMARK_TIMEOUT_MS } from "./hwaccel.js";
+
+const SAMPLE_DIRECTORY = fileURLToPath(new URL("../../assets/calibration/audio/", import.meta.url));
+
+function measure(ffmpegBin, file, operation, signal) {
+  return new Promise((resolve) => {
+    const startedAt = performance.now();
+    let mediaSeconds = 0;
+    let text = "";
+    let diagnostics = "";
+    const child = spawn(ffmpegBin, ["-hide_banner", "-loglevel", "info", "-nostats", "-benchmark",
+      "-i", file, "-map", "0:a:0", "-vn", "-c:a", operation,
+      ...(operation === "aac" ? ["-b:a", `${AUDIO_TRANSCODE_KBPS}k`, "-ac", "2"] : []),
+      "-f", "null", "-", "-progress", "pipe:1"],
+    { stdio: ["ignore", "pipe", "pipe"], windowsHide: true,
+      signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(ENCODE_BENCHMARK_TIMEOUT_MS)]) :
+        AbortSignal.timeout(ENCODE_BENCHMARK_TIMEOUT_MS) });
+    child.stdout.on("data", (data) => {
+      text += String(data);
+      const rows = text.split("\n");
+      text = rows.pop();
+      for (const row of rows) {
+        if (row.startsWith("out_time_us=")) {
+          const value = Number(row.slice("out_time_us=".length)) / 1e6;
+          if (Number.isFinite(value) && value > 0) mediaSeconds = value;
+        }
+      }
+    });
+    child.stderr.on("data", (data) => { diagnostics += String(data); });
+    child.on("error", () => resolve(null));
+    child.on("close", (code) => {
+      const elapsed = (performance.now() - startedAt) / 1000;
+      const runtime = diagnostics.match(/bench:.*rtime=(\d+\.(\d+))s/);
+      // The reported decimal precision bounds timing uncertainty, including a
+      // runtime printed as zero. This is a clock-resolution bound, not a weight.
+      const processingSeconds = runtime ? Number(runtime[1]) + 10 ** -runtime[2].length : null;
+      resolve(code === 0 && mediaSeconds > 0 && processingSeconds > 0 ?
+        { speed: mediaSeconds / processingSeconds, work: mediaSeconds, elapsed, processingSeconds,
+          startupSeconds: Math.max(0, elapsed - processingSeconds) } : null);
+    });
+  });
+}
+
+export async function benchmarkAudio({ ffmpegBin, logger, signal, sampleDirectory = SAMPLE_DIRECTORY }) {
+  const samples = JSON.parse(await readFile(path.join(sampleDirectory, "manifest.json"), "utf8"));
+  const readings = [];
+  const startedAt = performance.now();
+  for (const sample of samples) {
+    for (const operation of ["copy", "aac"]) {
+      if (signal?.aborted) return readings;
+      const reading = await measure(ffmpegBin, path.join(sampleDirectory, sample.file), operation, signal);
+      if (reading) readings.push({ ...sample, operation, ...reading });
+      else logger?.warn(`audio calibration ${sample.codec}->${operation} could not be measured`);
+    }
+  }
+  logger?.info(`audio calibration: ${readings.length} short packaged-sample measurements in ` +
+    `${((performance.now() - startedAt) / 1000).toFixed(2)}s`);
+  return readings;
+}
+
+export { audioReadingFor } from "./audio-work-rate.js";

package/services/encode/audio-work-rate.js

@@ -1,15 +1,15 @@
-/** Initial per-codec work model, replaced by the track's actual speed. */
-export function audioReadingFor(readings, { codec, transcode, channels, samplingFrequency, bitrateKbps }) {
-  const operation = transcode ? "aac" : "copy";
-  const normalized = String(codec ?? "").replace(/^pcm_.*/, "pcm");
-  const reference = readings?.find((reading) => reading.codec === normalized && reading.operation === operation);
-  if (!reference) return null;
-  const referenceBytesPerSecond = reference.bytes / reference.durationSeconds;
-  // Copy work is encoded bytes; processing work is audio samples. Unstated
-  // dimensions retain the reference measurement, rather than invented values.
-  const workRatio = !transcode && bitrateKbps > 0 ? (bitrateKbps * 1000 / 8) / referenceBytesPerSecond :
-    transcode && channels > 0 && samplingFrequency > 0 ?
-      (channels * samplingFrequency) / (reference.channels * reference.samplingFrequency) : 1;
-  return { ...reference, speed: reference.speed / workRatio,
-    basis: workRatio === 1 ? "measured-reference" : transcode ? "audio-samples" : "encoded-bytes" };
-}
+/** Initial per-codec work model, replaced by the track's actual speed. */
+export function audioReadingFor(readings, { codec, transcode, channels, samplingFrequency, bitrateKbps }) {
+  const operation = transcode ? "aac" : "copy";
+  const normalized = String(codec ?? "").replace(/^pcm_.*/, "pcm");
+  const reference = readings?.find((reading) => reading.codec === normalized && reading.operation === operation);
+  if (!reference) return null;
+  const referenceBytesPerSecond = reference.bytes / reference.durationSeconds;
+  // Copy work is encoded bytes; processing work is audio samples. Unstated
+  // dimensions retain the reference measurement, rather than invented values.
+  const workRatio = !transcode && bitrateKbps > 0 ? (bitrateKbps * 1000 / 8) / referenceBytesPerSecond :
+    transcode && channels > 0 && samplingFrequency > 0 ?
+      (channels * samplingFrequency) / (reference.channels * reference.samplingFrequency) : 1;
+  return { ...reference, speed: reference.speed / workRatio,
+    basis: workRatio === 1 ? "measured-reference" : transcode ? "audio-samples" : "encoded-bytes" };
+}

package/services/encode/output/index.js

@@ -1,6 +1,10 @@
 export { OutputSpec, VideoOutput, AudioOutput, CutGrid, isOutputName } from "./OutputSpec.js";
 export { PLAYLIST_FILE_NAME } from "./playlists.js";
 
+// FFmpeg may close a segment this far before its requested cut. Both the
+// command and coverage validation use this existing muxer configuration.
+export const SEGMENT_CUT_TIME_DELTA_SECONDS = 0.05;
+
 /**
  * The code an error carries when no output suits the viewer: nothing this proxy
  * could produce or already holds is admitted by their link. Its `details` say

package/services/encode/run-command.js

@@ -17,6 +17,7 @@
 
 import { PLAYLIST_FILE_NAME } from "./output/playlists.js";
 import { AUDIO_TRANSCODE_KBPS } from "./args.js";
+import { SEGMENT_CUT_TIME_DELTA_SECONDS } from "./output/index.js";
 
 /**
  * What ffmpeg's own CLI subtracts from an input seek, and therefore what has to
@@ -694,7 +695,7 @@ export function buildRunCommand({
       // recorded a hair late would skip to the next one and double the
       // segment. The tolerance absorbs that rounding.
       "-segment_time_delta",
-      "0.05",
+      String(SEGMENT_CUT_TIME_DELTA_SECONDS),
       "-segment_start_number",
       String(inputIndex),
       // THE ENCODER SAYS WHEN A PIECE IS FINISHED, on a channel of its own.