@torrent-tv/proxy 2.86.0 → 2.87.0

package/services/encode/segment-formats/init-compat.js

@@ -0,0 +1,324 @@
+/**
+ * @file Whether two init segments can serve one another's pieces.
+ *
+ * A player fetches `#EXT-X-MAP` once and decodes every fragment of that address
+ * against it. So a piece made under one header and served under another is only
+ * safe when the two headers say the same thing to a decoder — and "the same
+ * thing" has to be decided BEFORE any experiment, or the rule becomes a list of
+ * whatever differences happened to turn up while it was being written.
+ *
+ * THE RULE, conservative by construction:
+ *
+ * 1. the WHOLE header is compared, box by box, leaf by leaf, byte for byte;
+ * 2. every difference is reported by the path it sits at;
+ * 3. a difference makes the two headers INCOMPATIBLE unless it is covered by an
+ *    explicitly listed exception;
+ * 4. **each exception carries the reason it is one.** A field nobody can name
+ *    is not an exception — an unreadable or unexpected difference refuses.
+ *
+ * What this gives up deliberately: it will refuse pairs that would in fact have
+ * played. That is the direction to be wrong in — the other direction is a
+ * viewer whose picture turns to blocks, which this project has already paid for
+ * once (`research/budget-downshift-breaks-the-picture-2026-08-21.md`).
+ *
+ * Only a format that HAS an init is described here. MPEG-TS carries its
+ * parameter sets in the stream and has no header to compare, so two of its
+ * outputs are never joined by this question at all.
+ */
+
+import { walkBoxes } from "./mp4-boxes.js";
+
+/** Boxes whose payload is a list of child boxes, mirroring `walkBoxes`. */
+const CONTAINER_BOXES = new Set(["moov", "trak", "mdia", "minf", "stbl", "edts", "mvex"]);
+
+/**
+ * Sample entries, and how many bytes of fixed preamble stand before their child
+ * boxes.
+ *
+ * A sample entry is not a plain container: it opens with fields at fixed
+ * offsets and only then carries boxes. The two shapes are the ones ISO/IEC
+ * 14496-12 defines — `SampleEntry` is 8 bytes (six reserved plus a data
+ * reference index), `VisualSampleEntry` adds 70 more and `AudioSampleEntry`
+ * adds 20.
+ *
+ * WHY THIS MATTERS ENOUGH TO PARSE: without descending here, a difference
+ * anywhere inside a sample entry is reported as one difference of the whole
+ * `stsd`. Allowing it would allow `avcC` — the parameter sets — with it, which
+ * is the one thing this rule exists to refuse. The preamble is a stated
+ * constant and a wrong one would mis-read every child, so the descent is
+ * CHECKED rather than trusted: see `tilesExactly`.
+ *
+ * @type {ReadonlyMap<string, number>}
+ */
+const SAMPLE_ENTRY_PREAMBLE = new Map([
+  ["avc1", 78], ["avc3", 78], ["hvc1", 78], ["hev1", 78], ["av01", 78], ["vp09", 78],
+  ["mp4a", 28], ["ac-3", 28], ["ec-3", 28], ["opus", 28], ["fLaC", 28]
+]);
+
+/** `stsd` is a FullBox: version and flags, then an entry count, then entries. */
+const SAMPLE_DESCRIPTION_PREAMBLE = 8;
+
+/**
+ * The differences that do NOT refuse, each with the reason it does not.
+ *
+ * An entry is either an exact `path`, or a `sampleEntryBox` — a box type
+ * appearing directly inside a sample entry, whichever track and whichever
+ * entry. The second form exists because the reason is a property of the BOX
+ * and not of where it sits: a file with the picture on the second track states
+ * the same thing about it.
+ *
+ * @type {ReadonlyArray<{ path?: string, sampleEntryBox?: string, why: string }>}
+ */
+export const INIT_DIFFERENCE_EXCEPTIONS = [
+  {
+    sampleEntryBox: "btrt",
+    why:
+      "BitRateBox declares a decoding buffer size and a maximum and average rate. " +
+      "Four readings, none of them from this experiment's own output: the W3C " +
+      "Media Source Extensions byte stream format for ISOBMFF, which is what " +
+      "governs a browser's handling of an initialization segment, does not " +
+      "mention the box at all and requires only `ftyp` and `moov`; hls.js — the " +
+      "player this product ships — never reads one, and writes a HARDCODED " +
+      "3 Mbit/s into every init it builds for every stream whatever its real " +
+      "rate (`MP4.types.btrt`, the avc1 and hvc1 sample entries); this proxy's " +
+      "own code neither reads nor writes it; and what actually configures a " +
+      "decoder, `avcC`, is compared separately by this same rule and a " +
+      "difference there still refuses."
+  }
+];
+
+/**
+ * Where a box's child boxes begin, counted from the start of its payload, or
+ * null when the box is a leaf.
+ *
+ * @param {string} type
+ * @returns {number | null}
+ */
+function childrenStartOf(type) {
+  if (CONTAINER_BOXES.has(type)) {
+    return 0;
+  }
+  if (type === "stsd") {
+    return SAMPLE_DESCRIPTION_PREAMBLE;
+  }
+  return SAMPLE_ENTRY_PREAMBLE.get(type) ?? null;
+}
+
+/**
+ * Whether the range holds whole boxes and nothing else.
+ *
+ * The descent above rests on stated offsets, and a stated offset can be wrong —
+ * for a sample entry shape this does not know, or a box carrying a 64-bit size.
+ * Rather than mis-read the children, the walk asks first whether they tile the
+ * range exactly. They do not, and the parent is compared whole, which is the
+ * conservative answer.
+ *
+ * Size handling mirrors `walkBoxes`, so what this accepts is exactly what that
+ * would then walk.
+ *
+ * @param {Buffer} buffer
+ * @param {number} start
+ * @param {number} end
+ * @returns {boolean}
+ */
+function tilesExactly(buffer, start, end) {
+  if (!(end > start)) {
+    return false;
+  }
+  let offset = start;
+  while (offset + 8 <= end) {
+    let size = buffer.readUInt32BE(offset);
+    let headerSize = 8;
+    if (size === 1) {
+      if (offset + 16 > end) {
+        return false;
+      }
+      size = Number(buffer.readBigUInt64BE(offset + 8));
+      headerSize = 16;
+    } else if (size === 0) {
+      size = end - offset;
+    }
+    if (size < headerSize || offset + size > end) {
+      return false;
+    }
+    offset += size;
+  }
+  return offset === end;
+}
+
+/**
+ * Every leaf of the box tree, by the path it sits at.
+ *
+ * A path is `type` names joined by `/`, each carrying its index among its
+ * siblings OF THE SAME TYPE — so two tracks are `moov/trak[0]` and
+ * `moov/trak[1]` and a field of one is never compared against the other's.
+ *
+ * @param {Buffer} initSegment
+ * @returns {Map<string, Buffer>} path → the leaf's payload bytes
+ */
+export function initLeaves(initSegment) {
+  /** @type {Map<string, Buffer>} */
+  const leaves = new Map();
+  if (!Buffer.isBuffer(initSegment) || initSegment.length === 0) {
+    return leaves;
+  }
+  const walk = (start, end, prefix) => {
+    /** @type {Map<string, number>} */
+    const seen = new Map();
+    // `walkBoxes` recurses into containers itself, so this pass is given one
+    // level at a time and asks for the children of each container explicitly.
+    // That is what makes the path unambiguous: a bare walk would report a
+    // grandchild without saying whose it is.
+    walkBoxes(
+      initSegment,
+      (type, bodyStart, bodyEnd) => {
+        const index = seen.get(type) ?? 0;
+        seen.set(type, index + 1);
+        const path = `${prefix}${type}[${index}]`;
+        const childrenAt = childrenStartOf(type);
+        if (childrenAt !== null && tilesExactly(initSegment, bodyStart + childrenAt, bodyEnd)) {
+          walk(bodyStart + childrenAt, bodyEnd, `${path}/`);
+          return;
+        }
+        leaves.set(path, initSegment.subarray(bodyStart, bodyEnd));
+      },
+      start,
+      end,
+      // One level only: the recursion above is this function's own, and it is
+      // what makes a path say whose child a box is.
+      () => false
+    );
+    return leaves;
+  };
+  walk(0, initSegment.length, "");
+  return leaves;
+}
+
+/**
+ * The box type a path's last step names, without its index.
+ *
+ * @param {string} path
+ * @param {number} [fromEnd=0] - 0 is the leaf, 1 its parent.
+ * @returns {string}
+ */
+function stepOf(path, fromEnd = 0) {
+  const steps = path.split("/");
+  const step = steps[steps.length - 1 - fromEnd] ?? "";
+  return step.replace(/\[\d+\]$/, "");
+}
+
+/**
+ * Whether an exception covers this path.
+ *
+ * @param {ReadonlyArray<{ path?: string, sampleEntryBox?: string }>} exceptions
+ * @param {string} path
+ * @returns {boolean}
+ */
+function permits(exceptions, path) {
+  for (const exception of exceptions) {
+    if (exception.path !== undefined && exception.path === path) {
+      return true;
+    }
+    if (
+      exception.sampleEntryBox !== undefined &&
+      stepOf(path) === exception.sampleEntryBox &&
+      SAMPLE_ENTRY_PREAMBLE.has(stepOf(path, 1))
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * What a difference at this path amounts to, in words.
+ *
+ * Decoding is for READING the report, never for deciding it: the verdict is the
+ * byte comparison, and a path this cannot decode is still a difference.
+ *
+ * @param {string} path
+ * @param {Buffer | null} left
+ * @param {Buffer | null} right
+ * @returns {string}
+ */
+function describeDifference(path, left, right) {
+  if (!left) return `${path}: absent from the first header`;
+  if (!right) return `${path}: absent from the second header`;
+  if (stepOf(path) === "btrt" && left.length >= 12 && right.length >= 12) {
+    const rates = (buffer) =>
+      `buffer ${buffer.readUInt32BE(0)}B, max ${buffer.readUInt32BE(4)}, avg ${buffer.readUInt32BE(8)} bit/s`;
+    return `${path}: ${rates(left)} against ${rates(right)}`;
+  }
+  if (path.includes("mdhd")) {
+    // FullBox: version+flags, then creation/modification, then timescale.
+    const at = left.readUInt8(0) === 1 ? 20 : 12;
+    if (left.length > at + 4 && right.length > at + 4) {
+      return `${path}: timescale ${left.readUInt32BE(at)} against ${right.readUInt32BE(at)}`;
+    }
+  }
+  if (stepOf(path) === "stsd") {
+    // Reached only where the entries did not tile, so the walk could not
+    // descend. The entry's own type is at a fixed place and is worth naming.
+    const entryType = (buffer) => (buffer.length >= 16 ? buffer.toString("latin1", 12, 16) : "?");
+    const left4 = entryType(left);
+    const right4 = entryType(right);
+    const kind = SAMPLE_ENTRY_PREAMBLE.has(left4) || SAMPLE_ENTRY_PREAMBLE.has(right4) ? "" : " (unrecognised)";
+    return `${path}: sample entry ${left4} against ${right4}${kind}, ${left.length} bytes against ${right.length}`;
+  }
+  if (left.length !== right.length) {
+    return `${path}: ${left.length} bytes against ${right.length}`;
+  }
+  let firstDifferent = 0;
+  while (firstDifferent < left.length && left[firstDifferent] === right[firstDifferent]) {
+    firstDifferent += 1;
+  }
+  return `${path}: differs from byte ${firstDifferent}`;
+}
+
+/**
+ * Whether a piece made under `made` may be served under `served`.
+ *
+ * @param {Buffer} served - The header the player already holds.
+ * @param {Buffer} made - The header the other output produces under.
+ * @param {{ exceptions?: ReadonlyArray<{ path?: string, sampleEntryBox?: string }> }} [options]
+ * @returns {{ compatible: boolean, differences: string[], refusedPaths: string[] }}
+ */
+export function compareInits(served, made, { exceptions = INIT_DIFFERENCE_EXCEPTIONS } = {}) {
+  if (!Buffer.isBuffer(served) || !Buffer.isBuffer(made) || served.length === 0 || made.length === 0) {
+    return {
+      compatible: false,
+      differences: ["one of the two headers is missing"],
+      refusedPaths: ["<missing>"]
+    };
+  }
+  if (served.equals(made)) {
+    return { compatible: true, differences: [], refusedPaths: [] };
+  }
+  const left = initLeaves(served);
+  const right = initLeaves(made);
+  /** @type {string[]} */
+  const differences = [];
+  /** @type {string[]} */
+  const refusedPaths = [];
+  const paths = new Set([...left.keys(), ...right.keys()]);
+  for (const path of paths) {
+    const a = left.get(path) ?? null;
+    const b = right.get(path) ?? null;
+    if (a && b && a.equals(b)) {
+      continue;
+    }
+    differences.push(describeDifference(path, a, b));
+    if (!permits(exceptions, path)) {
+      refusedPaths.push(path);
+    }
+  }
+  if (differences.length === 0) {
+    // The bytes differ but every leaf agrees, so what moved is a box header —
+    // a size field, or a box present in one tree and absent from the other at a
+    // level this walk treats as a container. Not attributable, therefore not
+    // permitted.
+    differences.push("the headers differ outside any leaf this walk can name");
+    refusedPaths.push("<unattributed>");
+  }
+  return { compatible: refusedPaths.length === 0, differences, refusedPaths };
+}

package/services/{segment-formats → encode/segment-formats}/mp4-boxes.js

@@ -25,9 +25,14 @@ const CONTAINER_BOXES = new Set(["moov", "trak", "mdia", "minf", "stbl", "edts",
  *   `bodyStart`/`bodyEnd` delimit the box payload (header excluded).
  * @param {number} [start=0]
  * @param {number} [end=buffer.length]
+ * @param {(type: string) => boolean} [descend] - Whether to walk into a box's
+ *   payload. The default is the container list above, which is what every
+ *   reader of a whole tree wants. A caller that needs to know WHOSE child a box
+ *   is passes `() => false` and recurses itself, because a walk that descends
+ *   for you cannot say which parent it descended from.
  * @returns {void}
  */
-export function walkBoxes(buffer, visit, start = 0, end = buffer.length) {
+export function walkBoxes(buffer, visit, start = 0, end = buffer.length, descend = (type) => CONTAINER_BOXES.has(type)) {
   let offset = start;
   while (offset + 8 <= end) {
     let size = buffer.readUInt32BE(offset);
@@ -48,8 +53,8 @@ export function walkBoxes(buffer, visit, start = 0, end = buffer.length) {
       return; // truncated or malformed — stop rather than read out of bounds
     }
     visit(type, offset + headerSize, offset + size);
-    if (CONTAINER_BOXES.has(type)) {
-      walkBoxes(buffer, visit, offset + headerSize, offset + size);
+    if (descend(type)) {
+      walkBoxes(buffer, visit, offset + headerSize, offset + size, descend);
     }
     offset += size;
   }

package/services/encode/throughput.js

@@ -0,0 +1,97 @@
+/**
+ * @file How fast a calibrated encoding mode runs at a given frame size, from
+ * the readings the startup calibration took (`encode/calibration.js`).
+ *
+ * Pure arithmetic over measured figures, kept apart from the calibration so
+ * that everything which prices an encode can read it without reaching the code
+ * that runs ffmpeg.
+ */
+
+/**
+ * @typedef {object} SizeReading
+ * @property {number} width
+ * @property {number} height
+ * @property {number} pixelsPerSec
+ */
+
+/**
+ * The rate of pictures this mode would encode at this frame, in pixels per
+ * second, or null where this machine has not shown it can.
+ *
+ * An entry with no readings by size — built before sizes were read, or by a
+ * check that states one figure — answers with its one figure, as it always did.
+ *
+ * @param {{ pixelsPerSec: number, bySize?: SizeReading[], interpolationError?: number | null }} entry
+ * @param {{ width: number, height: number } | null | undefined} frame
+ * @returns {number | null}
+ */
+export function throughputAt(entry, frame) {
+  const bySize = Array.isArray(entry?.bySize) ? entry.bySize : [];
+  const width = Number(frame?.width) || 0;
+  const height = Number(frame?.height) || 0;
+  if (bySize.length === 0 || width <= 0 || height <= 0) {
+    const figure = Number(entry?.pixelsPerSec);
+    return Number.isFinite(figure) && figure > 0 ? figure : null;
+  }
+  const area = width * height;
+  const sorted = [...bySize].sort((left, right) => left.width * left.height - right.width * right.height);
+  const smallest = sorted[0];
+  const largest = sorted[sorted.length - 1];
+  if (area < smallest.width * smallest.height || area > largest.width * largest.height) {
+    return null;
+  }
+  for (let index = 0; index < sorted.length; index += 1) {
+    const reading = sorted[index];
+    const readArea = reading.width * reading.height;
+    if (readArea === area) {
+      return reading.pixelsPerSec;
+    }
+    if (readArea > area) {
+      const error = entry.interpolationError;
+      if (!(Number.isFinite(error) && error >= 0 && error < 1)) {
+        return null;
+      }
+      const below = sorted[index - 1];
+      return interpolate(below, reading, area) * (1 - error);
+    }
+  }
+  return null;
+}
+
+/**
+ * Pixels per second at an area between two readings, straight in logarithms
+ * of both: throughput against frame area is a power law over any short
+ * stretch, and a straight line between two points of it is the least that
+ * can be said without inventing a shape.
+ *
+ * @param {SizeReading} below
+ * @param {SizeReading} above
+ * @param {number} area
+ * @returns {number}
+ */
+function interpolate(below, above, area) {
+  const lowArea = below.width * below.height;
+  const highArea = above.width * above.height;
+  const share = Math.log(area / lowArea) / Math.log(highArea / lowArea);
+  return Math.exp(Math.log(below.pixelsPerSec) + share * (Math.log(above.pixelsPerSec) - Math.log(below.pixelsPerSec)));
+}
+
+/**
+ * How far an interpolation between readings of this mode was measured to be
+ * wrong: the largest relative error of a middle reading predicted from the two
+ * outside it.
+ *
+ * @param {SizeReading[]} bySize - Smallest first.
+ * @returns {number | null} Null with fewer than three readings.
+ */
+export function interpolationErrorOf(bySize) {
+  let largest = null;
+  for (let index = 1; index + 1 < bySize.length; index += 1) {
+    const middle = bySize[index];
+    const predicted = interpolate(bySize[index - 1], bySize[index + 1], middle.width * middle.height);
+    const error = Math.abs(predicted - middle.pixelsPerSec) / middle.pixelsPerSec;
+    largest = largest === null ? error : Math.max(largest, error);
+  }
+  return largest;
+}
+

package/services/{orchestrators → media}/ContainerOrchestrator.js

@@ -7,15 +7,37 @@
  * readRange, knows nothing about torrents or HTTP.
  */
 
-import { ContainerFactory } from "../container/ContainerFactory.js";
+import { ContainerFactory } from "./container/ContainerFactory.js";
+
+/**
+ * How much of a file's head its track table lives in.
+ *
+ * A fact of the FORMATS, not of the torrent: Matroska puts its Tracks element
+ * in the head and MP4's `moov` is there or pointed to from there. Whoever
+ * fetches bytes before asking is told this figure rather than choosing one.
+ */
+export const CONTAINER_HEAD_BYTES = 256 * 1024;
 import { logger } from "../../utils/logger.js";
 
 export class ContainerOrchestrator {
   constructor() {
-    /** @type {Map<string, import("../container/Container.js").Container|null>} */
+    /** @type {Map<string, import("./container/Container.js").Container|null>} */
     this.cache = new Map();
-    /** @type {Map<string, Promise<import("../container/Container.js").Container|null>>} */
+    /** @type {Map<string, Promise<import("./container/Container.js").Container|null>>} */
     this.pending = new Map();
+    /**
+     * What each file's container declares, once it has said anything.
+     *
+     * The container parses on every ask, so without this the header is read
+     * again for each question — and the track table is asked for the audio
+     * menu, the subtitle defaults and the video facts of one file. It used to
+     * be kept in the torrent thread (`torrent-worker/container-tracks.js`), which is where the
+     * parse used to happen; it is the same one cache in its new place, not a
+     * second one.
+     *
+     * @type {Map<string, import("./tracks/index.js").ContainerTrack[]>}
+     */
+    this.tracks = new Map();
   }
 
   /**
@@ -25,7 +47,7 @@ export class ContainerOrchestrator {
    * @param {(start:number,end:number)=>Promise<Buffer|null>} params.readRange
    * @param {number} params.fileSize
    * @param {string} [params.label]
-   * @returns {Promise<import("../container/Container.js").Container|null>}
+   * @returns {Promise<import("./container/Container.js").Container|null>}
    */
   async getContainer({ sourceKey, fileIndex, readRange, fileSize, label = "" }) {
     const key = `${sourceKey}:${fileIndex}`;
@@ -48,13 +70,27 @@ export class ContainerOrchestrator {
 
   /**
    * @param {object} params - same as getContainer
-   * @returns {Promise<import("../tracks/index.js").ContainerTrack[]>}
+   * @returns {Promise<import("./tracks/index.js").ContainerTrack[]>}
    */
   async getTracks(params) {
+    const key = `${params.sourceKey}:${params.fileIndex}`;
+    const known = this.tracks.get(key);
+    if (known) {
+      return known;
+    }
     const container = await this.getContainer(params);
     if (!container) return [];
     try {
-      return await container.readTracks();
+      const tracks = await container.readTracks();
+      // ONLY A NON-EMPTY READING IS KEPT. An empty one usually means the header
+      // has not arrived off the swarm yet, and caching that would answer "this
+      // file declares nothing" for the life of the process — which is what
+      // decides whether the viewer is offered a soundtrack at all.
+      if (Array.isArray(tracks) && tracks.length > 0) {
+        this.tracks.set(key, tracks);
+        return tracks;
+      }
+      return Array.isArray(tracks) ? tracks : [];
     } catch (e) {
       logger.warn(`container: readTracks failed for "${params.label}": ${e?.message ?? e}`);
       return [];
@@ -70,7 +106,7 @@ export class ContainerOrchestrator {
    * a final answer about the container.
    *
    * @param {object} params - same as getContainer
-   * @returns {Promise<import("../container/Container.js").ContainerMediaInfo|null>}
+   * @returns {Promise<import("./container/Container.js").ContainerMediaInfo|null>}
    */
   async getMediaInfo(params) {
     const container = await this.getContainer(params);
@@ -101,10 +137,12 @@ export class ContainerOrchestrator {
     if (fileIndex === undefined) {
       for (const k of [...this.cache.keys()]) if (k.startsWith(`${sourceKey}:`)) this.cache.delete(k);
       for (const k of [...this.pending.keys()]) if (k.startsWith(`${sourceKey}:`)) this.pending.delete(k);
+      for (const k of [...this.tracks.keys()]) if (k.startsWith(`${sourceKey}:`)) this.tracks.delete(k);
       return;
     }
     this.cache.delete(`${sourceKey}:${fileIndex}`);
     this.pending.delete(`${sourceKey}:${fileIndex}`);
+    this.tracks.delete(`${sourceKey}:${fileIndex}`);
   }
 }