@torrent-tv/proxy 2.97.30 → 2.98.0

package/bin/cli.js

@@ -88,6 +88,7 @@ program
   .option("--no-transcode-audio", "Disable optional HLS AAC audio transcoding")
   .option("--no-port-mapping", "Disable automatic UPnP/NAT-PMP port mapping")
   .option("--delivery-sink", "Serve /api/delivery-sink, a torrent-free byte stream for delivery testing")
+  .option("--pin-preset <name>", "Encode every software picture at this libx264 preset only, for runs that must be compared at one quality (off by default)")
   .option("--profile-start <seconds>", "Profile the main thread's CPU for this many seconds from the first source opened, written beside the state (off by default)")
   .option(
     "--send-chunk-bytes <bytes>",
@@ -394,6 +395,7 @@ try {
     logFile: options.logFile ?? "",
     deliverySink: options.deliverySink === true,
     profileStartSeconds: Number(options.profileStart) || 0,
+    pinnedPreset: typeof options.pinPreset === "string" && options.pinPreset.trim() ? options.pinPreset.trim() : null,
     // Late-bound the same way `webRtcManager` is below: the torrent pool is
     // built inside `startProxyServer`, before `dataChannelHandler` — the
     // thing that actually owns a channel to push down — exists.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.97.30",
+  "version": "2.98.0",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -127,6 +127,9 @@ export async function startProxyServer({
   deliverySink = false,
   // Seconds of main-thread CPU profile from the first source opened; 0 is off.
   profileStartSeconds = 0,
+  // The one software preset every picture is encoded at, for runs compared at
+  // one quality; null leaves the choice to the machine.
+  pinnedPreset = null,
   // What viewers' connections carry beyond the film, per byte of film; the
   // transport counts it and is built after this server, so it is late-bound.
   serviceShare = () => null,
@@ -543,7 +546,7 @@ export async function startProxyServer({
       // ranges closer than a piece hold the same pieces as one.
       const pieceLength = Number(torrentPool.knownTorrent(map.sourceKey)?.pieceLength);
       const passStartedAt = performance.now();
-      let segments = 0;
+      let rangeCount = 0;
       const converted = [];
       const demands = [];
       // What each output's next run would copy, asked of the swarm as one group
@@ -629,12 +632,18 @@ export async function startProxyServer({
         const { tracks: _choices, owner: _owner, sourceInterval: _sourceInterval, ...demand } = zone;
         const convert = ranges => {
           const joined = joinedRangesOf(ranges, pieceLength);
-          segments += 1;
+          rangeCount += joined.length;
           // What the segment reads is kept for the forecast in a table of its
           // own, whatever form the torrent is told it in.
           segmentInputs.set(map.sourceKey, map.fileIndex, zone.outputKey, zone.index, joined);
-          for (const [byteStart, byteEnd] of joined) converted.push({ ...demand,
-            downloadInterval: { from: zone.owner.from, to: zone.owner.to }, byteStart, byteEnd });
+          // ONE ZONE FOR THE SEGMENT, its demand stated once for all the
+          // ranges it reads. The demand — level, priority, deadline, the map
+          // zone it belongs to — is this pass's; the ranges are the list kept
+          // for the segment, unchanged from pass to pass. A zone for each
+          // range copied the demand into every one: ten thousand objects a
+          // pass over a two-hour AVI, made again 21-61 times a minute on Home
+          // Assistant (r26, torrent-tv/meta#178).
+          converted.push({ ...demand, downloadInterval: { from: zone.owner.from, to: zone.owner.to }, ranges: joined });
         };
         // The bytes one segment needs are a fact of the file and the output, so
         // they are worked out once. A whole film is thousands of segments and
@@ -694,7 +703,7 @@ export async function startProxyServer({
         convert(input.ranges);
       }
       const result = expanded();
-      notePass({ complete, segments, ranges: converted.length, zones: result.length, ms: performance.now() - passStartedAt });
+      notePass({ complete, segments: converted.length, ranges: rangeCount, zones: result.length, ms: performance.now() - passStartedAt });
       if (complete) resolvedPasses.set(fileKey, { ...pass, result });
       else resolvedPasses.delete(fileKey);
       return result;
@@ -792,6 +801,7 @@ export async function startProxyServer({
   });
   let sourcePreparation;
   const outputParts = wireOutputs({
+    pinnedPreset,
     indexMemory,
     serviceShare,
     readMetadataActivity: () => containerOrchestrator.activity(),

package/services/encode/Renditions.js

@@ -1358,8 +1358,23 @@ export class Renditions {
     if (!base) {
       return null;
     }
-    if (!this.#host.qualityOffer.offeredHeightsFor(base).includes(height)) {
-      return null;
+    const offered = this.#host.qualityOffer.offeredHeightsFor(base);
+    if (!offered.includes(height)) {
+      if (!this.#host.outputs.splicableHeights(base).includes(height)) {
+        return null;
+      }
+      // A STEP THE MASTER LISTS AND THE OFFER NO LONGER HOLDS. The offer moves
+      // with what the machine measures, and a page preparing a step it was
+      // asked for can arrive after the ask was withdrawn: field r28, 360p
+      // asked at 21:40:44.308, the offer down to 400p at 21:40:46.143, the
+      // warm-up at 46.170 answered "no such variant" (torrent-tv/meta#178).
+      // The step exists; it is not available now, and the page keeps what it
+      // plays and follows the next ask.
+      this.#host.logger.info(`transcode ${base.id} warming ${height}p for ${consumerId}: not offered now (offering ${offered.map(one => `${one}p`).join(" ") || "nothing"})`);
+      const error = new Error(`${height}p is not offered now.`);
+      error.code = OUTPUT_UNAVAILABLE;
+      error.details = { reason: `${height}p is not offered on this machine now`, figures: { height, offeredHeights: offered } };
+      throw error;
     }
     const index = this.#host.outputTimes.segmentIndexForTime(base, positionSeconds);
     const variant = await this.resolveVariantSession(baseSessionId, height, index, consumerId);

package/services/encode/hwaccel.js

@@ -1715,7 +1715,8 @@ export function buildResolutionLadder(ceilingWidth, ceilingHeight) {
  * @param {{ width: number, height: number }} ceiling
  * @param {number} outputFps
  * @param {{ decodeModel?: object | null, source?: { megapixelsPerSecond: number, megabitsPerSecond: number } | null, requiredSpeed?: number | null }} [cost]
- * @returns {{ width: number, height: number, preset: string, ladder: Array<{ width: number, height: number }>, rungIndex: number } | null}
+ * @returns {{ width: number, height: number, preset: string, ladder: Array<{ width: number, height: number }>, rungIndex: number, bar: number } | null}
+ *   `bar` is the speed the choice had to clear.
  */
 export function chooseSoftwareEncodeSettings(benchmark, ceiling, outputFps, cost = {}) {
   if (!Array.isArray(benchmark) || benchmark.length === 0) {
@@ -1750,7 +1751,7 @@ export function chooseSoftwareEncodeSettings(benchmark, ceiling, outputFps, cost
   }
   const chosen = ladder[chosenIndex];
   const preset = pickSoftwarePreset(benchmark, chosen.width * chosen.height * fps, cost, chosen);
-  return { width: chosen.width, height: chosen.height, preset, ladder, rungIndex: chosenIndex };
+  return { width: chosen.width, height: chosen.height, preset, ladder, rungIndex: chosenIndex, bar };
 }
 
 /**

package/services/encode/quality/EncodeCost.js

@@ -1103,7 +1103,8 @@ export class EncodeCost {
     sourceHeight,
     outputFps,
     source = null,
-    requiredSpeed = null
+    requiredSpeed = null,
+    observedDecodeCostSec = null
   }) {
     if (!transcodeVideo || this.#host().encoderKind !== "software" || !this.#host().benchmark) {
       return null;
@@ -1113,26 +1114,44 @@ export class EncodeCost {
       return null;
     }
     const concurrentCostSec = this.#committedMachineCostSec();
+    // THE FILE'S OWN DECODE COST, where its encoder has already been read: the
+    // opening had it and the choice priced decoding from the startup model
+    // instead (review of r28, torrent-tv/meta#178).
+    const decodeCostSec = Number.isFinite(observedDecodeCostSec) && observedDecodeCostSec > 0 ? observedDecodeCostSec : null;
+    /** Every size and preset the choice priced, with the speed it was priced at. */
+    const considered = [];
     const speedForPreset = (benchmark, frame, outputPixelsPerSec) => {
-      if (concurrentCostSec === null) {
-        return null;
-      }
-      return this.#speedOnThisMachine({
+      const speed = concurrentCostSec === null ? null : this.#speedOnThisMachine({
         benchmark,
         source,
         width: frame.width,
         height: frame.height,
         fps: outputPixelsPerSec / (frame.width * frame.height),
-        observedDecodeCostSec: null,
+        observedDecodeCostSec: decodeCostSec,
         concurrentCostSec
       });
+      considered.push({ preset: benchmark[0]?.preset, width: frame.width, height: frame.height, speed });
+      return speed;
     };
-    return chooseSoftwareEncodeSettings(
+    const choice = chooseSoftwareEncodeSettings(
       this.#host().benchmark,
       { width: ceiling.w, height: ceiling.h },
       outputFps,
-      { decodeModel: this.#host().decodeModel, source, requiredSpeed, speedForPreset }
+      { decodeModel: this.#host().decodeModel, source, requiredSpeed, speedForPreset, observedDecodeCostSec: decodeCostSec }
     );
+    // WHAT THE CHOICE WAS MADE FROM, said once per choice: three runs of one
+    // file on one host opened at three presets and nothing said why (r25, r26,
+    // r28, torrent-tv/meta#178).
+    if (choice) {
+      const availability = this.#host().availability;
+      const priced = considered.map(({ preset, width, height, speed }) =>
+        `${width}x${height} ${preset}=${Number.isFinite(speed) ? `${speed.toFixed(2)}x` : "unpriced"}`).join(", ");
+      logger.info(`encode choice: ${choice.width}x${choice.height}@${Number(outputFps).toFixed(3)} ${choice.preset} against ${choice.bar.toFixed(2)}x; ` +
+        `priced ${priced}; decoding ${decodeCostSec === null ? "from the startup model" : `${decodeCostSec.toFixed(4)} s per second, measured on this file`}; ` +
+        `machine ${availability?.known ? `${Math.round(availability.share * 100)}% to spare` : "share unknown"}; ` +
+        `committed work ${concurrentCostSec === null ? "unmeasured" : `${concurrentCostSec.toFixed(3)} s per second`}`);
+    }
+    return choice;
   }
 }
 

package/services/encode/quality/output-format.js

@@ -144,7 +144,8 @@ export function decideOutputFormat({
         sourceHeight: source.height,
         outputFps: fps,
         source: source.decode,
-        requiredSpeed: cost.requiredSpeed
+        requiredSpeed: cost.requiredSpeed,
+        observedDecodeCostSec: cost.observedDecodeCostSec ?? null
       });
   const width = budget?.width ?? ceiling?.w ?? 0;
   const height = budget?.height ?? ceiling?.h ?? 0;

package/services/server/wire-outputs.js

@@ -189,6 +189,9 @@ export function wireOutputs({
     keyframeTables = new KeyframeTables(),
     videoEncoder = null,
     softwarePresetBenchmark = null,
+    // A software preset every picture is encoded at, for runs compared at one
+    // quality (`--pin-preset`); null leaves the choice to the machine.
+    pinnedPreset = null,
     // What every encoder this host may use was qualified and measured to do at
     // startup, by kind (`encode/calibration.js`). Takes the place of the one
     // array above, which a test may still hand in on its own.
@@ -771,8 +774,26 @@ export function wireOutputs({
   // Null where nothing was measured, and then nothing is re-encoded here.
   parts.calibration = calibration ?? null;
   const givenModes = Array.isArray(softwarePresetBenchmark) ? softwarePresetBenchmark : null;
+  // ONE PRESET FOR RUNS COMPARED AT ONE QUALITY. Three runs of one film on one
+  // host opened at `faster`, `veryfast` and `fast`, so no change could be
+  // compared at the same picture (r25, r26, r28, torrent-tv/meta#178). Pinned
+  // here, every decision that prices or chooses a software mode — the choice,
+  // the offer, the machine's admission and the command — sees the one mode,
+  // measured as calibrated. A preset calibration did not qualify is not
+  // pinned: the machine's own list stands, and the log says so.
+  let pinWarned = false;
+  const pinned = (modes) => {
+    if (!pinnedPreset || !Array.isArray(modes) || parts.encoders.current?.kind !== "software") return modes;
+    const kept = modes.filter(mode => mode.preset === pinnedPreset);
+    if (kept.length > 0) return kept;
+    if (!pinWarned) {
+      pinWarned = true;
+      logger.warn(`encode: --pin-preset ${pinnedPreset} names no calibrated mode (${modes.map(mode => mode.preset).join(", ")}); not pinned`);
+    }
+    return modes;
+  };
   Object.defineProperty(parts, "softwarePresetBenchmark", {
-    get: () => (parts.calibration
+    get: () => pinned(parts.calibration
       ? parts.calibration.modesFor(parts.encoders.current?.kind)
       : givenModes),
     enumerable: true

package/services/torrent/map-zones.js

@@ -63,6 +63,18 @@ export function deadlineOfMapZone(zone, now = Date.now()) {
     Number.isFinite(zone.withinSeconds) ? now + Math.max(0, zone.withinSeconds) * 1000 : Number.POSITIVE_INFINITY;
 }
 
+/**
+ * The byte ranges a zone names: a segment's zone names all the ranges its
+ * segment reads, and states its demand once for them (`ranges`); any other
+ * zone names one (`byteStart`-`byteEnd`). Inclusive.
+ *
+ * @param {{ ranges?: Array<[number, number]>, byteStart?: number, byteEnd?: number }} zone
+ * @returns {Array<[number, number]>}
+ */
+export function rangesOfZone(zone) {
+  return Array.isArray(zone.ranges) ? zone.ranges : [[zone.byteStart, zone.byteEnd]];
+}
+
 /**
  * The zones with every repeat of the same bytes left out but the one the swarm
  * would rank first.
@@ -80,14 +92,9 @@ export function deadlineOfMapZone(zone, now = Date.now()) {
  * @returns {Zone[]} In their order, without the repeats.
  */
 export function withoutRepeatedZones(zones, now = Date.now()) {
-  const keys = zones.map(zone => `${zone.byteStart}:${zone.byteEnd}`);
-  const best = new Map();
-  zones.forEach((zone, index) => {
-    const kept = best.get(keys[index]);
-    const band = { urgency: levelOfMapZone(zone), priority: priorityOfMapZone(zone), deadlineAt: deadlineOfMapZone(zone, now) };
-    if (!kept || compareBands(band, kept.band) < 0) best.set(keys[index], { index, band });
-  });
-  return zones.filter((_zone, index) => best.get(keys[index]).index === index);
+  const bands = zones.map(zone => bandOf(zone, now));
+  const best = bestOfRepeats(zones.length, index => zones[index].byteStart, index => zones[index].byteEnd, index => bands[index]);
+  return zones.filter((_zone, index) => best.get(zones[index].byteStart).get(zones[index].byteEnd) === index);
 }
 
 /**
@@ -105,45 +112,103 @@ export function withoutRepeatedZones(zones, now = Date.now()) {
  * consecutive segments are consecutive bytes. A zone a metadata read states
  * keeps its own request and order, and is left as it is.
  *
- * @template {{ byteStart: number, byteEnd: number }} Zone
- * @param {Zone[]} zones
+ * A segment's zone states its demand once for all its ranges
+ * ({@link rangesOfZone}); each range is a part of its own here, so a repeat is
+ * found range by range, and the band and deadline every range is packed with
+ * are the zone's current ones.
+ *
+ * @param {Array<{ byteStart?: number, byteEnd?: number, ranges?: Array<[number, number]> }>} zones
  * @param {number} [now]
  * @returns {object[]}
  */
 export function coalescedZones(zones, now = Date.now()) {
+  // Every range of every zone is a part, kept in numbers rather than objects:
+  // a film's map is ten thousand of them (r26, torrent-tv/meta#178).
+  let count = 0;
+  for (const zone of zones) count += Array.isArray(zone.ranges) ? zone.ranges.length : 1;
+  const starts = new Float64Array(count);
+  const ends = new Float64Array(count);
+  const owners = new Int32Array(count);
+  const bands = zones.map(zone => bandOf(zone, now));
+  let at = 0;
+  zones.forEach((zone, index) => {
+    for (const [start, end] of rangesOfZone(zone)) {
+      starts[at] = start;
+      ends[at] = end;
+      owners[at] = index;
+      at += 1;
+    }
+  });
+  const best = bestOfRepeats(count, part => starts[part], part => ends[part], part => bands[owners[part]]);
   const kept = [];
+  /** Band key → the band and its parts, in the order the bands first appear. */
   const groups = new Map();
-  for (const zone of withoutRepeatedZones(zones, now)) {
+  for (let part = 0; part < count; part++) {
+    if (best.get(starts[part]).get(ends[part]) !== part) continue;
+    const zone = zones[owners[part]];
     if (zone.requestId !== undefined || zone.downloadOnly || (Number.isSafeInteger(zone.order) && zone.order > 0)) {
       kept.push(zone);
       continue;
     }
-    const urgency = levelOfMapZone(zone);
-    const priority = priorityOfMapZone(zone);
-    const key = `${urgency}:${priority}`;
+    const band = bands[owners[part]];
+    const key = band.urgency * 1000 + band.priority;
     let group = groups.get(key);
-    if (!group) groups.set(key, group = { urgency, priority, parts: [] });
-    group.parts.push({ start: zone.byteStart, end: zone.byteEnd, deadlineAt: deadlineOfMapZone(zone, now) });
+    if (!group) groups.set(key, group = { band, parts: [] });
+    group.parts.push(part);
   }
-  for (const { urgency, priority, parts } of groups.values()) {
-    parts.sort((left, right) => left.start - right.start);
-    let window = null;
+  for (const { band: { urgency, priority }, parts } of groups.values()) {
+    parts.sort((left, right) => starts[left] - starts[right]);
+    let windowStart = 0;
+    let windowEnd = -1;
+    let windowDeadline = Number.POSITIVE_INFINITY;
+    let open = false;
     const close = () => {
-      if (!window) return;
-      kept.push({ byteStart: window.start, byteEnd: window.end, priority, urgent: urgency === Urgency.NEAR,
+      if (!open) return;
+      kept.push({ byteStart: windowStart, byteEnd: windowEnd, priority, urgent: urgency === Urgency.NEAR,
         behind: urgency === Urgency.BEHIND, deferred: urgency === Urgency.TAIL,
-        ...(Number.isFinite(window.deadlineAt) ? { deadlineAt: window.deadlineAt } : {}) });
+        ...(Number.isFinite(windowDeadline) ? { deadlineAt: windowDeadline } : {}) });
     };
     for (const part of parts) {
-      if (window && part.start <= window.end + 1) {
-        window.end = Math.max(window.end, part.end);
-        window.deadlineAt = Math.min(window.deadlineAt, part.deadlineAt);
+      const deadlineAt = bands[owners[part]].deadlineAt;
+      if (open && starts[part] <= windowEnd + 1) {
+        windowEnd = Math.max(windowEnd, ends[part]);
+        windowDeadline = Math.min(windowDeadline, deadlineAt);
       } else {
         close();
-        window = { ...part };
+        open = true;
+        windowStart = starts[part];
+        windowEnd = ends[part];
+        windowDeadline = deadlineAt;
       }
     }
     close();
   }
   return kept;
 }
+
+/** The band a zone's bytes are fetched in. */
+function bandOf(zone, now) {
+  return { urgency: levelOfMapZone(zone), priority: priorityOfMapZone(zone), deadlineAt: deadlineOfMapZone(zone, now) };
+}
+
+/**
+ * Which of the parts naming each exact byte range is the best: by band
+ * ({@link compareBands}), the first of equals. Kept by the range's two numbers
+ * rather than a string made of them: making a key of each of a film's ten
+ * thousand ranges was most of what packing its map cost (r26,
+ * torrent-tv/meta#178).
+ *
+ * @returns {Map<number, Map<number, number>>} start → end → the best part.
+ */
+function bestOfRepeats(count, startOf, endOf, bandOfPart) {
+  const best = new Map();
+  for (let part = 0; part < count; part++) {
+    const start = startOf(part);
+    const end = endOf(part);
+    let byEnd = best.get(start);
+    if (!byEnd) best.set(start, byEnd = new Map());
+    const kept = byEnd.get(end);
+    if (kept === undefined || compareBands(bandOfPart(part), bandOfPart(kept)) < 0) byEnd.set(end, part);
+  }
+  return best;
+}

package/services/viewer/target-input-expansion.js

@@ -102,8 +102,11 @@ function merged(ranges) {
  * The map's byte zones with each output's next run input raised to its first
  * segment's place.
  *
- * @template {{ outputKey: string, index: number, priority: number, urgent?: boolean, behind?: boolean, deferred?: boolean, byteStart: number, byteEnd: number }} Zone
- * @param {Zone[]} zones - One or more per segment, as the map's pass made them.
+ * @template {{ outputKey: string, index: number, priority: number, urgent?: boolean, behind?: boolean, deferred?: boolean,
+ *   ranges?: Array<[number, number]>, byteStart?: number, byteEnd?: number }} Zone
+ * @param {Zone[]} zones - One or more per segment, as the map's pass made
+ *   them: a segment's zone names all its ranges (`ranges`), any other one
+ *   range (`byteStart`-`byteEnd`).
  * @param {(outputKey: string) => number | null} targetBytesOf - What the
  *   output's next run would be given to copy; nothing is raised without it.
  * @returns {Zone[]} In their order; raised zones are copies.
@@ -138,7 +141,7 @@ export function expandToTargetInputs(zones, targetBytesOf) {
     if (zone.index === run.first.index) {
       let ranges = held.get(zone.outputKey);
       if (!ranges) held.set(zone.outputKey, ranges = []);
-      ranges.push([zone.byteStart, zone.byteEnd]);
+      ranges.push(...rangesOf(zone));
     } else if (zone.index > run.first.index) {
       const list = run.after.get(zone.index);
       if (list) list.push(zone);
@@ -153,7 +156,7 @@ export function expandToTargetInputs(zones, targetBytesOf) {
     // or not yet worked out, ends it.
     for (let index = run.first.index + 1; run.after.has(index); index++) {
       const list = run.after.get(index);
-      const ranges = list.map(zone => [zone.byteStart, zone.byteEnd]);
+      const ranges = list.flatMap(rangesOf);
       if (holding.total + holding.adds(ranges) > run.target) break;
       holding.add(ranges);
       for (const zone of list) {
@@ -164,3 +167,8 @@ export function expandToTargetInputs(zones, targetBytesOf) {
   }
   return raised.size === 0 ? zones : zones.map(zone => raised.get(zone) ?? zone);
 }
+
+/** The ranges a zone names, inclusive. */
+function rangesOf(zone) {
+  return Array.isArray(zone.ranges) ? zone.ranges : [[zone.byteStart, zone.byteEnd]];
+}