@torrent-tv/proxy 2.97.28 → 2.97.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.97.28",
+  "version": "2.97.30",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -18,6 +18,7 @@ import { IndexMemory } from "./services/storage/IndexMemory.js";
 import { probePackets } from "./services/media/probe-packets.js";
 import { segmentDemands, sourceDeadline } from "./services/viewer/segment-demands.js";
 import { expandToTargetInputs } from "./services/viewer/target-input-expansion.js";
+import { SegmentInputTable } from "./services/viewer/segment-input-table.js";
 import { SourcePreparation } from "./services/viewer/SourcePreparation.js";
 import { handleHealthzGet } from "./routes/healthz/get.js";
 import { handleApiDeliverySinkGet } from "./routes/api/delivery-sink/get.js";
@@ -185,6 +186,7 @@ export async function startProxyServer({
       for (const key of segmentSourceRanges.keys()) if (key.startsWith(`${sourceKey}:`)) segmentSourceRanges.delete(key);
       for (const key of segmentReadings.keys()) if (key.startsWith(`${sourceKey}:`)) segmentReadings.delete(key);
       for (const key of resolvedPasses.keys()) if (key.startsWith(`${sourceKey}:`)) resolvedPasses.delete(key);
+      segmentInputs.forget(sourceKey);
       outputParts.encodeInputs?.bytesChanged();
     },
     // Pieces arriving is ANNOUNCED by the thread that owns the swarm; what is
@@ -476,6 +478,33 @@ export async function startProxyServer({
    * from and what it resolved (`playback-pass.js`); forgotten with the source.
    */
   const resolvedPasses = new Map();
+  /** Which source bytes each segment of an output reads, for the forecast (`segment-input-table.js`). */
+  const segmentInputs = new SegmentInputTable();
+  // WHAT THE PASSES COST, said once a minute: how many were taken whole from
+  // the last one and how many worked out again, how many segments and ranges
+  // those went through, how many zones they built, and the time from a pass's
+  // start to its result — which includes waits for reads, so it bounds the
+  // main thread's share rather than measuring it. A pass worked out again
+  // whose zones state the same bytes as the last is the repeated work worth
+  // removing; how often that happens is what the map's own "unchanged not
+  // sent" count says (torrent-tv/meta#178 review).
+  const passCosts = { since: Date.now(), reused: 0, worked: 0, incomplete: 0, segments: 0, ranges: 0, zones: 0, built: 0, buildMs: 0, ms: 0, maxMs: 0 };
+  const notePass = ({ reused = false, complete = true, segments = 0, ranges = 0, zones = 0, ms = 0 }) => {
+    if (reused) passCosts.reused += 1;
+    else {
+      passCosts.worked += 1;
+      if (!complete) passCosts.incomplete += 1;
+      passCosts.segments += segments; passCosts.ranges += ranges; passCosts.zones += zones;
+      passCosts.ms += ms; passCosts.maxMs = Math.max(passCosts.maxMs, ms);
+    }
+    const now = Date.now();
+    if (now - passCosts.since < 60_000) return;
+    logger.info(`download map passes in ${((now - passCosts.since) / 1000).toFixed(1)}s: ${passCosts.reused} taken whole from the last, ` +
+      `${passCosts.worked} worked out (${passCosts.incomplete} incomplete) over ${passCosts.segments} segment(s) and ${passCosts.ranges} range(s), ` +
+      `giving ${passCosts.zones} zone(s); ${Math.round(passCosts.ms)}ms from start to result in all, longest ${Math.round(passCosts.maxMs)}ms; ` +
+      `${passCosts.built} zone list(s) built for the map in ${Math.round(passCosts.buildMs)}ms`);
+    Object.assign(passCosts, { since: now, reused: 0, worked: 0, incomplete: 0, segments: 0, ranges: 0, zones: 0, built: 0, buildMs: 0, ms: 0, maxMs: 0 });
+  };
   /** A segment's ranges joined at the precision the torrent fetches, once per kept list and piece length. */
   const joinedRanges = new WeakMap();
   const joinedRangesOf = (ranges, pieceLength) => {
@@ -513,12 +542,20 @@ export async function startProxyServer({
       // A zone's ranges are stated to the torrent at the precision it fetches:
       // ranges closer than a piece hold the same pieces as one.
       const pieceLength = Number(torrentPool.knownTorrent(map.sourceKey)?.pieceLength);
+      const passStartedAt = performance.now();
+      let segments = 0;
       const converted = [];
       const demands = [];
       // What each output's next run would copy, asked of the swarm as one group
       // with its first segment (`target-input-expansion.js`).
       const targets = new Map();
-      const expanded = () => expandToTargetInputs(converted, outputKey => targets.get(outputKey) ?? null);
+      const expanded = () => {
+        const startedAt = performance.now();
+        const zones = expandToTargetInputs(converted, outputKey => targets.get(outputKey) ?? null);
+        passCosts.built += 1;
+        passCosts.buildMs += performance.now() - startedAt;
+        return zones;
+      };
       for (const output of outputParts.outputs.values()) {
         if (output.file.sourceKey !== map.sourceKey) continue;
         const consumers = [...outputParts.viewers.forOutput(output).values()].filter(viewer =>
@@ -552,7 +589,10 @@ export async function startProxyServer({
       const fileKey = `${map.sourceKey}:${map.fileIndex}`;
       const pass = { tracks: tracks.value, ...passInputs({ pieceLength, shift, targets, demands }) };
       const last = resolvedPasses.get(fileKey);
-      if (samePass(last, pass)) return last.result;
+      if (samePass(last, pass)) {
+        notePass({ reused: true });
+        return last.result;
+      }
       // Kept only from a pass that worked every segment out; one that met a
       // reading not there yet is worked out again until it is.
       let complete = true;
@@ -588,7 +628,12 @@ export async function startProxyServer({
         const { wanted, modes, interval, rangesKey } = reading;
         const { tracks: _choices, owner: _owner, sourceInterval: _sourceInterval, ...demand } = zone;
         const convert = ranges => {
-          for (const [byteStart, byteEnd] of joinedRangesOf(ranges, pieceLength)) converted.push({ ...demand,
+          const joined = joinedRangesOf(ranges, pieceLength);
+          segments += 1;
+          // What the segment reads is kept for the forecast in a table of its
+          // own, whatever form the torrent is told it in.
+          segmentInputs.set(map.sourceKey, map.fileIndex, zone.outputKey, zone.index, joined);
+          for (const [byteStart, byteEnd] of joined) converted.push({ ...demand,
             downloadInterval: { from: zone.owner.from, to: zone.owner.to }, byteStart, byteEnd });
         };
         // The bytes one segment needs are a fact of the file and the output, so
@@ -649,6 +694,7 @@ export async function startProxyServer({
         convert(input.ranges);
       }
       const result = expanded();
+      notePass({ complete, segments, ranges: converted.length, zones: result.length, ms: performance.now() - passStartedAt });
       if (complete) resolvedPasses.set(fileKey, { ...pass, result });
       else resolvedPasses.delete(fileKey);
       return result;
@@ -751,7 +797,7 @@ export async function startProxyServer({
     readMetadataActivity: () => containerOrchestrator.activity(),
     onViewerChanged: () => { void sourcePreparation?.refresh(); onStateChanged(); },
     onEncodersChanged: () => onStateChanged(),
-    sourceInputsFor: (output, index) => downloadMaps.inputsForOutput(output.file.sourceKey, output.outputKey, index),
+    sourceInputsFor: (output, index) => segmentInputs.inputsFor(output.file.sourceKey, output.outputKey, index),
     readSourceMedia: async (params) => {
       await playbackPlanner.getPlan(params);
       const media = playbackPlanner.getCachedMediaInfo(params);

package/services/viewer/DownloadMaps.js

@@ -20,8 +20,6 @@ export class DownloadMaps {
   #clock;
   /** What the torrent is told of the playback zones: as many windows as say the same thing. */
   #pack;
-  /** What a list of zones says each segment reads, by the list. @type {WeakMap<object[], Map>} */
-  #inputsOf = new WeakMap();
 
   constructor({ publish, resolvePlayback = async map => map.zones, log = null, now = Date.now,
     clock = () => performance.timeOrigin + performance.now(), pack = zones => zones }) {
@@ -59,10 +57,19 @@ export class DownloadMaps {
       }
       const deadlineAt = Number.isFinite(current.deadlineAt)
         ? current.deadlineAt + (Math.max(0, zone.from - current.from) - (zone.leadSeconds ?? 0)) * 1000 : current.deadlineAt;
-      if (Object.keys(current).some(key => key !== "from" && key !== "to" && !Object.is(zone[key], key === "deadlineAt" ? deadlineAt : current[key]))) changed = true;
+      // A deadline worked out again on a later clock is no change: the
+      // torrent orders by deadlines and is sent their order, which the pass
+      // below states with the rest.
+      if (Object.keys(current).some(key => key !== "from" && key !== "to" && key !== "deadlineAt" && key !== "withinSeconds" &&
+        !Object.is(zone[key], current[key]))) changed = true;
       return [{ ...zone, ...current, from: zone.from, to: zone.to, deadlineAt }];
     });
-    this.#setZones(file, retained);
+    // THE ZONES STAND AS THEY ARE UNLESS ONE IS WITHDRAWN OR RE-WORDED. Set
+    // anew on every pass, they were a new list each time, so every metadata
+    // statement during the pass packed the whole film again: 24 s of the main
+    // thread in fifteen minutes of a two-hour AVI on Home Assistant (r24,
+    // torrent-tv/meta#178).
+    if (changed) this.#setZones(file, retained);
     for (const [statement, metadata] of file.metadata) {
       if (metadata.interval && !this.wantsInterval(sourceKey, fileIndex, metadata.interval)) {
         file.metadata.delete(statement);
@@ -143,19 +150,6 @@ export class DownloadMaps {
     return this.#files.get(`${sourceKey}:${fileIndex}`)?.sourceZones.some(zone => zone.from === from && zone.to === to) ?? false;
   }
 
-  /** Exact source ranges already resolved by the download map, without new reads. */
-  inputsForOutput(sourceKey, outputKey, index) {
-    const sources = [];
-    for (const file of this.#files.values()) {
-      if (file.sourceKey !== sourceKey) continue;
-      const ranges = file.inputs.get(outputKey)?.get(index);
-      if (!ranges?.length) continue;
-      sources.push({ sourceId: `${sourceKey}:${file.fileIndex}`,
-        ranges });
-    }
-    return sources.length ? sources : null;
-  }
-
   metadata({ sourceKey, fileIndex, statement, result, priority = 100, urgent = true, deadlineAt = 0, interval, leadSeconds = 0, scope = "playback", order = 0 }) {
     if (!Number.isFinite(leadSeconds) || leadSeconds < 0) throw new TypeError("Metadata preparation time must be finite and nonnegative.");
     const file = this.#file(sourceKey, fileIndex);
@@ -246,7 +240,7 @@ export class DownloadMaps {
     const key = `${sourceKey}:${fileIndex}`;
     let file = this.#files.get(key);
     if (!file) {
-      file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], inputs: new Map(), nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve(), dirty: true };
+      file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve(), dirty: true };
       this.#files.set(key, file);
     }
     return file;
@@ -256,24 +250,6 @@ export class DownloadMaps {
     file.zones = zones;
     // What `#emit` reads has changed, whether or not it is published now.
     file.dirty = true;
-    // A pass that resolves what the pass before resolved hands back the same
-    // zones (`resolvedPasses` in server.js), and what they say of each segment
-    // is the same too.
-    const known = this.#inputsOf.get(zones);
-    if (known) {
-      file.inputs = known;
-      return;
-    }
-    file.inputs = new Map();
-    this.#inputsOf.set(zones, file.inputs);
-    for (const zone of zones) {
-      if (typeof zone.outputKey !== "string" || !Number.isSafeInteger(zone.index)) continue;
-      let output = file.inputs.get(zone.outputKey);
-      if (!output) file.inputs.set(zone.outputKey, output = new Map());
-      let ranges = output.get(zone.index);
-      if (!ranges) output.set(zone.index, ranges = []);
-      ranges.push({ start: zone.byteStart, end: zone.byteEnd + 1 });
-    }
   }
 
   #emit(file) {

package/services/viewer/segment-input-table.js

@@ -0,0 +1,53 @@
+/**
+ * @file Which source bytes each segment of an output reads, as the download
+ * map's pass worked them out (torrent-tv/meta#178).
+ *
+ * A fact of the file and the output, kept apart from what the torrent is told:
+ * the far part of a film is stated to the torrent as windows of a band
+ * (`download-bands.js`), and the forecast still has to find every segment's own
+ * bytes (`ViewerRequests` `#segmentsOf`). It used to read them off the zones
+ * sent, which named every segment.
+ */
+
+export class SegmentInputTable {
+  /** `sourceKey` → output → segment → file → its ranges. @type {Map<string, Map<string, Map<number, Map<number, { joined: Array<[number, number]>, ranges: Array<{ start: number, end: number }> }>>>>} */
+  #bySource = new Map();
+
+  /**
+   * @param {string} sourceKey
+   * @param {number} fileIndex
+   * @param {string} outputKey
+   * @param {number} index
+   * @param {Array<[number, number]>} joined - Inclusive ranges.
+   */
+  set(sourceKey, fileIndex, outputKey, index, joined) {
+    let outputs = this.#bySource.get(sourceKey);
+    if (!outputs) this.#bySource.set(sourceKey, outputs = new Map());
+    let segments = outputs.get(outputKey);
+    if (!segments) outputs.set(outputKey, segments = new Map());
+    let files = segments.get(index);
+    if (!files) segments.set(index, files = new Map());
+    // The same reading keeps its list: whoever compares inputs compares them by it.
+    if (files.get(fileIndex)?.joined === joined) return;
+    files.set(fileIndex, { joined, ranges: joined.map(([start, end]) => ({ start, end: end + 1 })) });
+  }
+
+  /**
+   * The bytes one segment reads, by file, or null where none are known.
+   *
+   * @param {string} sourceKey
+   * @param {string} outputKey
+   * @param {number} index
+   * @returns {Array<{ sourceId: string, ranges: Array<{ start: number, end: number }> }> | null}
+   */
+  inputsFor(sourceKey, outputKey, index) {
+    const files = this.#bySource.get(sourceKey)?.get(outputKey)?.get(index);
+    if (!files?.size) return null;
+    return [...files].map(([fileIndex, { ranges }]) => ({ sourceId: `${sourceKey}:${fileIndex}`, ranges }));
+  }
+
+  /** @param {string} sourceKey */
+  forget(sourceKey) {
+    this.#bySource.delete(sourceKey);
+  }
+}

package/services/viewer/target-input-expansion.js

@@ -19,22 +19,83 @@
  */
 
 /**
- * Bytes covered by a list of inclusive ranges, each byte counted once: the
- * declarations and indexes every segment of an original file names are held
- * once by the run that reads them.
- *
- * @param {Array<[number, number]>} ranges
- * @returns {number}
+ * The bytes a run's input holds, each counted once: the declarations and
+ * indexes every segment of an original file names are held once by the run
+ * that reads them. Kept merged and in order, so a segment's ranges are
+ * counted against it by search rather than by sorting everything held again —
+ * a soundtrack's segments add a few bytes each, and its stretch is walked far.
  */
-function coveredBytes(ranges) {
-  let total = 0;
-  let end = -1;
-  for (const [start, last] of [...ranges].sort((left, right) => left[0] - right[0])) {
-    if (last <= end) continue;
-    total += last - Math.max(start, end + 1) + 1;
-    end = last;
+class HeldBytes {
+  /** Inclusive [start, end], merged, ascending. @type {Array<[number, number]>} */
+  #ranges = [];
+  total = 0;
+
+  /** @param {Array<[number, number]>} ranges */
+  constructor(ranges = []) {
+    this.add(ranges);
+  }
+
+  /** The first held range that ends at or after `at`. */
+  #from(at) {
+    let low = 0;
+    let high = this.#ranges.length;
+    while (low < high) {
+      const middle = (low + high) >> 1;
+      if (this.#ranges[middle][1] < at) low = middle + 1;
+      else high = middle;
+    }
+    return low;
+  }
+
+  /**
+   * What these ranges would add, without adding them.
+   *
+   * @param {Array<[number, number]>} ranges
+   * @returns {number}
+   */
+  adds(ranges) {
+    let added = 0;
+    for (const [start, end] of merged(ranges)) {
+      let at = start;
+      for (let index = this.#from(start); at <= end && index < this.#ranges.length && this.#ranges[index][0] <= end; index++) {
+        const [heldStart, heldEnd] = this.#ranges[index];
+        if (heldStart > at) added += heldStart - at;
+        at = Math.max(at, heldEnd + 1);
+      }
+      if (at <= end) added += end - at + 1;
+    }
+    return added;
+  }
+
+  /** @param {Array<[number, number]>} ranges */
+  add(ranges) {
+    for (const [start, end] of merged(ranges)) {
+      this.total += this.adds([[start, end]]);
+      // Joined with every held range it touches or overlaps.
+      const first = this.#from(start - 1);
+      let last = first;
+      let joinedStart = start;
+      let joinedEnd = end;
+      while (last < this.#ranges.length && this.#ranges[last][0] <= end + 1) {
+        joinedStart = Math.min(joinedStart, this.#ranges[last][0]);
+        joinedEnd = Math.max(joinedEnd, this.#ranges[last][1]);
+        last++;
+      }
+      this.#ranges.splice(first, last - first, [joinedStart, joinedEnd]);
+    }
   }
-  return total;
+}
+
+/** Inclusive ranges merged where they overlap or touch, ascending. @param {Array<[number, number]>} ranges */
+function merged(ranges) {
+  const sorted = [...ranges].sort((left, right) => left[0] - right[0]);
+  const out = [];
+  for (const [start, end] of sorted) {
+    const previous = out.at(-1);
+    if (previous && start <= previous[1] + 1) previous[1] = Math.max(previous[1], end);
+    else out.push([start, end]);
+  }
+  return out;
 }
 
 /**
@@ -48,39 +109,56 @@ function coveredBytes(ranges) {
  * @returns {Zone[]} In their order; raised zones are copies.
  */
 export function expandToTargetInputs(zones, targetBytesOf) {
-  /** @type {Map<string, Map<number, Zone[]>>} */
-  const byOutput = new Map();
+  // ONE WALK FOR EACH OUTPUT'S FIRST SEGMENT, and the segments after it only
+  // as far as the run's input reaches. The map names every segment of a film,
+  // and each pass used to group all of them and work out each one's highest
+  // priority before looking at the few the run copies: 26 s of the main thread
+  // in fifteen minutes of a two-hour AVI on Home Assistant (r24,
+  // torrent-tv/meta#178).
+  /** @type {Map<string, { index: number, priority: number, zone: Zone }>} */
+  const first = new Map();
   for (const zone of zones) {
-    let segments = byOutput.get(zone.outputKey);
-    if (!segments) byOutput.set(zone.outputKey, segments = new Map());
-    const list = segments.get(zone.index);
-    if (list) list.push(zone);
-    else segments.set(zone.index, [zone]);
+    const best = first.get(zone.outputKey);
+    if (!best || zone.priority > best.priority || (zone.priority === best.priority && zone.index < best.index)) {
+      first.set(zone.outputKey, { index: zone.index, priority: zone.priority, zone });
+    }
   }
-  /** @type {Map<Zone, Zone>} */
-  const raised = new Map();
-  for (const [outputKey, segments] of byOutput) {
+  /** @type {Map<string, { first: { index: number, priority: number, zone: Zone }, target: number, after: Map<number, Zone[]> }>} */
+  const runs = new Map();
+  for (const [outputKey, best] of first) {
     const target = targetBytesOf(outputKey);
-    if (!(target > 0)) continue;
-    let first = null;
-    for (const [index, list] of segments) {
-      const priority = Math.max(...list.map(zone => zone.priority));
-      if (first === null || priority > first.priority || (priority === first.priority && index < first.index)) {
-        first = { index, priority, zone: list.find(zone => zone.priority === priority) };
-      }
+    if (target > 0) runs.set(outputKey, { first: best, target, after: new Map() });
+  }
+  if (runs.size === 0) return zones;
+  /** @type {Map<string, Array<[number, number]>>} */
+  const held = new Map();
+  for (const zone of zones) {
+    const run = runs.get(zone.outputKey);
+    if (!run) continue;
+    if (zone.index === run.first.index) {
+      let ranges = held.get(zone.outputKey);
+      if (!ranges) held.set(zone.outputKey, ranges = []);
+      ranges.push([zone.byteStart, zone.byteEnd]);
+    } else if (zone.index > run.first.index) {
+      const list = run.after.get(zone.index);
+      if (list) list.push(zone);
+      else run.after.set(zone.index, [zone]);
     }
-    if (first === null) continue;
-    const held = segments.get(first.index).map(zone => [zone.byteStart, zone.byteEnd]);
+  }
+  /** @type {Map<Zone, Zone>} */
+  const raised = new Map();
+  for (const [outputKey, run] of runs) {
+    const holding = new HeldBytes(held.get(outputKey) ?? []);
     // The run's input is a stretch of consecutive segments: one already made,
     // or not yet worked out, ends it.
-    for (let index = first.index + 1; segments.has(index); index++) {
-      const list = segments.get(index);
-      const ranges = [...held, ...list.map(zone => [zone.byteStart, zone.byteEnd])];
-      if (coveredBytes(ranges) > target) break;
-      held.push(...list.map(zone => [zone.byteStart, zone.byteEnd]));
+    for (let index = run.first.index + 1; run.after.has(index); index++) {
+      const list = run.after.get(index);
+      const ranges = list.map(zone => [zone.byteStart, zone.byteEnd]);
+      if (holding.total + holding.adds(ranges) > run.target) break;
+      holding.add(ranges);
       for (const zone of list) {
-        raised.set(zone, { ...zone, priority: first.priority, urgent: first.zone.urgent,
-          behind: first.zone.behind, deferred: first.zone.deferred });
+        raised.set(zone, { ...zone, priority: run.first.priority, urgent: run.first.zone.urgent,
+          behind: run.first.zone.behind, deferred: run.first.zone.deferred });
       }
     }
   }