@torrent-tv/proxy 2.97.29 → 2.97.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.97.29",
+  "version": "2.97.31",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -18,6 +18,7 @@ import { IndexMemory } from "./services/storage/IndexMemory.js";
 import { probePackets } from "./services/media/probe-packets.js";
 import { segmentDemands, sourceDeadline } from "./services/viewer/segment-demands.js";
 import { expandToTargetInputs } from "./services/viewer/target-input-expansion.js";
+import { SegmentInputTable } from "./services/viewer/segment-input-table.js";
 import { SourcePreparation } from "./services/viewer/SourcePreparation.js";
 import { handleHealthzGet } from "./routes/healthz/get.js";
 import { handleApiDeliverySinkGet } from "./routes/api/delivery-sink/get.js";
@@ -185,6 +186,7 @@ export async function startProxyServer({
       for (const key of segmentSourceRanges.keys()) if (key.startsWith(`${sourceKey}:`)) segmentSourceRanges.delete(key);
       for (const key of segmentReadings.keys()) if (key.startsWith(`${sourceKey}:`)) segmentReadings.delete(key);
       for (const key of resolvedPasses.keys()) if (key.startsWith(`${sourceKey}:`)) resolvedPasses.delete(key);
+      segmentInputs.forget(sourceKey);
       outputParts.encodeInputs?.bytesChanged();
     },
     // Pieces arriving is ANNOUNCED by the thread that owns the swarm; what is
@@ -476,6 +478,33 @@ export async function startProxyServer({
    * from and what it resolved (`playback-pass.js`); forgotten with the source.
    */
   const resolvedPasses = new Map();
+  /** Which source bytes each segment of an output reads, for the forecast (`segment-input-table.js`). */
+  const segmentInputs = new SegmentInputTable();
+  // WHAT THE PASSES COST, said once a minute: how many were taken whole from
+  // the last one and how many worked out again, how many segments and ranges
+  // those went through, how many zones they built, and the time from a pass's
+  // start to its result — which includes waits for reads, so it bounds the
+  // main thread's share rather than measuring it. A pass worked out again
+  // whose zones state the same bytes as the last is the repeated work worth
+  // removing; how often that happens is what the map's own "unchanged not
+  // sent" count says (torrent-tv/meta#178 review).
+  const passCosts = { since: Date.now(), reused: 0, worked: 0, incomplete: 0, segments: 0, ranges: 0, zones: 0, built: 0, buildMs: 0, ms: 0, maxMs: 0 };
+  const notePass = ({ reused = false, complete = true, segments = 0, ranges = 0, zones = 0, ms = 0 }) => {
+    if (reused) passCosts.reused += 1;
+    else {
+      passCosts.worked += 1;
+      if (!complete) passCosts.incomplete += 1;
+      passCosts.segments += segments; passCosts.ranges += ranges; passCosts.zones += zones;
+      passCosts.ms += ms; passCosts.maxMs = Math.max(passCosts.maxMs, ms);
+    }
+    const now = Date.now();
+    if (now - passCosts.since < 60_000) return;
+    logger.info(`download map passes in ${((now - passCosts.since) / 1000).toFixed(1)}s: ${passCosts.reused} taken whole from the last, ` +
+      `${passCosts.worked} worked out (${passCosts.incomplete} incomplete) over ${passCosts.segments} segment(s) and ${passCosts.ranges} range(s), ` +
+      `giving ${passCosts.zones} zone(s); ${Math.round(passCosts.ms)}ms from start to result in all, longest ${Math.round(passCosts.maxMs)}ms; ` +
+      `${passCosts.built} zone list(s) built for the map in ${Math.round(passCosts.buildMs)}ms`);
+    Object.assign(passCosts, { since: now, reused: 0, worked: 0, incomplete: 0, segments: 0, ranges: 0, zones: 0, built: 0, buildMs: 0, ms: 0, maxMs: 0 });
+  };
   /** A segment's ranges joined at the precision the torrent fetches, once per kept list and piece length. */
   const joinedRanges = new WeakMap();
   const joinedRangesOf = (ranges, pieceLength) => {
@@ -513,12 +542,20 @@ export async function startProxyServer({
       // A zone's ranges are stated to the torrent at the precision it fetches:
       // ranges closer than a piece hold the same pieces as one.
       const pieceLength = Number(torrentPool.knownTorrent(map.sourceKey)?.pieceLength);
+      const passStartedAt = performance.now();
+      let rangeCount = 0;
       const converted = [];
       const demands = [];
       // What each output's next run would copy, asked of the swarm as one group
       // with its first segment (`target-input-expansion.js`).
       const targets = new Map();
-      const expanded = () => expandToTargetInputs(converted, outputKey => targets.get(outputKey) ?? null);
+      const expanded = () => {
+        const startedAt = performance.now();
+        const zones = expandToTargetInputs(converted, outputKey => targets.get(outputKey) ?? null);
+        passCosts.built += 1;
+        passCosts.buildMs += performance.now() - startedAt;
+        return zones;
+      };
       for (const output of outputParts.outputs.values()) {
         if (output.file.sourceKey !== map.sourceKey) continue;
         const consumers = [...outputParts.viewers.forOutput(output).values()].filter(viewer =>
@@ -552,7 +589,10 @@ export async function startProxyServer({
       const fileKey = `${map.sourceKey}:${map.fileIndex}`;
       const pass = { tracks: tracks.value, ...passInputs({ pieceLength, shift, targets, demands }) };
       const last = resolvedPasses.get(fileKey);
-      if (samePass(last, pass)) return last.result;
+      if (samePass(last, pass)) {
+        notePass({ reused: true });
+        return last.result;
+      }
       // Kept only from a pass that worked every segment out; one that met a
       // reading not there yet is worked out again until it is.
       let complete = true;
@@ -588,8 +628,19 @@ export async function startProxyServer({
         const { wanted, modes, interval, rangesKey } = reading;
         const { tracks: _choices, owner: _owner, sourceInterval: _sourceInterval, ...demand } = zone;
         const convert = ranges => {
-          for (const [byteStart, byteEnd] of joinedRangesOf(ranges, pieceLength)) converted.push({ ...demand,
-            downloadInterval: { from: zone.owner.from, to: zone.owner.to }, byteStart, byteEnd });
+          const joined = joinedRangesOf(ranges, pieceLength);
+          rangeCount += joined.length;
+          // What the segment reads is kept for the forecast in a table of its
+          // own, whatever form the torrent is told it in.
+          segmentInputs.set(map.sourceKey, map.fileIndex, zone.outputKey, zone.index, joined);
+          // ONE ZONE FOR THE SEGMENT, its demand stated once for all the
+          // ranges it reads. The demand — level, priority, deadline, the map
+          // zone it belongs to — is this pass's; the ranges are the list kept
+          // for the segment, unchanged from pass to pass. A zone for each
+          // range copied the demand into every one: ten thousand objects a
+          // pass over a two-hour AVI, made again 21-61 times a minute on Home
+          // Assistant (r26, torrent-tv/meta#178).
+          converted.push({ ...demand, downloadInterval: { from: zone.owner.from, to: zone.owner.to }, ranges: joined });
         };
         // The bytes one segment needs are a fact of the file and the output, so
         // they are worked out once. A whole film is thousands of segments and
@@ -649,6 +700,7 @@ export async function startProxyServer({
         convert(input.ranges);
       }
       const result = expanded();
+      notePass({ complete, segments: converted.length, ranges: rangeCount, zones: result.length, ms: performance.now() - passStartedAt });
       if (complete) resolvedPasses.set(fileKey, { ...pass, result });
       else resolvedPasses.delete(fileKey);
       return result;
@@ -751,7 +803,7 @@ export async function startProxyServer({
     readMetadataActivity: () => containerOrchestrator.activity(),
     onViewerChanged: () => { void sourcePreparation?.refresh(); onStateChanged(); },
     onEncodersChanged: () => onStateChanged(),
-    sourceInputsFor: (output, index) => downloadMaps.inputsForOutput(output.file.sourceKey, output.outputKey, index),
+    sourceInputsFor: (output, index) => segmentInputs.inputsFor(output.file.sourceKey, output.outputKey, index),
     readSourceMedia: async (params) => {
       await playbackPlanner.getPlan(params);
       const media = playbackPlanner.getCachedMediaInfo(params);

package/services/torrent/map-zones.js

@@ -63,6 +63,18 @@ export function deadlineOfMapZone(zone, now = Date.now()) {
     Number.isFinite(zone.withinSeconds) ? now + Math.max(0, zone.withinSeconds) * 1000 : Number.POSITIVE_INFINITY;
 }
 
+/**
+ * The byte ranges a zone names: a segment's zone names all the ranges its
+ * segment reads, and states its demand once for them (`ranges`); any other
+ * zone names one (`byteStart`-`byteEnd`). Inclusive.
+ *
+ * @param {{ ranges?: Array<[number, number]>, byteStart?: number, byteEnd?: number }} zone
+ * @returns {Array<[number, number]>}
+ */
+export function rangesOfZone(zone) {
+  return Array.isArray(zone.ranges) ? zone.ranges : [[zone.byteStart, zone.byteEnd]];
+}
+
 /**
  * The zones with every repeat of the same bytes left out but the one the swarm
  * would rank first.
@@ -80,14 +92,9 @@ export function deadlineOfMapZone(zone, now = Date.now()) {
  * @returns {Zone[]} In their order, without the repeats.
  */
 export function withoutRepeatedZones(zones, now = Date.now()) {
-  const keys = zones.map(zone => `${zone.byteStart}:${zone.byteEnd}`);
-  const best = new Map();
-  zones.forEach((zone, index) => {
-    const kept = best.get(keys[index]);
-    const band = { urgency: levelOfMapZone(zone), priority: priorityOfMapZone(zone), deadlineAt: deadlineOfMapZone(zone, now) };
-    if (!kept || compareBands(band, kept.band) < 0) best.set(keys[index], { index, band });
-  });
-  return zones.filter((_zone, index) => best.get(keys[index]).index === index);
+  const bands = zones.map(zone => bandOf(zone, now));
+  const best = bestOfRepeats(zones.length, index => zones[index].byteStart, index => zones[index].byteEnd, index => bands[index]);
+  return zones.filter((_zone, index) => best.get(zones[index].byteStart).get(zones[index].byteEnd) === index);
 }
 
 /**
@@ -105,45 +112,103 @@ export function withoutRepeatedZones(zones, now = Date.now()) {
  * consecutive segments are consecutive bytes. A zone a metadata read states
  * keeps its own request and order, and is left as it is.
  *
- * @template {{ byteStart: number, byteEnd: number }} Zone
- * @param {Zone[]} zones
+ * A segment's zone states its demand once for all its ranges
+ * ({@link rangesOfZone}); each range is a part of its own here, so a repeat is
+ * found range by range, and the band and deadline every range is packed with
+ * are the zone's current ones.
+ *
+ * @param {Array<{ byteStart?: number, byteEnd?: number, ranges?: Array<[number, number]> }>} zones
  * @param {number} [now]
  * @returns {object[]}
  */
 export function coalescedZones(zones, now = Date.now()) {
+  // Every range of every zone is a part, kept in numbers rather than objects:
+  // a film's map is ten thousand of them (r26, torrent-tv/meta#178).
+  let count = 0;
+  for (const zone of zones) count += Array.isArray(zone.ranges) ? zone.ranges.length : 1;
+  const starts = new Float64Array(count);
+  const ends = new Float64Array(count);
+  const owners = new Int32Array(count);
+  const bands = zones.map(zone => bandOf(zone, now));
+  let at = 0;
+  zones.forEach((zone, index) => {
+    for (const [start, end] of rangesOfZone(zone)) {
+      starts[at] = start;
+      ends[at] = end;
+      owners[at] = index;
+      at += 1;
+    }
+  });
+  const best = bestOfRepeats(count, part => starts[part], part => ends[part], part => bands[owners[part]]);
   const kept = [];
+  /** Band key → the band and its parts, in the order the bands first appear. */
   const groups = new Map();
-  for (const zone of withoutRepeatedZones(zones, now)) {
+  for (let part = 0; part < count; part++) {
+    if (best.get(starts[part]).get(ends[part]) !== part) continue;
+    const zone = zones[owners[part]];
     if (zone.requestId !== undefined || zone.downloadOnly || (Number.isSafeInteger(zone.order) && zone.order > 0)) {
       kept.push(zone);
       continue;
     }
-    const urgency = levelOfMapZone(zone);
-    const priority = priorityOfMapZone(zone);
-    const key = `${urgency}:${priority}`;
+    const band = bands[owners[part]];
+    const key = band.urgency * 1000 + band.priority;
     let group = groups.get(key);
-    if (!group) groups.set(key, group = { urgency, priority, parts: [] });
-    group.parts.push({ start: zone.byteStart, end: zone.byteEnd, deadlineAt: deadlineOfMapZone(zone, now) });
+    if (!group) groups.set(key, group = { band, parts: [] });
+    group.parts.push(part);
   }
-  for (const { urgency, priority, parts } of groups.values()) {
-    parts.sort((left, right) => left.start - right.start);
-    let window = null;
+  for (const { band: { urgency, priority }, parts } of groups.values()) {
+    parts.sort((left, right) => starts[left] - starts[right]);
+    let windowStart = 0;
+    let windowEnd = -1;
+    let windowDeadline = Number.POSITIVE_INFINITY;
+    let open = false;
     const close = () => {
-      if (!window) return;
-      kept.push({ byteStart: window.start, byteEnd: window.end, priority, urgent: urgency === Urgency.NEAR,
+      if (!open) return;
+      kept.push({ byteStart: windowStart, byteEnd: windowEnd, priority, urgent: urgency === Urgency.NEAR,
         behind: urgency === Urgency.BEHIND, deferred: urgency === Urgency.TAIL,
-        ...(Number.isFinite(window.deadlineAt) ? { deadlineAt: window.deadlineAt } : {}) });
+        ...(Number.isFinite(windowDeadline) ? { deadlineAt: windowDeadline } : {}) });
     };
     for (const part of parts) {
-      if (window && part.start <= window.end + 1) {
-        window.end = Math.max(window.end, part.end);
-        window.deadlineAt = Math.min(window.deadlineAt, part.deadlineAt);
+      const deadlineAt = bands[owners[part]].deadlineAt;
+      if (open && starts[part] <= windowEnd + 1) {
+        windowEnd = Math.max(windowEnd, ends[part]);
+        windowDeadline = Math.min(windowDeadline, deadlineAt);
       } else {
         close();
-        window = { ...part };
+        open = true;
+        windowStart = starts[part];
+        windowEnd = ends[part];
+        windowDeadline = deadlineAt;
       }
     }
     close();
   }
   return kept;
 }
+
+/** The band a zone's bytes are fetched in. */
+function bandOf(zone, now) {
+  return { urgency: levelOfMapZone(zone), priority: priorityOfMapZone(zone), deadlineAt: deadlineOfMapZone(zone, now) };
+}
+
+/**
+ * Which of the parts naming each exact byte range is the best: by band
+ * ({@link compareBands}), the first of equals. Kept by the range's two numbers
+ * rather than a string made of them: making a key of each of a film's ten
+ * thousand ranges was most of what packing its map cost (r26,
+ * torrent-tv/meta#178).
+ *
+ * @returns {Map<number, Map<number, number>>} start → end → the best part.
+ */
+function bestOfRepeats(count, startOf, endOf, bandOfPart) {
+  const best = new Map();
+  for (let part = 0; part < count; part++) {
+    const start = startOf(part);
+    const end = endOf(part);
+    let byEnd = best.get(start);
+    if (!byEnd) best.set(start, byEnd = new Map());
+    const kept = byEnd.get(end);
+    if (kept === undefined || compareBands(bandOfPart(part), bandOfPart(kept)) < 0) byEnd.set(end, part);
+  }
+  return best;
+}

package/services/viewer/DownloadMaps.js

@@ -20,8 +20,6 @@ export class DownloadMaps {
   #clock;
   /** What the torrent is told of the playback zones: as many windows as say the same thing. */
   #pack;
-  /** What a list of zones says each segment reads, by the list. @type {WeakMap<object[], Map>} */
-  #inputsOf = new WeakMap();
 
   constructor({ publish, resolvePlayback = async map => map.zones, log = null, now = Date.now,
     clock = () => performance.timeOrigin + performance.now(), pack = zones => zones }) {
@@ -152,19 +150,6 @@ export class DownloadMaps {
     return this.#files.get(`${sourceKey}:${fileIndex}`)?.sourceZones.some(zone => zone.from === from && zone.to === to) ?? false;
   }
 
-  /** Exact source ranges already resolved by the download map, without new reads. */
-  inputsForOutput(sourceKey, outputKey, index) {
-    const sources = [];
-    for (const file of this.#files.values()) {
-      if (file.sourceKey !== sourceKey) continue;
-      const ranges = file.inputs.get(outputKey)?.get(index);
-      if (!ranges?.length) continue;
-      sources.push({ sourceId: `${sourceKey}:${file.fileIndex}`,
-        ranges });
-    }
-    return sources.length ? sources : null;
-  }
-
   metadata({ sourceKey, fileIndex, statement, result, priority = 100, urgent = true, deadlineAt = 0, interval, leadSeconds = 0, scope = "playback", order = 0 }) {
     if (!Number.isFinite(leadSeconds) || leadSeconds < 0) throw new TypeError("Metadata preparation time must be finite and nonnegative.");
     const file = this.#file(sourceKey, fileIndex);
@@ -255,7 +240,7 @@ export class DownloadMaps {
     const key = `${sourceKey}:${fileIndex}`;
     let file = this.#files.get(key);
     if (!file) {
-      file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], inputs: new Map(), nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve(), dirty: true };
+      file = { sourceKey, fileIndex, durationSeconds: 0, zones: [], nativeZones: [], sourceZones: [], revision: 0, epoch: ++this.#nextEpoch, metadata: new Map(), pending: Promise.resolve(), dirty: true };
       this.#files.set(key, file);
     }
     return file;
@@ -265,24 +250,6 @@ export class DownloadMaps {
     file.zones = zones;
     // What `#emit` reads has changed, whether or not it is published now.
     file.dirty = true;
-    // A pass that resolves what the pass before resolved hands back the same
-    // zones (`resolvedPasses` in server.js), and what they say of each segment
-    // is the same too.
-    const known = this.#inputsOf.get(zones);
-    if (known) {
-      file.inputs = known;
-      return;
-    }
-    file.inputs = new Map();
-    this.#inputsOf.set(zones, file.inputs);
-    for (const zone of zones) {
-      if (typeof zone.outputKey !== "string" || !Number.isSafeInteger(zone.index)) continue;
-      let output = file.inputs.get(zone.outputKey);
-      if (!output) file.inputs.set(zone.outputKey, output = new Map());
-      let ranges = output.get(zone.index);
-      if (!ranges) output.set(zone.index, ranges = []);
-      ranges.push({ start: zone.byteStart, end: zone.byteEnd + 1 });
-    }
   }
 
   #emit(file) {

package/services/viewer/segment-input-table.js

@@ -0,0 +1,53 @@
+/**
+ * @file Which source bytes each segment of an output reads, as the download
+ * map's pass worked them out (torrent-tv/meta#178).
+ *
+ * A fact of the file and the output, kept apart from what the torrent is told:
+ * the far part of a film is stated to the torrent as windows of a band
+ * (`download-bands.js`), and the forecast still has to find every segment's own
+ * bytes (`ViewerRequests` `#segmentsOf`). It used to read them off the zones
+ * sent, which named every segment.
+ */
+
+export class SegmentInputTable {
+  /** `sourceKey` → output → segment → file → its ranges. @type {Map<string, Map<string, Map<number, Map<number, { joined: Array<[number, number]>, ranges: Array<{ start: number, end: number }> }>>>>} */
+  #bySource = new Map();
+
+  /**
+   * @param {string} sourceKey
+   * @param {number} fileIndex
+   * @param {string} outputKey
+   * @param {number} index
+   * @param {Array<[number, number]>} joined - Inclusive ranges.
+   */
+  set(sourceKey, fileIndex, outputKey, index, joined) {
+    let outputs = this.#bySource.get(sourceKey);
+    if (!outputs) this.#bySource.set(sourceKey, outputs = new Map());
+    let segments = outputs.get(outputKey);
+    if (!segments) outputs.set(outputKey, segments = new Map());
+    let files = segments.get(index);
+    if (!files) segments.set(index, files = new Map());
+    // The same reading keeps its list: whoever compares inputs compares them by it.
+    if (files.get(fileIndex)?.joined === joined) return;
+    files.set(fileIndex, { joined, ranges: joined.map(([start, end]) => ({ start, end: end + 1 })) });
+  }
+
+  /**
+   * The bytes one segment reads, by file, or null where none are known.
+   *
+   * @param {string} sourceKey
+   * @param {string} outputKey
+   * @param {number} index
+   * @returns {Array<{ sourceId: string, ranges: Array<{ start: number, end: number }> }> | null}
+   */
+  inputsFor(sourceKey, outputKey, index) {
+    const files = this.#bySource.get(sourceKey)?.get(outputKey)?.get(index);
+    if (!files?.size) return null;
+    return [...files].map(([fileIndex, { ranges }]) => ({ sourceId: `${sourceKey}:${fileIndex}`, ranges }));
+  }
+
+  /** @param {string} sourceKey */
+  forget(sourceKey) {
+    this.#bySource.delete(sourceKey);
+  }
+}

package/services/viewer/target-input-expansion.js

@@ -102,8 +102,11 @@ function merged(ranges) {
  * The map's byte zones with each output's next run input raised to its first
  * segment's place.
  *
- * @template {{ outputKey: string, index: number, priority: number, urgent?: boolean, behind?: boolean, deferred?: boolean, byteStart: number, byteEnd: number }} Zone
- * @param {Zone[]} zones - One or more per segment, as the map's pass made them.
+ * @template {{ outputKey: string, index: number, priority: number, urgent?: boolean, behind?: boolean, deferred?: boolean,
+ *   ranges?: Array<[number, number]>, byteStart?: number, byteEnd?: number }} Zone
+ * @param {Zone[]} zones - One or more per segment, as the map's pass made
+ *   them: a segment's zone names all its ranges (`ranges`), any other one
+ *   range (`byteStart`-`byteEnd`).
  * @param {(outputKey: string) => number | null} targetBytesOf - What the
  *   output's next run would be given to copy; nothing is raised without it.
  * @returns {Zone[]} In their order; raised zones are copies.
@@ -138,7 +141,7 @@ export function expandToTargetInputs(zones, targetBytesOf) {
     if (zone.index === run.first.index) {
       let ranges = held.get(zone.outputKey);
       if (!ranges) held.set(zone.outputKey, ranges = []);
-      ranges.push([zone.byteStart, zone.byteEnd]);
+      ranges.push(...rangesOf(zone));
     } else if (zone.index > run.first.index) {
       const list = run.after.get(zone.index);
       if (list) list.push(zone);
@@ -153,7 +156,7 @@ export function expandToTargetInputs(zones, targetBytesOf) {
     // or not yet worked out, ends it.
     for (let index = run.first.index + 1; run.after.has(index); index++) {
       const list = run.after.get(index);
-      const ranges = list.map(zone => [zone.byteStart, zone.byteEnd]);
+      const ranges = list.flatMap(rangesOf);
       if (holding.total + holding.adds(ranges) > run.target) break;
       holding.add(ranges);
       for (const zone of list) {
@@ -164,3 +167,8 @@ export function expandToTargetInputs(zones, targetBytesOf) {
   }
   return raised.size === 0 ? zones : zones.map(zone => raised.get(zone) ?? zone);
 }
+
+/** The ranges a zone names, inclusive. */
+function rangesOf(zone) {
+  return Array.isArray(zone.ranges) ? zone.ranges : [[zone.byteStart, zone.byteEnd]];
+}