@torrent-tv/proxy 2.98.8 → 2.98.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.98.8",
+  "version": "2.98.10",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/services/encode/EncodeRuns.js

@@ -10,6 +10,8 @@
 import path from "node:path";
 import { spawn } from "node:child_process";
 import { contentionPenalty } from "./contention.js";
+import { LOOKAHEAD_PAUSE_SECONDS } from "./CushionReport.js";
+import { SpeedReadings } from "./SpeedReadings.js";
 import { ENCODE_RUN_STATE, liveRunsOf, runStateOf, wireState } from "./encode-run-state.js";
 import { ENCODE_EXIT } from "./encode-exit.js";
 import { EncodeRun, endOfRun } from "./EncodeRun.js";
@@ -171,6 +173,8 @@ export class EncodeRuns {
    * @type {WeakMap<object, { cpuSeconds: number, filmSeconds: number, reading: boolean, beside: number }>}
    */
   #cpuOfRun = new WeakMap();
+  /** Every speed reading of each output's runs, for the start forecast (`SpeedReadings.js`). */
+  #speedReadings = new SpeedReadings(LOOKAHEAD_PAUSE_SECONDS);
 
   #lastInputToken = 0;
   #inputFailures = new InputFailures();
@@ -492,6 +496,17 @@ export class EncodeRuns {
     }
   }
 
+  /**
+   * The speed readings of this output's runs within the forecast's horizon,
+   * in the order they were taken.
+   *
+   * @param {object} output
+   * @returns {Array<{ speed: number, at: number }>}
+   */
+  speedReadingsOf(output) {
+    return this.#speedReadings.windowOf(output);
+  }
+
   /**
    * The media intervals of a stored piece of this output, or undefined.
    *
@@ -1179,7 +1194,10 @@ export class EncodeRuns {
       // group before it in hand — the leading pictures the run that starts
       // there cannot make.
       onTail: (name) => this.#keepLeadingPictures(session, admittedInput, name),
-      onSpeedMeasured: () => this.#host.noteRunSpeedMeasured?.(session, run),
+      onSpeedMeasured: (reading) => {
+        this.#speedReadings.note(session, reading);
+        this.#host.noteRunSpeedMeasured?.(session, run);
+      },
       onEnded: (ended) => {
         this.#runsByInputToken.delete(inputToken);
         if (admittedInput.original) {

package/services/encode/SpeedReadings.js

@@ -0,0 +1,49 @@
+/**
+ * @file The speed readings of each output's runs, in the order the runs
+ * measured them, over the start forecast's horizon (torrent-tv/meta#178).
+ *
+ * A run announces a reading at each closed piece; the forecast summarizes the
+ * whole window on each poll. Two runs of one output can announce out of order —
+ * each reading carries the time of its own run's last progress report — so the
+ * window is kept in the order the readings were TAKEN, and two taken in the
+ * same millisecond are two readings. The forecast used to take only the newest
+ * reading of a run alive when it asked: on a cold start on Home Assistant the
+ * runs were three pieces long and ended between its polls, and of the readings
+ * 0.78, 1.80, 2.25, 2.84 and 2.69x it got 0.78 and 2.69x, and held the picture
+ * on 0.78x for 13 s (r37).
+ */
+
+export class SpeedReadings {
+  /** @type {WeakMap<object, Array<{ speed: number, at: number }>>} */
+  #byOutput = new WeakMap();
+  #keepMs;
+
+  /** @param {number} keepSeconds - How far back a reading is kept: the forecast's horizon. */
+  constructor(keepSeconds) {
+    this.#keepMs = Math.max(0, keepSeconds) * 1000;
+  }
+
+  /**
+   * @param {object} output
+   * @param {{ speed: number, at: number } | null} reading
+   */
+  note(output, reading) {
+    if (!output || !(reading?.speed > 0) || !Number.isFinite(reading.at)) return;
+    let readings = this.#byOutput.get(output);
+    if (!readings) this.#byOutput.set(output, readings = []);
+    let place = readings.length;
+    while (place > 0 && readings[place - 1].at > reading.at) place -= 1;
+    readings.splice(place, 0, { speed: reading.speed, at: reading.at });
+    const cutoff = readings.at(-1).at - this.#keepMs;
+    // As the forecast's own trend keeps them: the newest one before the horizon stays.
+    while (readings.length > 1 && readings[1].at <= cutoff) readings.shift();
+  }
+
+  /**
+   * @param {object} output
+   * @returns {Array<{ speed: number, at: number }>} The window, in the order the readings were taken.
+   */
+  windowOf(output) {
+    return [...(this.#byOutput.get(output) ?? [])];
+  }
+}

package/services/server/ViewerRequests.js

@@ -60,6 +60,27 @@ export class ViewerRequests {
   #host;
   /** @type {WeakMap<object, Map<string, object>>} */
   #rateStateBySession = new WeakMap();
+  /**
+   * What the start forecasts cost, said once a minute: how many there were,
+   * their time on the main thread, and for each track whether the times of its
+   * pieces were kept from the last poll or why they were worked out again. In
+   * r37 the forecast took 19.2 s of Home Assistant's main thread in 15 minutes
+   * and the times were mostly worked out again (torrent-tv/meta#178).
+   */
+  #forecastCosts = { since: Date.now(), polls: 0, ms: 0, maxMs: 0, times: new Map() };
+
+  #noteForecast(ms, reasons) {
+    const costs = this.#forecastCosts;
+    costs.polls += 1; costs.ms += ms; costs.maxMs = Math.max(costs.maxMs, ms);
+    for (const reason of reasons) costs.times.set(reason, (costs.times.get(reason) ?? 0) + 1);
+    const now = Date.now();
+    if (now - costs.since < 60_000) return;
+    logger.info(`start forecasts in ${((now - costs.since) / 1000).toFixed(1)}s: ${costs.polls}, ` +
+      `${Math.round(costs.ms)}ms in all, ${(costs.ms / costs.polls).toFixed(1)}ms each, longest ${Math.round(costs.maxMs)}ms; ` +
+      `times of a track's pieces: ${[...costs.times].map(([reason, count]) => `${reason} ${count}`).join(", ")}`);
+    Object.assign(costs, { since: now, polls: 0, ms: 0, maxMs: 0, times: new Map() });
+  }
+
   /**
    * The pieces of each output as the start forecast read them last (see
    * `#segmentsOf`), and what the forecast keeps of their time geometry while
@@ -70,7 +91,7 @@ export class ViewerRequests {
   #forecastPieces = new WeakMap();
 
   /**
-   * @param {object} host - `opening`, `activeOutputFor`, `viewerSecondsOn`, `minimumBufferSecondsFor`, `getSourceStats`, `disposeSession`, `planEncodersSoon`, `waitUntilReady`, `encodeRuns`, `encodeSpeedReadingOf`, `outputTimes`, `outputs`, `lookaheadSeconds`, `quality`, `qualityOffer`, `renditions`, `segmentDurationSec`, `segmentStore`, `sourceFiles`, `startupWaitMs`, `viewers`, `acceptsGeneration`, `generationOfRequest`, `noteGivenOutput`, `holdForResponse`, `noteServingVerdict`, `servingVerdictOf`
+   * @param {object} host - `opening`, `activeOutputFor`, `viewerSecondsOn`, `minimumBufferSecondsFor`, `getSourceStats`, `disposeSession`, `planEncodersSoon`, `waitUntilReady`, `encodeRuns`, `encodeSpeedReadingsOf`, `outputTimes`, `outputs`, `lookaheadSeconds`, `quality`, `qualityOffer`, `renditions`, `segmentDurationSec`, `segmentStore`, `sourceFiles`, `startupWaitMs`, `viewers`, `acceptsGeneration`, `generationOfRequest`, `noteGivenOutput`, `holdForResponse`, `noteServingVerdict`, `servingVerdictOf`
    */
   constructor(host) {
     this.#host = host;
@@ -583,7 +604,7 @@ export class ViewerRequests {
     const tracks = [];
     for (const output of outputs) {
       const trackProgress = output === session ? progress : this.#host.encodeRuns.progressOf(output);
-      const trackRates = this.#trackRateReadings(measurement, output, now);
+      const trackRates = this.#trackRateReadings(output, now);
       const trackName = output === session ? "video" : "audio";
       // The page states the offset its player applied to this track, and the
       // init it applied it to is the session's. Until both are known, the
@@ -670,12 +691,6 @@ export class ViewerRequests {
       });
     }
 
-    const activeTrackIds = new Set(tracks.map(({ id }) => id));
-    for (const id of measurement.tracks.keys()) {
-      if (!activeTrackIds.has(id)) {
-        measurement.tracks.delete(id);
-      }
-    }
     const bufferedAheadSeconds = Number(viewerReading?.bufferedAheadSec);
     const positionSeconds = Number.isFinite(viewerReading?.positionSeconds)
       ? viewerReading.positionSeconds
@@ -717,7 +732,10 @@ export class ViewerRequests {
     // fetched whole by the page before it is chosen, so it never matches here.
     const subtitle = viewer?.subtitle?.sourceKey === session.file.sourceKey &&
       viewer.subtitle.fileIndex === session.file.fileIndex ? viewer.subtitle : null;
-    const forecast = withSubtitleReadiness(this.#host.playbackReadiness.predict(input), subtitle,
+    const startedAt = performance.now();
+    const predicted = this.#host.playbackReadiness.predict(input);
+    this.#noteForecast(performance.now() - startedAt, tracks.map(track => track.geometry?.reason ?? "none"));
+    const forecast = withSubtitleReadiness(predicted, subtitle,
       subtitle ? this.#host.subtitleReadyFor?.(viewer) !== false : true);
     forecast.operations = outputs.flatMap((output, index) => {
       const common = { speed: this.#host.playbackReadiness.forecastRate(tracks[index].readings, now, 0, "piece"),
@@ -856,23 +874,23 @@ export class ViewerRequests {
     const id = typeof consumerId === "string" ? consumerId : "";
     let state = byViewer.get(id);
     if (!state) {
-      state = { link: new this.#host.playbackReadiness.RateTrend(this.#host.lookaheadSeconds), tracks: new Map(), sources: new Map() };
+      state = { link: new this.#host.playbackReadiness.RateTrend(this.#host.lookaheadSeconds), sources: new Map() };
       byViewer.set(id, state);
     }
     return state;
   }
 
-  #trackRateReadings(state, output, now) {
-    const id = output.outputKey;
-    let trend = state.tracks.get(id);
-    if (!trend) {
-      trend = new this.#host.playbackReadiness.RateTrend(this.#host.lookaheadSeconds);
-      state.tracks.set(id, trend);
-    }
-    const reading = this.#host.encodeSpeedReadingOf?.(output, now) ?? null;
-    if (Number.isFinite(reading?.speed) && reading.speed > 0 && Number.isFinite(reading.at)) {
-      trend.add(reading.at, reading.speed);
-    } else if (!trend.snapshot()) {
+  /**
+   * The encoder's rate for the forecast: the output's whole window of readings,
+   * summarized again on each poll. A run that ended between two polls took its
+   * readings with it when the forecast asked for the newest one alone; and two
+   * runs of one output can report out of order, so a summary kept and added to
+   * would drop the later-told, earlier-taken one (torrent-tv/meta#178).
+   */
+  #trackRateReadings(output, now) {
+    const readings = (this.#host.encodeSpeedReadingsOf?.(output) ?? []).filter(reading => reading.at <= now);
+    const summary = readings.length > 0 ? this.#host.playbackReadiness.rateSummaryOf(readings) : null;
+    if (!summary) {
       // ONLY UNTIL A PIECE IS MEASURED does the prediction stand for the
       // encoder, and it is not kept among the readings. Added on every poll
       // between runs, it put the machine's share of that moment — 1 % at a
@@ -885,7 +903,7 @@ export class ViewerRequests {
         return seed.snapshot();
       }
     }
-    return trend.snapshot();
+    return summary;
   }
 
   /**

package/services/server/wire-outputs.js

@@ -71,7 +71,7 @@ import { ViewerRequests } from "./ViewerRequests.js";
 import { OutputLifecycle } from "./OutputLifecycle.js";
 import { SegmentServing } from "./SegmentServing.js";
 import { audioStartSecondsFor } from "../viewer/audio-start.js";
-import { forecastRate, predictPlaybackReadiness, RateTrend } from "../viewer/playback-readiness.js";
+import { forecastRate, rateSummaryOf, predictPlaybackReadiness, RateTrend } from "../viewer/playback-readiness.js";
 import { audioRenditionName } from "../media/audio-inventory.js";
 import { Renditions } from "../encode/Renditions.js";
 import { CushionReport, LOOKAHEAD_PAUSE_SECONDS } from "../encode/CushionReport.js";
@@ -578,12 +578,12 @@ export function wireOutputs({
     get encodeRuns() { return parts.encodeRuns; },
     // When this viewer's playback can start and run to the end: the viewer
     // component's forecast, which models the viewer's browser.
-    playbackReadiness: { predict: predictPlaybackReadiness, forecastRate, RateTrend },
+    playbackReadiness: { predict: predictPlaybackReadiness, forecastRate, rateSummaryOf, RateTrend },
     // Whether the subtitle a viewer chose has been read where they stand — the
     // source preparation's answer, set on these parts by the assembly.
     subtitleReadyFor: (viewer) => parts.subtitleReadyFor?.(viewer) ?? true,
     sourceInputsFor: (output, index) => parts.sourceInputsFor?.(output, index),
-    encodeSpeedReadingOf: (output) => parts.encodeCost.latestSpeedReadingOf(output),
+    encodeSpeedReadingsOf: (output) => parts.encodeRuns.speedReadingsOf(output),
     projectedEncodeSpeedOf: (output) => parts.encodeCost.projectedSpeedOf(output),
     // The speed the offer predicted for this output when it chose its mode: a
     // forecast with no measured piece yet stands on it rather than on the

package/services/viewer/playback-readiness.js

@@ -58,6 +58,22 @@ export function forecastRate(readings, now = Date.now(), horizonSeconds = 0, wei
   return horizon > 0 ? curve.workBy(horizon) / horizon : curve.rateAt(0);
 }
 
+/**
+ * The summary the forecast reads, of a window of readings taken in any order:
+ * ordered by when each was taken, readings taken in the same millisecond kept
+ * apart. Null for no reading.
+ *
+ * @param {Array<{ at: number, speed?: number, value?: number }>} readings
+ * @returns {object | null}
+ */
+export function rateSummaryOf(readings) {
+  const ordered = (Array.isArray(readings) ? readings : [])
+    .map(reading => ({ at: reading?.at, value: reading?.value ?? reading?.speed }))
+    .filter(reading => Number.isFinite(reading.at) && Number.isFinite(reading.value) && reading.value >= 0)
+    .sort((left, right) => left.at - right.at);
+  return ordered.length > 0 ? summarizeRate(ordered) : null;
+}
+
 /**
  * Measurements over the output's existing look-ahead horizon. The oldest
  * boundary sample is retained until another sample replaces it, so a repeated
@@ -648,8 +664,11 @@ function timesOf(segment) {
  * thread in r35's first 15 minutes on Home Assistant (torrent-tv/meta#178).
  * `track.geometry` is kept by whoever keeps the pieces; without it nothing is
  * kept, and the answer is the same either way: a placement is reused only for
- * the same pieces, the same timescale and the same clock offset, so no time is
- * rounded for the reuse.
+ * pieces with the same edges and read ranges, the same timescale and the same
+ * clock offset, so no time is rounded for the reuse. Only the times are kept: a
+ * piece whose torrent inputs were told again is a new object with the same
+ * times, and it is read with its inputs of this poll. In r37 most polls placed
+ * every piece again, the MPEG-TS output having no read ranges to keep them by.
  *
  * @param {{ segments?: object[], geometry?: object }} track
  * @returns {bigint} The least timescale holding every edge and every served tick of its pieces.
@@ -657,7 +676,9 @@ function timesOf(segment) {
 function timescaleOfPieces(track) {
   const segments = Array.isArray(track.segments) ? track.segments : [];
   const geometry = track.geometry;
-  if (geometry && samePieces(geometry.pieces, segments)) return geometry.timescale;
+  const changed = geometry ? piecesChanged(geometry.pieces, segments) : null;
+  if (geometry && changed === null) return geometry.timescale;
+  if (geometry) geometry.reason = changed;
   const distinct = new Set();
   for (const segment of segments) {
     if (segment.mediaRanges?.timescale) distinct.add(segment.mediaRanges.timescale);
@@ -667,7 +688,7 @@ function timescaleOfPieces(track) {
   }
   const timescale = commonTimescale([...distinct].map((value) => mediaTime(0n, value)));
   if (geometry) {
-    geometry.pieces = segments.map((segment) => ({ segment, index: segment.index, startSeconds: segment.startSeconds,
+    geometry.pieces = segments.map((segment) => ({ index: segment.index, startSeconds: segment.startSeconds,
       endSeconds: segment.endSeconds, mediaRanges: segment.mediaRanges }));
     geometry.timescale = timescale;
     geometry.placed = null;
@@ -677,8 +698,9 @@ function timescaleOfPieces(track) {
 
 /**
  * A track's pieces on the page's clock in `unit`: read pieces at their served
- * ranges, the others placed between them. Reused while the pieces are those
- * {@link timescaleOfPieces} last took and `unit` and `clock` are the same.
+ * ranges, the others placed between them. The times are reused while the
+ * pieces are those {@link timescaleOfPieces} last took and `unit` and `clock`
+ * are the same; everything else of a piece is this poll's.
  *
  * @param {{ segments?: object[], geometry?: object }} track
  * @param {import("./media-time.js").MediaTime} clock - In `unit`.
@@ -686,23 +708,34 @@ function timescaleOfPieces(track) {
  */
 function placedPieces(track, clock, unit) {
   const geometry = track.geometry;
-  if (geometry?.placed && geometry.placedUnit === unit && geometry.placedClock === clock.ticks) return geometry.placed;
-  const placed = predictedMediaSegments((Array.isArray(track.segments) ? track.segments : []).map((segment) => ({
-    ...segment, exact: exactTimesOf(segment), ranges: playerRanges(segment.mediaRanges, clock, unit)
-  })), unit);
-  if (geometry) Object.assign(geometry, { placed, placedUnit: unit, placedClock: clock.ticks });
-  return placed;
+  const segments = Array.isArray(track.segments) ? track.segments : [];
+  let times = geometry?.placed && geometry.placedUnit === unit && geometry.placedClock === clock.ticks ? geometry.placed : null;
+  if (geometry) geometry.reason = times ? "kept" : !geometry.placed ? geometry.reason ?? "first" : geometry.placedUnit !== unit ? "timescale" : "clock";
+  if (!times) {
+    times = predictedMediaSegments(segments.map((segment) => ({
+      ...segment, exact: exactTimesOf(segment), ranges: playerRanges(segment.mediaRanges, clock, unit)
+    })), unit).map(({ exact, ranges }) => ({ exact, ranges }));
+    if (geometry) Object.assign(geometry, { placed: times, placedUnit: unit, placedClock: clock.ticks });
+  }
+  return segments.map((segment, index) => ({ ...segment, exact: times[index].exact, ranges: times[index].ranges }));
 }
 
-/** Whether `segments` are the pieces `kept` recorded, field by field the forecast reads. */
-function samePieces(kept, segments) {
-  if (!Array.isArray(kept) || kept.length !== segments.length) return false;
+/**
+ * How `segments` differ from the pieces `kept` recorded, by the first field
+ * their times come from that differs, or null when they are the same pieces.
+ *
+ * @returns {"first" | "count" | "index" | "edges" | "read ranges" | null}
+ */
+function piecesChanged(kept, segments) {
+  if (!Array.isArray(kept)) return "first";
+  if (kept.length !== segments.length) return "count";
   for (let index = 0; index < segments.length; index++) {
     const segment = segments[index], was = kept[index];
-    if (segment !== was.segment || segment.index !== was.index || segment.startSeconds !== was.startSeconds ||
-      segment.endSeconds !== was.endSeconds || segment.mediaRanges !== was.mediaRanges) return false;
+    if (segment.index !== was.index) return "index";
+    if (segment.startSeconds !== was.startSeconds || segment.endSeconds !== was.endSeconds) return "edges";
+    if (segment.mediaRanges !== was.mediaRanges) return "read ranges";
   }
-  return true;
+  return null;
 }
 
 /**