@torrent-tv/proxy 2.102.0 → 2.102.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.102.0",
+  "version": "2.102.2",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/server.js

@@ -12,6 +12,7 @@ import fastifyHelmet from "@fastify/helmet";
 import getPort from "get-port";
 import { createRequire } from "node:module";
 import { performance } from "node:perf_hooks";
+import { SourceWaitTrace } from "./utils/wait-trace.js";
 import { handleHealthGet } from "./routes/health/get.js";
 import { handleMediaGet } from "./routes/media/get.js";
 import { ProbeRequests } from "./services/media/ProbeRequests.js";
@@ -157,6 +158,8 @@ export async function startProxyServer({
     bodyLimit: 256 * 1024 * 1024 // 256 MB
   });
   const forwardedSessionResponseStarts = new WeakMap();
+  const waitTrace = new SourceWaitTrace({ emit: value => logger.info(`[wait-trace] ${JSON.stringify(value)}`) });
+  waitTrace.install(app);
   const logForwardedSessionResponseStage = (stage, req) => {
     const startedAt = forwardedSessionResponseStarts.get(req);
     const requestId = req.headers["x-ttv-forwarded-request-id"];
@@ -175,7 +178,7 @@ export async function startProxyServer({
   await app.register(fastifyCors, {
     origin: true,
     methods: ["GET", "POST", "OPTIONS"],
-    allowedHeaders: ["Content-Type", "Range"]
+    allowedHeaders: ["Content-Type", "Range", "X-Ttv-Wait-Trace", "X-Ttv-Wait-Attempt", "X-Ttv-Wait-Operation"]
   });
 
   // Allow browser requests from an HTTPS page to this private-network proxy
@@ -220,6 +223,7 @@ export async function startProxyServer({
   // 10 MB file took 12-23 s against 125 ms to hand it to the channel. The
   // adapter keeps TorrentPool's interface, so nothing downstream changed.
   const torrentPool = new WorkerTorrentPool({
+    onObserved: fields => waitTrace.source(fields.sourceKey, "torrent-ready", fields),
     memoryBytes,
     stateDir,
     heapSnapshots,
@@ -250,6 +254,8 @@ export async function startProxyServer({
     // only once the worker thread is up and a piece has verified, which is long
     // after the declarations below have run.
     onPiecesArrived: ({ sourceKey, fileIndexes, arrived }) => {
+      waitTrace.source(sourceKey, "held-bytes-changed", { fileIndexes, arrived: arrived?.slice?.(0, 32),
+        arrivedCount: arrived?.length });
       for (const fileIndex of fileIndexes) {
         mediaReads.bytesChanged(sourceKey, fileIndex);
         probeReads.bytesChanged(sourceKey, fileIndex);
@@ -871,6 +877,7 @@ export async function startProxyServer({
         } }),
       label: String(file.name ?? ""),
       fileOffset: file.offset,
+      onObserved: fields => waitTrace.source(sourceKey, "container-read", { fileIndex, requestId, ...fields }),
       onTracks: (tracks) => contentsOf(torrent).noteVideo(fileIndex, tracks.some((track) => track.type === "video")),
       onReadStart: () => ({ storage: mediaReads.revision(sourceKey, fileIndex), memory: mediaReads.memoryRevision(), demand: downloadMaps.epoch(sourceKey, fileIndex) }),
       onReadResult: async (statement, result, revision) => {
@@ -918,6 +925,7 @@ export async function startProxyServer({
   });
   let sourcePreparation;
   const outputParts = wireOutputs({
+    observeWait: (sourceKey, event, fields) => waitTrace.source(sourceKey, event, fields),
     pinnedPreset,
     indexMemory,
     serviceShare,

package/services/encode/EncodeRuns.js

@@ -1106,6 +1106,7 @@ export class EncodeRuns {
     // proven, and it is cleared up when the run ends.
     // Told when this run closes its first piece; see the next stretch below.
     let atFirstPiece = null;
+    const observedStart = performance.now();
     const run = new EncodeRun({
       address: session.outputKey ?? session.id,
       encoder: this.#host.videoEncoder,
@@ -1196,6 +1197,10 @@ export class EncodeRuns {
           { code: "ERR_INCOMPLETE_MEDIA" });
         const published = this.#host.segmentFiles.publish(address, name, session.segmentFormat, { mediaRanges, bytes });
         if (!published) throw new Error(`Closed piece #${index} could not be stored.`);
+        if (index === safeIndex) this.#host.observeWait?.(session.file.sourceKey, "first-piece-published", {
+          output: address, segment: index, startedAt: observedStart, endedAt: performance.now(),
+          encoderWrittenAfterMs: Number.isFinite(writtenAt) ? writtenAt - run.startedAt : null,
+          namedAfterMs: Number.isFinite(namedAt) ? namedAt - run.startedAt : null });
         this.#leadingByOutput.get(address)?.delete(index);
         // WHERE A LATE PIECE'S TIME WENT: the encoder writing it, this process
         // getting to its name, or its publication waiting for the file after

package/services/encode/LocalObservations.js

@@ -305,6 +305,16 @@ export class LocalObservations {
    * @returns {number | null}
    */
   firstFrameSeconds(features) {
+    return this.firstFrameEstimate(features)?.seconds ?? null;
+  }
+
+  /**
+   * Return the current stage-median forecast and the measurements that formed it.
+   *
+   * @param {object} features
+   * @returns {{ seconds: number, stagesMs: object, observations: number } | null}
+   */
+  firstFrameEstimate(features) {
     if (!features || typeof features !== "object") return null;
     const stages = ["transportMs", "planMs", "prepareMs", "prebufferMs", "presentationMs"];
     const candidates = [];
@@ -319,7 +329,11 @@ export class LocalObservations {
     const prices = stages.map(stage => median(candidates.map(reading => reading[stage])
       .filter(Number.isFinite)));
     if (prices.some(value => value === null)) return null;
-    return prices.reduce((sum, value) => sum + value, 0) / 1000;
+    return {
+      seconds: prices.reduce((sum, value) => sum + value, 0) / 1000,
+      stagesMs: Object.fromEntries(stages.map((stage, index) => [stage, prices[index]])),
+      observations: candidates.length
+    };
   }
 
   #firstFrameKey() {

package/services/media/ContainerOrchestrator.js

@@ -156,13 +156,17 @@ export class ContainerOrchestrator {
     // table of the whole film it does not use (r38, torrent-tv/meta#178).
     const aside = this.#answersWithoutReading(key, statement);
     const previous = aside ? Promise.resolve() : this.#reads.get(key) ?? Promise.resolve();
-    const askedAt = Date.now();
+    const askedAt = performance.now();
     const reading = previous.catch(() => undefined).then(async () => {
       if (!isCurrent()) return { result: { kind: "terminal", reason: "request-obsolete", requestId: params.requestId } };
-      const startedAt = Date.now();
+      const startedAt = performance.now();
       const revision = params.onReadStart?.(statement);
       const result = await this.#inspectRead({ ...params, isCurrent }, statement);
-      params.onTimed?.(statement, startedAt - askedAt, Date.now() - startedAt);
+      const endedAt = performance.now();
+      params.onTimed?.(statement, startedAt - askedAt, endedAt - startedAt);
+      if (statement !== "source-ranges" || endedAt - askedAt >= 10) params.onObserved?.({ statement, askedAt,
+        startedAt, endedAt, queuedMs: startedAt - askedAt, readMs: endedAt - startedAt, cached: aside,
+        result: result.kind, reason: result.reason, ranges: result.ranges });
       return { result, revision };
     });
     if (!aside) this.#reads.set(key, reading);

package/services/server/ViewerRequests.js

@@ -125,11 +125,13 @@ export class ViewerRequests {
       const changed = this.#recordEarlyWaitSnapshot(context, context.features);
       if (changed) context.revision += 1;
       this.#pruneEarlyWaits();
-      const totalSeconds = Number.isFinite(context.activeTotalSeconds)
-        ? context.activeTotalSeconds
-        : this.#host.firstFrameSecondsOf?.(context.features) ?? null;
-      const seconds = earlyWaitRemainingSeconds(totalSeconds, waitElapsedSince(context, waitElapsedMs));
+      const elapsedMs = waitElapsedSince(context, waitElapsedMs);
+      const historical = Number.isFinite(context.activeTotalSeconds) ? null : this.#historicalFirstFrame(context.features);
+      const totalSeconds = Number.isFinite(context.activeTotalSeconds) ? context.activeTotalSeconds : historical?.seconds ?? null;
+      const seconds = earlyWaitRemainingSeconds(totalSeconds, elapsedMs);
       if (!Number.isFinite(seconds) || seconds < 0) return null;
+      this.#logEarlyForecast(context, { elapsedMs, seconds, totalSeconds, historical,
+        firstRequiredSegmentsSeconds: context.activeFirstRequiredSegmentsSeconds });
       return { version: 1, waitId, revision: context.revision, remainingSeconds: seconds, ageMs: 0 };
     }
     const sameWait = current?.waitId === waitId;
@@ -139,11 +141,13 @@ export class ViewerRequests {
     const changed = this.#recordEarlyWaitSnapshot(context, context.features);
     if (sameWait && changed) context.revision += 1;
     this.#pruneEarlyWaits();
-    const totalSeconds = Number.isFinite(context.activeTotalSeconds)
-      ? context.activeTotalSeconds
-      : this.#host.firstFrameSecondsOf?.(context.features) ?? null;
-    const seconds = earlyWaitRemainingSeconds(totalSeconds, waitElapsedSince(context, waitElapsedMs));
+    const elapsedMs = waitElapsedSince(context, waitElapsedMs);
+    const historical = Number.isFinite(context.activeTotalSeconds) ? null : this.#historicalFirstFrame(context.features);
+    const totalSeconds = Number.isFinite(context.activeTotalSeconds) ? context.activeTotalSeconds : historical?.seconds ?? null;
+    const seconds = earlyWaitRemainingSeconds(totalSeconds, elapsedMs);
     if (!Number.isFinite(seconds) || seconds < 0) return null;
+    this.#logEarlyForecast(context, { elapsedMs, seconds, totalSeconds, historical,
+      firstRequiredSegmentsSeconds: context.activeFirstRequiredSegmentsSeconds });
     return { version: 1, waitId, revision: context.revision, remainingSeconds: seconds, ageMs: 0 };
   }
 
@@ -192,20 +196,44 @@ export class ViewerRequests {
       ? elapsedMs / 1000 + measuredWorkSeconds
       : context.activeTotalSeconds;
     const activeEstimateChanged = hasActiveEstimate && activeTotalSeconds !== context.activeTotalSeconds;
-    if (hasActiveEstimate) context.activeTotalSeconds = activeTotalSeconds;
+    if (hasActiveEstimate) {
+      context.activeTotalSeconds = activeTotalSeconds;
+      context.activeFirstRequiredSegmentsSeconds = measuredWorkSeconds;
+    }
     const changed = featuresChanged || activeEstimateChanged;
     if (sameWait && changed) context.revision += 1;
     this.#pruneEarlyWaits();
     const progressHasComponentForecast = Object.hasOwn(processing, "firstRequiredSegmentsSeconds");
     let totalSeconds = activeTotalSeconds;
+    let historical = null;
     if (!Number.isFinite(totalSeconds) && !progressHasComponentForecast) {
-      totalSeconds = this.#host.firstFrameSecondsOf?.(context.features) ?? null;
+      historical = this.#historicalFirstFrame(context.features);
+      totalSeconds = historical?.seconds ?? null;
     }
     const seconds = earlyWaitRemainingSeconds(totalSeconds, elapsedMs);
     if (!Number.isFinite(seconds) || seconds < 0) return null;
+    this.#logEarlyForecast(context, { elapsedMs, seconds, totalSeconds, historical,
+      firstRequiredSegmentsSeconds: hasActiveEstimate ? measuredWorkSeconds : context.activeFirstRequiredSegmentsSeconds });
     return { version: 1, waitId, revision: context.revision, remainingSeconds: seconds, ageMs: 0 };
   }
 
+  #historicalFirstFrame(features) {
+    const estimate = this.#host.firstFrameSecondsOf?.(features) ?? null;
+    return Number.isFinite(estimate) ? { seconds: estimate, stagesMs: null, observations: null } : estimate;
+  }
+
+  #logEarlyForecast(context, { elapsedMs, seconds, totalSeconds, historical, firstRequiredSegmentsSeconds }) {
+    if (context.loggedForecastRevision === context.revision) return;
+    const basis = Number.isFinite(firstRequiredSegmentsSeconds) ? "first-required" : "historical-stages";
+    const estimate = basis === "first-required"
+      ? { firstRequiredSegmentsSeconds }
+      : { stagesMs: historical?.stagesMs ?? null, observations: historical?.observations ?? null };
+    logger.info(`early wait forecast ${JSON.stringify({ waitId: context.waitId, consumer: context.consumerId,
+      generation: context.generation, features: context.features, elapsedMs, revision: context.revision,
+      predicted: { remainingSeconds: seconds, totalSeconds, basis, ...estimate } })}`);
+    context.loggedForecastRevision = context.revision;
+  }
+
   /**
    * Learn from the browser's presented-frame measurement for the current
    * viewing. The saved source and link readings are the ones available at

package/services/server/wire-outputs.js

@@ -177,6 +177,7 @@ function buildHttpBaseUrl(host, port) {
  * @returns {object} The components, and the settings they read.
  */
 export function wireOutputs({
+    observeWait = null,
     enabled,
     ffmpegBin,
     localBindHost,
@@ -363,6 +364,7 @@ export function wireOutputs({
     get segmentDurationSec() { return parts.segmentDurationSec; },
     readsThroughCache: () => parts.inputCacheAvailable === true,
     segmentFiles,
+    observeWait,
     get videoEncoder() { return parts.encoders.current; },
     get encoders() { return parts.encoders; },
     invalidateWaits: (output) => parts.serving.invalidateWaits(output),
@@ -649,7 +651,7 @@ export function wireOutputs({
       const predicted = Number(qualityStateOf(output).predictedSpeedWhenOffered);
       return Number.isFinite(predicted) && predicted > 0 ? predicted : null;
     },
-    firstFrameSecondsOf: (features) => parts.localObservations?.firstFrameSeconds(features) ?? null,
+    firstFrameSecondsOf: (features) => parts.localObservations?.firstFrameEstimate(features) ?? null,
     noteFirstFrame: (observation) => parts.localObservations?.noteFirstFrame(observation),
     audioDescriptionOf,
     openSecondsOf: (output) => runStartSecondsOf(output),

package/services/torrent/worker/pool-adapter.js

@@ -31,12 +31,14 @@ export class WorkerTorrentPool {
   /** Stand-ins by source key, so repeat calls return the same object. */
   #torrents = new Map();
   #pending = new Map();
+  #observed;
 
   /**
    * @param {{ memoryBytes?: number, stateDir?: string }} [options]
    * @param {object | null} [client] - The worker interface; tests supply a fake.
    */
   constructor(options = {}, client = null) {
+    this.#observed = options.onObserved;
     this.#client = client ?? new TorrentWorkerClient({ ...options, onSourceForgotten: event => {
       this.forget(event.sourceKey);
       options.onSourceForgotten?.(event);
@@ -51,13 +53,16 @@ export class WorkerTorrentPool {
    * @returns {Promise<object>}
    */
   async getTorrent(sourceType, source) {
+    const startedAt = performance.now();
     const sourceKey = await deriveSourceKey(sourceType, source);
+    const observed = outcome => this.#observed?.({ sourceKey, startedAt, endedAt: performance.now(), outcome });
     const existing = this.#torrents.get(sourceKey);
     if (existing) {
+      observed("cached");
       return existing;
     }
     const joined = this.#pending.get(sourceKey);
-    if (joined) return joined;
+    if (joined) return joined.finally(() => observed("joined"));
     const pending = this.#client.getTorrent({ sourceKey, sourceType, source }).then(torrent => {
       if (this.#pending.get(sourceKey) !== pending) {
         const error = new Error("Source was forgotten while its metadata was being read.");
@@ -66,7 +71,10 @@ export class WorkerTorrentPool {
       }
       this.#torrents.set(sourceKey, torrent);
       return torrent;
-    }).finally(() => { if (this.#pending.get(sourceKey) === pending) this.#pending.delete(sourceKey); });
+    }).finally(() => {
+      observed(this.#torrents.has(sourceKey) ? "ready" : "error");
+      if (this.#pending.get(sourceKey) === pending) this.#pending.delete(sourceKey);
+    });
     this.#pending.set(sourceKey, pending);
     return pending;
   }

package/utils/wait-trace.js

@@ -0,0 +1,68 @@
+/** Bounded, opt-in startup observations. No torrent or application imports. */
+export class SourceWaitTrace {
+  constructor({ emit, now = () => performance.now() }) {
+    Object.assign(this, { emit, now, sources: new Map(), requests: new WeakMap() });
+  }
+
+  write(context, event, fields = {}) {
+    this.emit({ version: 1, clock: "proxy-main", wait: context.wait, attempt: context.attempt,
+      operation: context.operation, event, at: this.now(), ...fields });
+  }
+
+  remember(sourceKey, context) {
+    if (typeof sourceKey !== "string" || !sourceKey) return;
+    for (const [key, value] of this.sources) if (this.now() - value.seen > 60_000) this.sources.delete(key);
+    if (this.sources.size >= 64 && !this.sources.has(sourceKey)) this.sources.delete(this.sources.keys().next().value);
+    this.sources.set(sourceKey, { ...context, seen: this.now() });
+  }
+
+  source(sourceKey, event, fields = {}) {
+    const context = this.sources.get(sourceKey);
+    if (!context || this.now() - context.seen > 60_000) return;
+    this.write({ ...context, operation: null }, event, { sourceKey, ...fields });
+  }
+
+  install(app) {
+    app.addHook("onRequest", async (req, reply) => {
+      const wait = req.headers["x-ttv-wait-trace"];
+      const operation = req.headers["x-ttv-wait-operation"];
+      const attempt = Number(req.headers["x-ttv-wait-attempt"]);
+      if (typeof wait !== "string" || !/^[a-zA-Z0-9.-]{1,80}$/.test(wait) ||
+        typeof operation !== "string" || !/^b[0-9]{1,12}$/.test(operation) ||
+        !Number.isSafeInteger(attempt) || attempt < 0) return;
+      const context = { wait, attempt, operation, startedAt: this.now() };
+      this.requests.set(req, context);
+      this.write(context, "http-start", { method: req.method, path: req.url,
+        requestId: req.headers["x-ttv-forwarded-request-id"] ?? req.id });
+      let finished = false;
+      const end = outcome => {
+        if (finished) return;
+        finished = true;
+        this.write(context, "http-end", { outcome, status: reply.statusCode, duration: this.now() - context.startedAt });
+      };
+      reply.raw.once("finish", () => end("finished"));
+      reply.raw.once("close", () => end(reply.raw.writableFinished ? "finished" : "closed"));
+    });
+    app.addHook("preHandler", async req => {
+      const context = this.requests.get(req);
+      if (!context) return;
+      const match = /^\/api\/sources\/([^/?]+)/.exec(req.url);
+      const sourceKey = req.body?.sourceKey ?? req.query?.sourceKey ?? (match ? decodeURIComponent(match[1]) : null);
+      this.remember(sourceKey, context);
+      this.write(context, "http-handler", { sourceKey });
+    });
+    app.addHook("preSerialization", async (req, _reply, payload) => {
+      const context = this.requests.get(req);
+      if (context) {
+        this.remember(payload?.sourceKey, context);
+        this.write(context, "http-serialize");
+      }
+      return payload;
+    });
+    app.addHook("onSend", async (req, _reply, payload) => {
+      const context = this.requests.get(req);
+      if (context) this.write(context, "http-send");
+      return payload;
+    });
+  }
+}