@torrent-tv/proxy 2.102.2 → 2.102.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@torrent-tv/proxy",
-  "version": "2.102.2",
+  "version": "2.102.4",
   "description": "Torrent proxy client that exposes webseed-like HTTP stream endpoint.",
   "license": "GPL-3.0-or-later",
   "repository": {

package/routes/api/sources/post.js

@@ -23,7 +23,7 @@ function getPayload(body) {
   return {};
 }
 
-export async function handleApiSourcesPost(req, reply, { sourceRegistry, viewers }) {
+export async function handleApiSourcesPost(req, reply, { sourceRegistry, viewers, associateWaitTrace }) {
   const payload = getPayload(req.body);
   const sourceType = typeof payload.sourceType === "string" ? payload.sourceType : "";
   const source = typeof payload.source === "string" ? payload.source : "";
@@ -34,6 +34,7 @@ export async function handleApiSourcesPost(req, reply, { sourceRegistry, viewers
   const consumerId = typeof payload.consumerId === "string" ? payload.consumerId.trim() : "";
   const viewer = consumerId ? viewers?.present(consumerId) : null;
   const sourceKey = await sourceRegistry.upsert(sourceType, source);
+  associateWaitTrace?.(req, sourceKey);
   if (viewer && viewers.get(consumerId) === viewer) viewers.selectsSource(consumerId, sourceKey);
   return reply.send({ sourceKey, playbackMapVersion: 1 });
 }

package/routes/encode-input/get.js

@@ -2,7 +2,7 @@ import { parseRange } from "../../utils/parse-range.js";
 import { Readable } from "node:stream";
 
 /** FFmpeg reads only original byte ranges already owned by its live run. */
-export async function handleEncodeInputGet(req, reply, { inputOf, refused = () => {} }) {
+export async function handleEncodeInputGet(req, reply, { inputOf, refused = () => {}, noteFirstInputBytes = () => {} }) {
   const address = req.raw?.socket?.remoteAddress;
   if (!["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(address)) return reply.code(403).send();
   const token = Number(req.params.token), fileIndex = Number(req.params.fileIndex);
@@ -53,6 +53,16 @@ export async function handleEncodeInputGet(req, reply, { inputOf, refused = () =
     return reply;
   }
   const body = held.stream ?? (held.chunks ? Readable.from(held.chunks, { objectMode: false }) : held.bytes);
+  if (body && typeof body.once === "function") {
+    let noted = false;
+    body.once("data", chunk => {
+      if (noted) return;
+      noted = true;
+      noteFirstInputBytes(token, fileIndex, chunk?.length ?? 0);
+    });
+  } else {
+    noteFirstInputBytes(token, fileIndex, body?.length ?? 0);
+  }
   // What was handed to the system for this answer before its connection
   // closed, and whether all of it was: an answer runs to the end of its
   // admitted range and FFmpeg closes the connection once it has what it wants,

package/routes/stream/get.js

@@ -229,7 +229,8 @@ function serveWholeFile(req, reply, file, read) {
   }
 }
 
-export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool, noteInputBytes = null, noteInputWaiting = null }) {
+export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool, noteInputBytes = null, noteInputWaiting = null,
+  noteFirstInputBytes = null }) {
   const fileIndexRaw = typeof req.query.fileIndex === "string" ? req.query.fileIndex : "";
   const fileIndex = Number(fileIndexRaw);
   const { sourceType, source, sourceKey } = getSourceParams(req.query, sourceRegistry);
@@ -386,6 +387,9 @@ export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool,
               reply.raw.write(fragment.bytes, (error) => (error ? reject(error) : resolve()));
             });
             sent += fragment.bytes.length;
+            if (sent === fragment.bytes.length && noteFirstInputBytes && Number.isSafeInteger(runToken)) {
+              noteFirstInputBytes(runToken, fileIndex, fragment.bytes.length);
+            }
           // Say so, if this read belongs to a transcode session. It is the only
           // proof that a session which has produced nothing yet is nevertheless
           // being fed: the encoder's own progress cannot move until its first
@@ -412,10 +416,15 @@ export async function handleStreamGet(req, reply, { sourceRegistry, torrentPool,
       if (continueFrom) {
         fragments.cancel();
         const fromTorrent = sent;
+        let firstWholeInputNoted = false;
         sent += await writeFromWholeFile(reply.raw, continueFrom, start + sent, end, (length) => {
           if (noteInputBytes) {
             noteInputBytes(sessionId, length);
           }
+          if (!firstWholeInputNoted && noteFirstInputBytes && Number.isSafeInteger(runToken)) {
+            firstWholeInputNoted = true;
+            noteFirstInputBytes(runToken, fileIndex, length);
+          }
         });
         logger.info(
           `stream: read of "${file.name}" bytes ${start}-${end} continued from the whole file: ` +

package/server.js

@@ -178,7 +178,8 @@ export async function startProxyServer({
   await app.register(fastifyCors, {
     origin: true,
     methods: ["GET", "POST", "OPTIONS"],
-    allowedHeaders: ["Content-Type", "Range", "X-Ttv-Wait-Trace", "X-Ttv-Wait-Attempt", "X-Ttv-Wait-Operation"]
+    allowedHeaders: ["Content-Type", "Range", "X-Ttv-Wait-Trace", "X-Ttv-Wait-Attempt", "X-Ttv-Wait-Operation",
+      "X-Ttv-Wait-Process"]
   });
 
   // Allow browser requests from an HTTPS page to this private-network proxy
@@ -223,7 +224,15 @@ export async function startProxyServer({
   // 10 MB file took 12-23 s against 125 ms to hand it to the channel. The
   // adapter keeps TorrentPool's interface, so nothing downstream changed.
   const torrentPool = new WorkerTorrentPool({
-    onObserved: fields => waitTrace.source(fields.sourceKey, "torrent-ready", fields),
+    isWaitTraced: sourceKey => waitTrace.has(sourceKey),
+    onWaitTrace: event => waitTrace.process(event.sourceKey, event.process, event.phase, event),
+    onObserved: fields => {
+      waitTrace.source(fields.sourceKey, "process", { process: "P1", phase: "begin", at: fields.startedAt,
+        outcome: fields.outcome });
+      waitTrace.source(fields.sourceKey, "process", { process: "P1", phase: "end", at: fields.endedAt,
+        startedAt: fields.startedAt, endedAt: fields.endedAt, durationMs: fields.endedAt - fields.startedAt,
+        outcome: fields.outcome });
+    },
     memoryBytes,
     stateDir,
     heapSnapshots,
@@ -254,8 +263,8 @@ export async function startProxyServer({
     // only once the worker thread is up and a piece has verified, which is long
     // after the declarations below have run.
     onPiecesArrived: ({ sourceKey, fileIndexes, arrived }) => {
-      waitTrace.source(sourceKey, "held-bytes-changed", { fileIndexes, arrived: arrived?.slice?.(0, 32),
-        arrivedCount: arrived?.length });
+      waitTrace.source(sourceKey, "process", { process: "W9", phase: "end", fileIndexes,
+        arrived: arrived?.slice?.(0, 32), arrivedCount: arrived?.length });
       for (const fileIndex of fileIndexes) {
         mediaReads.bytesChanged(sourceKey, fileIndex);
         probeReads.bytesChanged(sourceKey, fileIndex);
@@ -654,6 +663,7 @@ export async function startProxyServer({
     // already from the projection; this packs them with the native ones.
     pack: (zones) => coalescedZones(zones),
     log: (line) => logger.info(line),
+    observeWait: (sourceKey, event, fields) => waitTrace.source(sourceKey, event, fields),
     resolvePlayback: async (map) => {
       const fileKey = `${map.sourceKey}:${map.fileIndex}`;
       if (map.zones.length === 0) {
@@ -877,7 +887,16 @@ export async function startProxyServer({
         } }),
       label: String(file.name ?? ""),
       fileOffset: file.offset,
-      onObserved: fields => waitTrace.source(sourceKey, "container-read", { fileIndex, requestId, ...fields }),
+      onObserved: fields => {
+        const process = fields.statement === "keyframes" ? "P5"
+          : String(fields.statement).startsWith("ffprobe:") ? "X1" : "P2";
+        const details = { process, fileIndex, requestId, statement: fields.statement, cached: fields.cached,
+          result: fields.result, reason: fields.reason, ranges: fields.ranges,
+          queuedMs: fields.queuedMs, readMs: fields.readMs };
+        waitTrace.source(sourceKey, "process", { ...details, phase: "begin", at: fields.askedAt });
+        waitTrace.source(sourceKey, "process", { ...details, phase: "end", at: fields.endedAt,
+          startedAt: fields.askedAt, endedAt: fields.endedAt });
+      },
       onTracks: (tracks) => contentsOf(torrent).noteVideo(fileIndex, tracks.some((track) => track.type === "video")),
       onReadStart: () => ({ storage: mediaReads.revision(sourceKey, fileIndex), memory: mediaReads.memoryRevision(), demand: downloadMaps.epoch(sourceKey, fileIndex) }),
       onReadResult: async (statement, result, revision) => {
@@ -1474,7 +1493,8 @@ export async function startProxyServer({
   app.post("/api/sources", async (req, reply) => {
     // A cold start begins with the first source opened (torrent-tv/meta#177).
     startupProfile.begin();
-    return handleApiSourcesPost(req, reply, { sourceRegistry, viewers: outputParts.viewers });
+    return handleApiSourcesPost(req, reply, { sourceRegistry, viewers: outputParts.viewers,
+      associateWaitTrace: (request, sourceKey) => waitTrace.associate(request, sourceKey) });
   });
   app.get("/api/sources/:sourceKey/stats", async (req, reply) =>
     handleApiSourceStatsGet(req, reply, { sourceRegistry, torrentPool, viewerRequests: outputParts.viewerRequests })
@@ -1563,6 +1583,7 @@ export async function startProxyServer({
   );
   app.get("/encode-input/:token/:fileIndex", (req, reply) =>
     handleEncodeInputGet(req, reply, { inputOf: token => outputParts.encodeRuns.originalInputOf(token),
+      noteFirstInputBytes: (token, fileIndex, bytes) => outputParts.encodeRuns.noteFirstInputBytes(token, fileIndex, bytes),
       refused: ({ fileIndex, start, runTag }) => logger.warn(`encode input: run ${runTag ?? "?"} asked for byte ${start} of file ${fileIndex}, which it was not given`) })
   );
   app.get("/stream", async (req, reply) =>
@@ -1573,6 +1594,7 @@ export async function startProxyServer({
       // fed. The route knows only files; the session id rides on the URL the
       // session itself built.
       noteInputBytes: (sessionId, bytes) => outputParts.encodeRuns.noteInputBytes(sessionId, bytes),
+      noteFirstInputBytes: (token, fileIndex, bytes) => outputParts.encodeRuns.noteFirstInputBytes(token, fileIndex, bytes),
       // When the encoder reading this waits for its input, so its speed is
       // measured over its own work and not over the swarm's.
       noteInputWaiting: (runToken, waiting) => outputParts.encodeRuns.noteInputWaiting(runToken, waiting)

package/services/encode/EncodeInputs.js

@@ -65,10 +65,13 @@ export class EncodeInputs {
    * @type {(output: object, from: number, to: number, input: object) => void}
    */
   #stretchCopied;
+  #observeWait;
 
   constructor({ resolve, readRanges, openWholeFile = null, heldRanges = null, reviseBudget, changed, failed, capacity = () => null, urgent = () => true,
-    rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {} }) {
+    rankAt = () => null, openOf = () => null, log = () => {}, now = Date.now, stretchCopied = () => {},
+    observeWait = null }) {
     this.#stretchCopied = stretchCopied;
+    this.#observeWait = observeWait;
     this.#openOf = openOf;
     this.#resolve = resolve;
     this.#read = readRanges;
@@ -103,6 +106,14 @@ export class EncodeInputs {
       `${rank ? `, rank ${rank.rank} of ${rank.topRank}` : ""})${detail}`);
   }
 
+  #observe(request, phase, fields = {}) {
+    const sourceKey = request.output?.file?.sourceKey;
+    if (typeof sourceKey !== "string") return;
+    const process = request.output?.spec?.carries === "audio-only" ? "P8.a" : "P8.v";
+    this.#observeWait?.(sourceKey, "process", { process, phase, at: performance.now(),
+      output: request.output.outputKey ?? request.output.id, from: request.from, to: request.to, ...fields });
+  }
+
   /** The container's statement of an interval, its time added to the request's account. */
   async #resolveTimed(timing, output, from, to, isCurrent = null) {
     const startedAt = this.#now();
@@ -258,7 +269,8 @@ export class EncodeInputs {
       return existing.result;
     }
     if (existing?.pending || existing?.revision === this.#revision || existing?.result?.kind === "terminal") return null;
-    const request = existing ?? { output, from, to, key, serves };
+    const request = existing ?? { output, from, to, key, serves, traceStartedAt: performance.now() };
+    if (!existing) this.#observe(request, "begin", { request: key });
     this.#requests.set(key, request);
     this.#prepare(request);
     return null;
@@ -403,6 +415,7 @@ export class EncodeInputs {
     request.timing ??= { askedAt: this.#now(), attempts: 0, resolutions: 0, resolveMs: 0,
       reads: 0, queuedMs: 0, readMs: 0, heldMs: 0, budgetMs: 0 };
     request.timing.attempts += 1;
+    this.#observe(request, "attempt", { request: request.key, attempt: request.timing.attempts });
     request.promise = (async () => {
       // A request the plan withdraws stops its reads: a soundtrack piece's
       // packet walk, left to run, held the file's reads and the piece the
@@ -449,10 +462,20 @@ export class EncodeInputs {
           : `, copied in ${result.copyMs}ms`) : "")
         : `${result.kind}${result.reason ? ` ${result.reason}` : ""}${Number.isFinite(result.bytes) ? ` ${result.bytes} bytes` : ""}`,
       describeTiming(request.timing, this.#now()));
+      if (result.kind === "result" || result.kind === "terminal") {
+        this.#observe(request, "end", { request: request.key, startedAt: request.traceStartedAt,
+          endedAt: performance.now(), outcome: result.kind, reason: result.reason ?? null,
+          bytes: result.bytes ?? null, attempts: request.timing.attempts });
+      } else {
+        this.#observe(request, "waiting", { request: request.key, reason: result.kind,
+          bytes: result.bytes ?? null, attempts: request.timing.attempts });
+      }
       if (result.kind === "result" || result.kind === "terminal") this.#changed(request.output, result);
     })().catch(error => {
       if (this.#requests.get(request.key) === request) {
         request.result = { kind: "terminal", reason: "input-preparation-failed", message: error.message };
+        this.#observe(request, "end", { request: request.key, startedAt: request.traceStartedAt,
+          endedAt: performance.now(), outcome: "error", reason: "input-preparation-failed", attempts: request.timing.attempts });
         this.#failed(request.output, error);
       }
     }).finally(() => {

package/services/encode/EncodeRuns.js

@@ -285,6 +285,21 @@ export class EncodeRuns {
     }
   }
 
+  noteFirstInputBytes(runToken, fileIndex, bytes) {
+    const run = this.#runsByInputToken.get(runToken);
+    if (!run || run.firstWaitTraceInputAt !== undefined) return;
+    const endedAt = performance.now();
+    run.firstWaitTraceInputAt = endedAt;
+    const { sourceKey, process } = run.waitTraceInput ?? {};
+    this.#host.observeWait?.(sourceKey, "process", { process, phase: "end", runToken, fileIndex, bytes,
+      startedAt: run.startedAt, endedAt, durationMs: endedAt - run.startedAt });
+    const f2Process = process === "F1.a" ? "F2.a" : "F2.v";
+    run.waitTraceInput.f2Process = f2Process;
+    run.waitTraceInput.f2StartedAt = endedAt;
+    this.#host.observeWait?.(sourceKey, "process", { process: f2Process, phase: "begin", runToken,
+      fileIndex, bytes, startedAt: endedAt });
+  }
+
   originalInputOf(runToken) {
     return this.#runsByInputToken.get(runToken)?.originalInput ?? null;
   }
@@ -561,6 +576,7 @@ export class EncodeRuns {
   }
 
   planEncodersNow() {
+    const planStartedAt = performance.now();
     const byOutput = this.#outputsWithSessions();
     for (const [address, sessions] of byOutput) {
       // From the TIMELINE, which is where how a file is cut has lived since
@@ -609,7 +625,22 @@ export class EncodeRuns {
         ) ?? []
       );
     }
+    for (const [address, sessions] of byOutput) {
+      const sourceKey = sessions[0]?.file?.sourceKey;
+      if (typeof sourceKey !== "string") continue;
+      const process = sessions[0]?.spec?.carries === "audio-only" ? "P7.a" : "P7.v";
+      this.#host.observeWait?.(sourceKey, "process", { process, phase: "begin", at: planStartedAt,
+        output: address, sessions: sessions.length });
+    }
     this.#host.encodeOrchestrator.reconcile();
+    const planEndedAt = performance.now();
+    for (const [address, sessions] of byOutput) {
+      const sourceKey = sessions[0]?.file?.sourceKey;
+      if (typeof sourceKey !== "string") continue;
+      const process = sessions[0]?.spec?.carries === "audio-only" ? "P7.a" : "P7.v";
+      this.#host.observeWait?.(sourceKey, "process", { process, phase: "end", at: planEndedAt,
+        startedAt: planStartedAt, endedAt: planEndedAt, output: address, sessions: sessions.length });
+    }
   }
 
   /**
@@ -1164,6 +1195,13 @@ export class EncodeRuns {
         // It belongs to no requested output segment and must not condemn the
         // complete input or overwrite a neighboring run's segment.
         if (Number.isInteger(index) && (index < safeIndex || index > run.to)) return null;
+        if (index === safeIndex && run.waitTraceInput?.f2StartedAt && !run.waitTraceInput.f2EndedAt) {
+          const endedAt = performance.now();
+          run.waitTraceInput.f2EndedAt = endedAt;
+          this.#host.observeWait?.(session.file.sourceKey, "process", { process: run.waitTraceInput.f2Process,
+            phase: "end", runToken: inputToken, segment: index, startedAt: run.waitTraceInput.f2StartedAt,
+            endedAt, durationMs: endedAt - run.waitTraceInput.f2StartedAt, outcome: "piece-closed" });
+        }
         let bytes = null;
         const address = session.outputKey ?? "";
         // Read before anything rewrites it: when the encoder finished it.
@@ -1223,6 +1261,13 @@ export class EncodeRuns {
         this.#host.noteRunSpeedMeasured?.(session, run);
       },
       onEnded: (ended) => {
+        if (run.waitTraceInput?.f2StartedAt && !run.waitTraceInput.f2EndedAt) {
+          const endedAt = performance.now();
+          run.waitTraceInput.f2EndedAt = endedAt;
+          this.#host.observeWait?.(session.file.sourceKey, "process", { process: run.waitTraceInput.f2Process,
+            phase: "end", runToken: inputToken, segment: safeIndex, startedAt: run.waitTraceInput.f2StartedAt,
+            endedAt, durationMs: endedAt - run.waitTraceInput.f2StartedAt, outcome: ended.ending ?? "ended" });
+        }
         this.#runsByInputToken.delete(inputToken);
         if (admittedInput.original) {
           const reads = admittedInput.reads?.();
@@ -1244,6 +1289,11 @@ export class EncodeRuns {
         this.#host.encodersChanged?.();
       }
     });
+    const f1Process = session.spec.carries === "audio-only" ? "F1.a" : "F1.v";
+    run.waitTraceInput = { sourceKey: session.file.sourceKey, process: f1Process };
+    this.#host.observeWait?.(session.file.sourceKey, "process", { process: f1Process,
+      phase: "begin", runToken: inputToken, startedAt: run.startedAt, output: session.outputKey,
+      from: run.from, to: run.to });
     if (admittedInput.original) {
       run.originalInput = admittedInput;
       // Every number of the stretch answers to its one key, so a failure

package/services/media/ContainerOrchestrator.js

@@ -164,9 +164,8 @@ export class ContainerOrchestrator {
       const result = await this.#inspectRead({ ...params, isCurrent }, statement);
       const endedAt = performance.now();
       params.onTimed?.(statement, startedAt - askedAt, endedAt - startedAt);
-      if (statement !== "source-ranges" || endedAt - askedAt >= 10) params.onObserved?.({ statement, askedAt,
-        startedAt, endedAt, queuedMs: startedAt - askedAt, readMs: endedAt - startedAt, cached: aside,
-        result: result.kind, reason: result.reason, ranges: result.ranges });
+      params.onObserved?.({ statement, askedAt, startedAt, endedAt, queuedMs: startedAt - askedAt,
+        readMs: endedAt - startedAt, cached: aside, result: result.kind, reason: result.reason, ranges: result.ranges });
       return { result, revision };
     });
     if (!aside) this.#reads.set(key, reading);

package/services/server/SegmentServing.js

@@ -110,7 +110,17 @@ export class SegmentServing {
   async getFileStream(sessionId, fileName, options = {}) {
     const output = this.#host.outputs.get(sessionId);
     if (!output) return { kind: "not-found" };
-    return this.#withMaterialRead(output.outputKey, () => this.#getFileStream(sessionId, fileName, options));
+    const isInit = output.segmentFormat?.initFileName !== null && fileName === output.segmentFormat?.initFileName;
+    const startedAt = performance.now();
+    const answer = await this.#withMaterialRead(output.outputKey, () => this.#getFileStream(sessionId, fileName, options));
+    if (isInit && typeof output.file?.sourceKey === "string") {
+      const process = output.spec?.carries === "audio-only" ? "P13.a" : "P13.v";
+      const fields = { output: output.outputKey, fileName, outcome: answer?.kind ?? "unknown" };
+      this.#host.observeWait?.(output.file.sourceKey, "process", { ...fields, process, phase: "begin", at: startedAt });
+      this.#host.observeWait?.(output.file.sourceKey, "process", { ...fields, process, phase: "end",
+        at: performance.now(), startedAt, endedAt: performance.now() });
+    }
+    return answer;
   }
 
   async #withMaterialRead(key, read) {

package/services/server/ViewerRequests.js

@@ -120,7 +120,7 @@ export class ViewerRequests {
     };
     const current = this.#earlyWaits.get(key);
     if (current?.waitId === waitId && current.hasSourceObservation) {
-      const context = this.#setEarlyWait({ consumerId, generation, fileIndex, waitId, features,
+      const context = this.#setEarlyWait({ consumerId, generation, sourceKey, fileIndex, waitId, features,
         revision: current.revision, current, waitElapsedMs });
       const changed = this.#recordEarlyWaitSnapshot(context, context.features);
       if (changed) context.revision += 1;
@@ -136,7 +136,7 @@ export class ViewerRequests {
     }
     const sameWait = current?.waitId === waitId;
     const revision = sameWait ? current.revision : 0;
-    const context = this.#setEarlyWait({ consumerId, generation, fileIndex, waitId, features, revision, current,
+    const context = this.#setEarlyWait({ consumerId, generation, sourceKey, fileIndex, waitId, features, revision, current,
       waitElapsedMs });
     const changed = this.#recordEarlyWaitSnapshot(context, context.features);
     if (sameWait && changed) context.revision += 1;
@@ -186,7 +186,7 @@ export class ViewerRequests {
     const current = this.#earlyWaits.get(key);
     const sameWait = current?.waitId === waitId;
     const revision = sameWait ? current.revision : 0;
-    const context = this.#setEarlyWait({ consumerId, generation, fileIndex, waitId, features, revision, current,
+    const context = this.#setEarlyWait({ consumerId, generation, sourceKey, fileIndex, waitId, features, revision, current,
       sourceObserved: true, waitElapsedMs });
     const featuresChanged = this.#recordEarlyWaitSnapshot(context, context.features);
     const elapsedMs = waitElapsedSince(context, waitElapsedMs);
@@ -231,6 +231,9 @@ export class ViewerRequests {
     logger.info(`early wait forecast ${JSON.stringify({ waitId: context.waitId, consumer: context.consumerId,
       generation: context.generation, features: context.features, elapsedMs, revision: context.revision,
       predicted: { remainingSeconds: seconds, totalSeconds, basis, ...estimate } })}`);
+    this.#host.observeWait?.(context.sourceKey, "process", { process: "P15", phase: "event",
+      waitId: context.waitId, consumerId: context.consumerId, generation: context.generation,
+      elapsedMs, revision: context.revision, remainingSeconds: seconds, totalSeconds, basis, ...estimate });
     context.loggedForecastRevision = context.revision;
   }
 
@@ -258,12 +261,13 @@ export class ViewerRequests {
     early.updatedAt = Date.now();
   }
 
-  #setEarlyWait({ consumerId, generation, fileIndex, waitId, features, revision, current,
+  #setEarlyWait({ consumerId, generation, sourceKey, fileIndex, waitId, features, revision, current,
     sourceObserved = false, waitElapsedMs }) {
     const sameWait = current?.waitId === waitId;
     const context = sameWait ? current : {
       consumerId,
       generation,
+      sourceKey,
       fileIndex,
       waitId,
       snapshots: [],
@@ -279,7 +283,7 @@ export class ViewerRequests {
       ? Math.max(0, measuredAt - context.featuresMeasuredAt)
       : 0;
     const mergedFeatures = mergeWaitFeatures(context.features, features, ageElapsedMs);
-    Object.assign(context, { consumerId, generation, fileIndex, waitId, features: mergedFeatures,
+    Object.assign(context, { consumerId, generation, sourceKey, fileIndex, waitId, features: mergedFeatures,
       featuresMeasuredAt: measuredAt, startedAt, updatedAt: Date.now(), revision,
       hasSourceObservation: context.hasSourceObservation || sourceObserved });
     this.#earlyWaits.set(`${consumerId}:${waitId}`, context);
@@ -1034,6 +1038,11 @@ export class ViewerRequests {
     if (measurement.forecastReason !== forecast.reason || measurement.forecastInputKey !== forecastInputKey) {
       measurement.forecastReason = forecast.reason;
       measurement.forecastInputKey = forecastInputKey;
+      if (forecast.subtitles) this.#host.observeWait?.(session.file.sourceKey, "process", {
+        process: "C3", phase: "event", fileIndex: session.file.fileIndex,
+        trackIndex: forecast.subtitles.trackIndex, ready: forecast.subtitles.ready,
+        reason: forecast.reason, consumerId
+      });
       logger.info(`playback readiness ${session.id} consumer=${consumerId} ${JSON.stringify({
         ...forecast,
         positionSeconds,

package/services/server/wire-outputs.js

@@ -535,6 +535,7 @@ export function wireOutputs({
   });
   // Answering a request for a playlist, an init segment or a segment: from the store when the piece is made and whole, otherwise held until it is.
   parts.serving = new SegmentServing({
+    observeWait,
     // What a key names, read back from the key: how to find a gone output's
     // stored pieces and whether they are cut at given times.
     segmentFormatOfKey,
@@ -624,6 +625,7 @@ export function wireOutputs({
   });
   // What a viewer asks for: an output of a file, a position, a report of progress.
   parts.viewerRequests = new ViewerRequests({
+    observeWait,
     // Whether this machine has a place for one more viewer on this output —
     // asked in the same synchronous stretch as they are put on it.
     admitsWatching: (output) => parts.admission.admitsWatching(output?.outputKey ?? ""),
@@ -1189,6 +1191,7 @@ export function wireOutputs({
     // The piece's rank in the output's own map, said with an input's state.
     rankAt: (output, index) => parts.encodeOrchestrator?.rankAt?.(output.outputKey ?? "", index) ?? null,
     log: line => logger.info(line),
+    observeWait,
     changed: (output, result) => {
       if (result.kind === "terminal") {
         logger.warn(`encode input output=${output.outputKey} terminal=${result.reason}: ${result.message ?? ""}` +

package/services/torrent/piece-timing.js

@@ -38,6 +38,7 @@ export class PieceTimings {
    */
   asked(piece, offset, peer, now) {
     let record = this.#pieces.get(piece);
+    const firstAsk = !record;
     if (!record) {
       record = { firstAskAt: now, asks: 0, peers: new Set(), blocks: new Map(), arrived: new Set(), askedAgain: 0, arrivedTwice: 0,
         slowestMs: 0, slowestPeer: null, lastInFlightMs: 0, lastPeer: null };
@@ -47,6 +48,7 @@ export class PieceTimings {
     record.peers.add(peer);
     if (!record.blocks.has(offset)) record.blocks.set(offset, now);
     else record.askedAgain += 1;
+    return firstAsk;
   }
 
   /**

package/services/torrent/torrent-pool.js

@@ -969,6 +969,8 @@ export class TorrentPool {
    * @type {WeakMap<import("webtorrent").Torrent, { addedAt: number, firstPeerAt: number | null }>}
    */
   #swarmTimingByTorrent = new WeakMap();
+  #waitTraceByTorrent = new WeakMap();
+  #waitTraceHooks = new WeakSet();
 
   /**
    * Sources every file of which this proxy holds whole.
@@ -1502,7 +1504,60 @@ export class TorrentPool {
    * @param {import("webtorrent").Torrent} torrent
    * @returns {void}
    */
-  #attachSwarmDiagnostics(torrent) {
+  #attachSwarmDiagnostics(torrent, waitTrace = null) {
+    if (waitTrace?.trace === true && typeof waitTrace.sourceKey === "string") {
+      this.#waitTraceByTorrent.set(torrent, waitTrace);
+    }
+    const trace = (process, phase, fields = {}) => {
+      const context = this.#waitTraceByTorrent.get(torrent);
+      if (context?.trace === true) context.emitWaitTrace?.(context.sourceKey, process, phase, fields);
+    };
+    const wrap = (method, process) => {
+      const original = torrent[method];
+      if (typeof original !== "function") return;
+      torrent[method] = function (...args) {
+        const startedAt = performance.now();
+        trace(process, "begin", { startedAt });
+        try {
+          const result = original.apply(this, args);
+          if (result && typeof result.then === "function") {
+            return result.then(value => {
+              trace(process, "end", { startedAt, endedAt: performance.now(), outcome: "resolved" });
+              return value;
+            }, error => {
+              trace(process, "end", { startedAt, endedAt: performance.now(), outcome: "error",
+                error: String(error?.message ?? error) });
+              throw error;
+            });
+          }
+          trace(process, "end", { startedAt, endedAt: performance.now(), outcome: "returned" });
+          return result;
+        } catch (error) {
+          trace(process, "end", { startedAt, endedAt: performance.now(), outcome: "error",
+            error: String(error?.message ?? error) });
+          throw error;
+        }
+      };
+    };
+    if (waitTrace?.trace === true && !this.#waitTraceHooks.has(torrent)) {
+      this.#waitTraceHooks.add(torrent);
+      wrap("_onListening", "W1");
+      wrap("_startDiscovery", "W2");
+      wrap("_onStore", "W6");
+      const verify = torrent._verifyPieces;
+      if (typeof verify === "function") {
+        torrent._verifyPieces = function (callback, ...args) {
+          const startedAt = performance.now();
+          trace("W5", "begin", { startedAt });
+          return verify.call(this, (...results) => {
+            trace("W5", "end", { startedAt, endedAt: performance.now(), outcome: "verified" });
+            callback?.(...results);
+          }, ...args);
+        };
+      }
+      torrent.on("metadata", metadata => trace("W4", "event", { files: metadata?.files?.length ?? null }));
+      torrent.on("wire", wire => trace("W3", "event", { peer: String(wire?.remoteAddress ?? "?") }));
+    }
     // Attached ONCE per torrent. WebTorrent answers a duplicate add by handing
     // back the torrent it already has, and this used to run again on it: a
     // torrent with twelve connections and a first peer five minutes old had its
@@ -1510,7 +1565,8 @@ export class TorrentPool {
     // connection printed "first peer connected after 0.3s" — a false statement
     // about a swarm that had been healthy for minutes. The same film opened
     // once as a .torrent and once as a magnet is exactly that case, and the
-    // roadmap already records it happening.
+    // roadmap already records it happening. Trace hooks above are installed
+    // separately and only for an explicitly traced source.
     if (this.#swarmTimingByTorrent.has(torrent)) {
       return;
     }
@@ -1547,6 +1603,7 @@ export class TorrentPool {
     // its blocks were in flight, and from whom (`piece-timing.js`). One line
     // per piece: a 2.3 GB film is 563.
     const timings = new PieceTimings();
+    const waitTracePieceStarts = new Map();
     tailRepeats.set(torrent, new TailRepeat({ torrent, requiredPieces: () => demandFor(torrent).selection.requiredPieces(),
       log: (line) => logger.info(`download ${label()} ${line}`) }));
     torrent.on("wire", (wire) => {
@@ -1561,17 +1618,33 @@ export class TorrentPool {
         const answer = request.call(this, index, offset, length, callback);
         if (this.requests.length > queued) {
           const now = Date.now();
-          timings.asked(index, offset, peer, now);
+          const firstAsk = timings.asked(index, offset, peer, now);
+          if (firstAsk) {
+            const startedAt = performance.now();
+            waitTracePieceStarts.set(index, startedAt);
+            trace("W8", "begin", { piece: index, startedAt, firstOffset: offset, blockLength: length });
+          }
           // When this wire was asked for the block: how long it has held it is
           // what decides whether a second peer is asked (`duplicateTailFor`).
           this.requests.at(-1).askedAt = now;
         }
         return answer;
       };
-      wire.on("piece", (index, offset) => timings.arrived(index, offset, peer, Date.now()));
+      wire.on("piece", (index, offset) => {
+        timings.arrived(index, offset, peer, Date.now());
+      });
     });
     torrent.on("verified", (piece) => {
-      logger.info(`download ${label()} verified piece=${piece}${describePieceTiming(timings.verified(piece, Date.now()))}`);
+      const pieceTiming = timings.verified(piece, Date.now());
+      const startedAt = waitTracePieceStarts.get(piece);
+      if (startedAt !== undefined) {
+        waitTracePieceStarts.delete(piece);
+        const endedAt = performance.now();
+        trace("W8", "end", { piece, startedAt, endedAt, durationMs: endedAt - startedAt,
+          outcome: "verified", asks: pieceTiming?.asks ?? null, peers: pieceTiming?.peers ?? null,
+          askedAgain: pieceTiming?.askedAgain ?? null, slowestBlockMs: pieceTiming?.slowestMs ?? null });
+      }
+      logger.info(`download ${label()} verified piece=${piece}${describePieceTiming(pieceTiming)}`);
     });
 
     // The first connected peer, said once, because the wait for it can BE the
@@ -1694,7 +1767,7 @@ export class TorrentPool {
    * @param {string} source - Magnet URI or base64-encoded .torrent bytes.
    * @returns {Promise<import("webtorrent").Torrent>}
    */
-  async getTorrent(sourceType, source) {
+  async getTorrent(sourceType, source, waitTrace = null) {
     // Keyed by the torrent's own infohash, not the source bytes — a magnet
     // and a `.torrent` file for the same content must name the same swarm.
     const key = await deriveSourceKey(sourceType, source);
@@ -1702,6 +1775,7 @@ export class TorrentPool {
     // Already resolved — return immediately.
     const existing = this.torrents.get(key);
     if (existing) {
+      this.#attachSwarmDiagnostics(existing, waitTrace);
       this.#lastAccess.set(existing, Date.now());
       return existing;
     }
@@ -1740,7 +1814,7 @@ export class TorrentPool {
             );
             existing.destroy({ destroyStore: false }, () => {
               this.#pending.delete(key);
-              resolve(this.getTorrent(sourceType, source));
+              resolve(this.getTorrent(sourceType, source, waitTrace));
             });
             return;
           }
@@ -1792,10 +1866,10 @@ export class TorrentPool {
                   this.torrents.set(key, replacement);
                   this.#lastAccess.set(replacement, Date.now());
                   this.#pending.delete(key);
-                  this.#attachSwarmDiagnostics(replacement);
+                  this.#attachSwarmDiagnostics(replacement, waitTrace);
                   resolve(replacement);
                 });
-                this.#attachSwarmDiagnostics(addedReplacement);
+                this.#attachSwarmDiagnostics(addedReplacement, waitTrace);
               });
               return;
             }
@@ -1816,6 +1890,10 @@ export class TorrentPool {
       // piece across threads detached memory still in use. Ours owns what it
       // hands out, holds pieces in shared memory the main thread can read
       // directly, and spills to disk instead of losing them.
+      const skipVerify = this.#wholeSources.has(key);
+      if (skipVerify && waitTrace?.trace === true) {
+        waitTrace.emitWaitTrace?.(waitTrace.sourceKey, "W5", "event", { outcome: "skipped" });
+      }
       const added = this.client.add(torrentId, {
         store: this.#pieceStore,
         storeCacheSlots: 0,
@@ -1831,7 +1909,7 @@ export class TorrentPool {
         // whole: what would be verified was written out of pieces this client
         // had hashed, and the file's size was checked against what the torrent
         // says. See `#wholeSources`.
-        skipVerify: this.#wholeSources.has(key)
+        skipVerify
       }, (readyTorrent) => {
         this.client.off("error", onError);
         this.torrents.set(key, readyTorrent);
@@ -1842,7 +1920,7 @@ export class TorrentPool {
         // back the one it already had. Watching only what `add` returned would
         // leave the survivor unwatched. Attaching twice costs nothing — the
         // guard makes the second call a no-op when it is the same object.
-        this.#attachSwarmDiagnostics(readyTorrent);
+        this.#attachSwarmDiagnostics(readyTorrent, waitTrace);
         // On the clock from the moment it exists. A torrent nobody ever states
         // anything about — a file list fetched and never played — used to be
         // held for the life of the process, because the only thing that ever
@@ -1858,7 +1936,7 @@ export class TorrentPool {
       // `wire` is emitted at the moment of connection and never replayed. The
       // torrent that waited minutes for its first peer would have been the one
       // case this could not measure.
-      this.#attachSwarmDiagnostics(added);
+      this.#attachSwarmDiagnostics(added, waitTrace);
     });
 
     this.#pending.set(key, promise);

package/services/torrent/worker/client.js

@@ -137,11 +137,13 @@ export class TorrentWorkerClient {
   #onSourceForgotten;
   /** Told when what this proxy holds changed; see `Event.HOLDINGS_CHANGED`. */
   #onHoldingsChanged;
+  #onWaitTrace;
 
   /**
    * @param {{ memoryBytes?: number, stateDir?: string, onPiecesArrived?: (event: object) => void }} [options]
    */
-  constructor({ memoryBytes, stateDir, heapSnapshots = false, onPiecesArrived, onSourceForgotten, onHoldingsChanged } = {}) {
+  constructor({ memoryBytes, stateDir, heapSnapshots = false, onPiecesArrived, onSourceForgotten, onHoldingsChanged,
+    onWaitTrace } = {}) {
     this.#worker = new Worker(fileURLToPath(WORKER_URL), {
       // `stateDir` travels because the worker writes heap snapshots of its own
       // isolate there. It cannot choose a directory any other way: a worker may
@@ -153,6 +155,7 @@ export class TorrentWorkerClient {
     this.#onPiecesArrived = onPiecesArrived ?? (() => undefined);
     this.#onSourceForgotten = onSourceForgotten ?? (() => undefined);
     this.#onHoldingsChanged = onHoldingsChanged ?? (() => undefined);
+    this.#onWaitTrace = onWaitTrace ?? (() => undefined);
 
     this.#worker.on("message", (message) => {
       // A failed read must fail its stream. This is checked BEFORE the caller
@@ -254,6 +257,9 @@ export class TorrentWorkerClient {
           // taken twice on two different histories.
           writeAlreadyDecided(message.level ?? "info", `torrent-worker: ${message.message}`);
           break;
+        case Event.WAIT_TRACE:
+          this.#onWaitTrace(message);
+          break;
         case Event.FILE_COMPLETE:
           // Recorded here so the stream route can answer from the file without
           // going near the torrent thread at all.
@@ -346,8 +352,8 @@ export class TorrentWorkerClient {
    * @param {{ sourceKey: string, sourceType: "magnet" | "torrent", source: string }} params
    * @returns {Promise<{ infoHash: string, name: string, files: { index: number, name: string, path: string, length: number }[] }>}
    */
-  async addSource({ sourceKey, sourceType, source }) {
-    return this.#caller.call(Command.ADD_SOURCE, { sourceKey, sourceType, source });
+  async addSource({ sourceKey, sourceType, source, trace = false }) {
+    return this.#caller.call(Command.ADD_SOURCE, { sourceKey, sourceType, source, trace });
   }
 
   /**
@@ -675,8 +681,8 @@ export class TorrentWorkerClient {
    * @param {{ sourceKey: string, sourceType: "magnet" | "torrent", source: string }} params
    * @returns {Promise<{ infoHash: string, name: string, pieceLength: number, sourceKey: string, files: object[] }>}
    */
-  async getTorrent({ sourceKey, sourceType, source }) {
-    const info = await this.addSource({ sourceKey, sourceType, source });
+  async getTorrent({ sourceKey, sourceType, source, trace = false }) {
+    const info = await this.addSource({ sourceKey, sourceType, source, trace });
     const client = this;
     return {
       infoHash: info.infoHash,

package/services/torrent/worker/pool-adapter.js

@@ -32,6 +32,7 @@ export class WorkerTorrentPool {
   #torrents = new Map();
   #pending = new Map();
   #observed;
+  #isWaitTraced;
 
   /**
    * @param {{ memoryBytes?: number, stateDir?: string }} [options]
@@ -39,6 +40,7 @@ export class WorkerTorrentPool {
    */
   constructor(options = {}, client = null) {
     this.#observed = options.onObserved;
+    this.#isWaitTraced = options.isWaitTraced ?? (() => false);
     this.#client = client ?? new TorrentWorkerClient({ ...options, onSourceForgotten: event => {
       this.forget(event.sourceKey);
       options.onSourceForgotten?.(event);
@@ -52,7 +54,7 @@ export class WorkerTorrentPool {
    * @param {string} source
    * @returns {Promise<object>}
    */
-  async getTorrent(sourceType, source) {
+  async getTorrent(sourceType, source, options = {}) {
     const startedAt = performance.now();
     const sourceKey = await deriveSourceKey(sourceType, source);
     const observed = outcome => this.#observed?.({ sourceKey, startedAt, endedAt: performance.now(), outcome });
@@ -63,7 +65,8 @@ export class WorkerTorrentPool {
     }
     const joined = this.#pending.get(sourceKey);
     if (joined) return joined.finally(() => observed("joined"));
-    const pending = this.#client.getTorrent({ sourceKey, sourceType, source }).then(torrent => {
+    const pending = this.#client.getTorrent({ sourceKey, sourceType, source,
+      trace: options.trace === true || this.#isWaitTraced(sourceKey) }).then(torrent => {
       if (this.#pending.get(sourceKey) !== pending) {
         const error = new Error("Source was forgotten while its metadata was being read.");
         error.code = "SOURCE_FORGOTTEN";

package/services/torrent/worker/protocol.js

@@ -152,6 +152,8 @@ export const Event = {
   CHUNK_ACK: "chunk-ack",
   /** A log line, so worker output reaches the same place as everything else. */
   LOG: "log",
+  /** A structured process observation from the torrent thread. */
+  WAIT_TRACE: "wait-trace",
   /**
    * Pieces of these files have arrived.
    *

package/services/torrent/worker/worker.js

@@ -92,6 +92,13 @@ const pool = new TorrentPool({
 const torrentsByKey = new Map();
 /** Resolved handles for reads that must never wait for or revive a source. */
 const resolvedTorrents = new Map();
+const waitTraceSources = new Set();
+
+function emitWaitTrace(sourceKey, process, phase, fields = {}) {
+  if (!waitTraceSources.has(sourceKey)) return;
+  parentPort.postMessage({ type: Event.WAIT_TRACE, version: 1, clock: "torrent-worker", sourceKey,
+    process, phase, at: performance.now(), ...fields });
+}
 
 /** In-flight reads, so a cancel can stop one mid-body. */
 const readsById = new Map();
@@ -327,6 +334,7 @@ async function streamRange({ id, sourceKey, fileIndex, start, end }) {
 async function runCommand(command, params, id) {
   switch (command) {
     case Command.ADD_SOURCE: {
+      if (params.trace === true) waitTraceSources.add(params.sourceKey);
       const retained = wholeSources.describe(params.sourceKey);
       if (retained) {
         announceWholeFiles(params.sourceKey, retained);
@@ -338,7 +346,10 @@ async function runCommand(command, params, id) {
       // racing to open the same torrent into one.
       let pending = torrentsByKey.get(params.sourceKey);
       if (!pending) {
-        pending = pool.getTorrent(params.sourceType, params.source);
+        emitWaitTrace(params.sourceKey, "W1", "event", { moment: "add-request" });
+        pending = pool.getTorrent(params.sourceType, params.source, {
+          sourceKey: params.sourceKey, trace: params.trace === true, emitWaitTrace
+        });
         torrentsByKey.set(params.sourceKey, pending);
         // A failed add must not be remembered, or every later attempt at this
         // source replays the same failure. The handler also marks the rejection
@@ -463,9 +474,14 @@ async function runCommand(command, params, id) {
 
 
     case Command.FILE_STATS: {
+      const statsStartedAt = performance.now();
+      emitWaitTrace(params.sourceKey, "W10", "begin", { startedAt: statsStartedAt,
+        fileIndex: params.fileIndex, forecast: params.forecast === true });
       const torrent = await knownTorrent(params.sourceKey);
       if (!torrent) {
         const whole = wholeSources.fileOf(params.sourceKey, params.fileIndex);
+        emitWaitTrace(params.sourceKey, "W10", "end", { startedAt: statsStartedAt,
+          endedAt: performance.now(), outcome: whole ? "whole-file" : "pending", fileIndex: params.fileIndex });
         return whole ? wholeFileStats(whole.length) : { pending: true };
       }
       // What this file's own interruptions demand, measured by the reader in
@@ -485,6 +501,9 @@ async function runCommand(command, params, id) {
         const end = Math.min(torrent.length, (piece + 1) * torrent.pieceLength);
         if (end > start) ranges.push({ start, end, availableAt });
       }
+      emitWaitTrace(params.sourceKey, "W10", "end", { startedAt: statsStartedAt,
+        endedAt: performance.now(), outcome: "ok", fileIndex: params.fileIndex,
+        forecast: params.forecast === true });
       return {
         ...stats,
         ...(params.forecast === true ? { downloadForecast: {
@@ -514,9 +533,12 @@ async function runCommand(command, params, id) {
       // zone becomes a window of the register and the swarm is told again
       // (torrent-tv/meta#166).
       const startedAt = performance.now();
+      emitWaitTrace(params.sourceKey, "W7", "begin", { fileIndex: params.fileIndex });
       ensureArrivalsWired(params.sourceKey, torrent);
       pool.applyPriorityMap(torrent, params.fileIndex, zones, params.durationSeconds);
       torrent.emit("priority-map-changed", params.fileIndex);
+      emitWaitTrace(params.sourceKey, "W7", "end", { startedAt, endedAt: performance.now(),
+        fileIndex: params.fileIndex });
       return { appliedMs: performance.now() - startedAt, receivedAt, repliedAt: performance.timeOrigin + performance.now() };
     }
 
@@ -918,6 +940,8 @@ function describePieceBuffers() {
  * @returns {void}
  */
 function announceArrivals(sourceKey, torrent, reason = "verified", piece = null) {
+  if (Number.isSafeInteger(piece)) emitWaitTrace(sourceKey, "W9", "event", { piece, reason,
+    bytes: bytesOfPiece(torrent, piece) });
   const fileIndexes = [...demandFor(torrent).register.files()];
   if (fileIndexes.length === 0) {
     return;

package/services/viewer/DownloadMaps.js

@@ -20,9 +20,10 @@ export class DownloadMaps {
   #clock;
   /** What the torrent is told of the playback zones: as many windows as say the same thing. */
   #pack;
+  #observeWait;
 
   constructor({ publish, resolvePlayback = async map => map.zones, log = null, now = Date.now,
-    clock = () => performance.timeOrigin + performance.now(), pack = zones => zones }) {
+    clock = () => performance.timeOrigin + performance.now(), pack = zones => zones, observeWait = null }) {
     if (typeof publish !== "function") throw new TypeError("Download map publication is required.");
     this.#publish = publish;
     this.#resolvePlayback = resolvePlayback;
@@ -30,6 +31,7 @@ export class DownloadMaps {
     this.#log = typeof log === "function" ? log : null;
     this.#now = now;
     this.#clock = clock;
+    this.#observeWait = typeof observeWait === "function" ? observeWait : null;
   }
 
   async playback({ sourceKey, fileIndex, durationSeconds, zones }) {
@@ -117,8 +119,23 @@ export class DownloadMaps {
       this.#setZones(file, withRetained(zones, retained));
       this.#emit(file).catch(() => undefined);
     };
-    const resolved = await this.#resolvePlayback({ sourceKey, fileIndex, durationSeconds, zones: file.sourceZones,
-      isCurrent, workingOut });
+    const resolverStartedAt = performance.now();
+    this.#observeWait?.(sourceKey, "process", { process: "P4", phase: "begin", fileIndex,
+      revision, zones: file.sourceZones.length, startedAt: resolverStartedAt });
+    let resolved;
+    let resolverOutcome = "ok";
+    try {
+      resolved = await this.#resolvePlayback({ sourceKey, fileIndex, durationSeconds, zones: file.sourceZones,
+        isCurrent, workingOut });
+    } catch (error) {
+      resolverOutcome = "error";
+      throw error;
+    } finally {
+      const resolverEndedAt = performance.now();
+      this.#observeWait?.(sourceKey, "process", { process: "P4", phase: "end", fileIndex, revision,
+        startedAt: resolverStartedAt, endedAt: resolverEndedAt, durationMs: resolverEndedAt - resolverStartedAt,
+        outcome: resolverOutcome, resolvedZones: Array.isArray(resolved) ? resolved.length : null });
+    }
     if (revision !== file.revision) return;
     this.#setZones(file, resolved);
     return this.#emit(file);

package/utils/wait-trace.js

@@ -1,4 +1,6 @@
 /** Bounded, opt-in startup observations. No torrent or application imports. */
+const SOURCE_CONTEXT_TTL_MS = 30 * 60_000;
+
 export class SourceWaitTrace {
   constructor({ emit, now = () => performance.now() }) {
     Object.assign(this, { emit, now, sources: new Map(), requests: new WeakMap() });
@@ -9,16 +11,34 @@ export class SourceWaitTrace {
       operation: context.operation, event, at: this.now(), ...fields });
   }
 
+  process(sourceKey, process, phase, fields = {}) {
+    const context = this.sources.get(sourceKey);
+    if (!context || this.now() - context.seen > SOURCE_CONTEXT_TTL_MS) return;
+    this.write({ ...context, operation: null }, "process", { sourceKey, process, phase, ...fields });
+  }
+
+  has(sourceKey) {
+    const context = this.sources.get(sourceKey);
+    return Boolean(context && this.now() - context.seen <= SOURCE_CONTEXT_TTL_MS);
+  }
+
+  associate(request, sourceKey) {
+    const context = this.requests.get(request);
+    if (!context || typeof sourceKey !== "string" || !sourceKey) return false;
+    this.remember(sourceKey, context);
+    return true;
+  }
+
   remember(sourceKey, context) {
     if (typeof sourceKey !== "string" || !sourceKey) return;
-    for (const [key, value] of this.sources) if (this.now() - value.seen > 60_000) this.sources.delete(key);
+    for (const [key, value] of this.sources) if (this.now() - value.seen > SOURCE_CONTEXT_TTL_MS) this.sources.delete(key);
     if (this.sources.size >= 64 && !this.sources.has(sourceKey)) this.sources.delete(this.sources.keys().next().value);
     this.sources.set(sourceKey, { ...context, seen: this.now() });
   }
 
   source(sourceKey, event, fields = {}) {
     const context = this.sources.get(sourceKey);
-    if (!context || this.now() - context.seen > 60_000) return;
+    if (!context || this.now() - context.seen > SOURCE_CONTEXT_TTL_MS) return;
     this.write({ ...context, operation: null }, event, { sourceKey, ...fields });
   }
 
@@ -26,19 +46,23 @@ export class SourceWaitTrace {
     app.addHook("onRequest", async (req, reply) => {
       const wait = req.headers["x-ttv-wait-trace"];
       const operation = req.headers["x-ttv-wait-operation"];
+      const statedProcess = req.headers["x-ttv-wait-process"];
       const attempt = Number(req.headers["x-ttv-wait-attempt"]);
       if (typeof wait !== "string" || !/^[a-zA-Z0-9.-]{1,80}$/.test(wait) ||
         typeof operation !== "string" || !/^b[0-9]{1,12}$/.test(operation) ||
         !Number.isSafeInteger(attempt) || attempt < 0) return;
       const context = { wait, attempt, operation, startedAt: this.now() };
       this.requests.set(req, context);
-      this.write(context, "http-start", { method: req.method, path: req.url,
+      const process = typeof statedProcess === "string" && /^[A-Z][0-9]+(?:\.(?:[0-9]+[a-z]?|[va]))?$/.test(statedProcess)
+        ? statedProcess : processForRequest(req.url);
+      this.write(context, "http-start", { process, phase: "begin", method: req.method, path: req.url,
         requestId: req.headers["x-ttv-forwarded-request-id"] ?? req.id });
       let finished = false;
       const end = outcome => {
         if (finished) return;
         finished = true;
-        this.write(context, "http-end", { outcome, status: reply.statusCode, duration: this.now() - context.startedAt });
+        this.write(context, "http-end", { process, phase: "end", outcome, status: reply.statusCode,
+          startedAt: context.startedAt, endedAt: this.now(), duration: this.now() - context.startedAt });
       };
       reply.raw.once("finish", () => end("finished"));
       reply.raw.once("close", () => end(reply.raw.writableFinished ? "finished" : "closed"));
@@ -66,3 +90,13 @@ export class SourceWaitTrace {
     });
   }
 }
+
+function processForRequest(path) {
+  if (path.startsWith("/api/transcode-sessions")) return "P6.v";
+  if (path.startsWith("/api/playback-plan")) return "P3";
+  if (/\/transcode\/.*\.m3u8(?:\?|$)/.test(path)) return "P12";
+  if (/\/transcode\/.*(?:init|\.mp4)(?:\?|$)/.test(path)) return "P12";
+  if (/\/transcode\/.*(?:\.m4s|\.ts)(?:\?|$)/.test(path)) return "P12";
+  if (path.startsWith("/api/session-progress")) return "P14";
+  return null;
+}