@hyperframes/producer 0.8.203 → 0.8.204

package/dist/distributed.js

@@ -1335,6 +1335,12 @@ import {
   prepareCaptureSessionForReuse,
   spawnStreamingEncoder
 } from "@hyperframes/engine";
+var DEFAULT_CAPTURE_STALL_MS = 6e4;
+function resolveCaptureStallTimeoutMs() {
+  const raw = process.env.HF_DE_STALL_MS ?? process.env.HF_DE_PARALLEL_STALL_MS;
+  const parsed = raw ? Number(raw) : Number.NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CAPTURE_STALL_MS;
+}
 
 // src/services/render/segmentManifest.ts
 import { renameWithRetry as renameWithRetry2 } from "@hyperframes/engine";
@@ -1442,9 +1448,325 @@ async function measureCaptureCostFromSession(session, totalFrames, fps, log) {
 
 // src/services/render/observability.ts
 import { redactTelemetryString } from "@hyperframes/core";
+import { CaptureFailure as CaptureFailure2 } from "@hyperframes/engine";
+var MAX_EVENTS = 160;
+var ALLOWED_STRING_DATA_KEYS = /* @__PURE__ */ new Set([
+  "browserGpuMode",
+  "captureMode",
+  "captureOperation",
+  "compositionHash",
+  "effectiveHdr",
+  "format",
+  "quality",
+  "renderJobId",
+  "requestedHdrMode",
+  "requestedWorkers",
+  // "calibrating" during pre-capture browser warm-up / calibration stages;
+  // "capturing" during capture_disk / capture_streaming / capture_hdr_*
+  // stages. Distinguishes healthy 0-frame heartbeats (browser starting up)
+  // from actual zero-frame stalls once capture is meant to be underway.
+  // Field signal ts=1784019503.
+  "stagePhase",
+  // Full-fidelity fast-capture fallback trigger — the specific reason
+  // drawElement gated off ("filter:blur", "filter:drop-shadow",
+  // "backdrop-filter", "clip-path", "at_risk_timeline", "swiftshader", …).
+  // Populated on the `capture_fallback_profile` checkpoint emitted by
+  // `fallbackCaptureProfile.ts` when the operator opts into
+  // `HF_PROFILE_FALLBACK_CAPTURE=true` so downstream metric consumers can
+  // characterize per-frame perf by the exact trigger. Complementary to
+  // `deGateReason`, which is the low-cardinality bucket for aggregation —
+  // this string preserves the fine-grained "which CSS property" for
+  // diagnostic reads. Sanitized like any observation message; low-cardinality
+  // by construction (bounded by the fallback-trigger enumeration).
+  "triggerReason"
+]);
+var RESERVED_LOG_KEYS = /* @__PURE__ */ new Set([
+  "data",
+  "durationMs",
+  "elapsedMs",
+  "message",
+  "phase",
+  "renderJobId",
+  "status"
+]);
+function sanitizeObservationMessage(value) {
+  return redactTelemetryString(value);
+}
+function sanitizeObservationData(data) {
+  if (!data) return void 0;
+  const sanitized = {};
+  for (const [key, value] of Object.entries(data)) {
+    if (RESERVED_LOG_KEYS.has(key)) continue;
+    if (typeof value === "string") {
+      if (!ALLOWED_STRING_DATA_KEYS.has(key)) continue;
+      sanitized[key] = sanitizeObservationMessage(value);
+    } else {
+      sanitized[key] = value;
+    }
+  }
+  return sanitized;
+}
+function isHttpErrorDiagnostic(line) {
+  return /\[Browser:HTTP\d{3}\]/.test(line);
+}
+function readUnsignedIntAfter(line, prefix) {
+  const start = line.indexOf(prefix);
+  if (start < 0) return void 0;
+  let value = 0;
+  let digits = 0;
+  for (let i = start + prefix.length; i < line.length; i++) {
+    const code = line.charCodeAt(i);
+    if (code < 48 || code > 57) break;
+    value = value * 10 + code - 48;
+    digits += 1;
+    if (value > Number.MAX_SAFE_INTEGER) return void 0;
+  }
+  return digits > 0 ? value : void 0;
+}
+function maxReading(current, next) {
+  if (next === void 0) return current;
+  return current === void 0 ? next : Math.max(current, next);
+}
+function summarizeInitObservability(lines, fallback) {
+  let initDurationMs = fallback?.initDurationMs;
+  let tweenCount = fallback?.tweenCount;
+  let elementCount = fallback?.elementCount;
+  for (const line of lines) {
+    if (!line.includes("[FrameCapture:INIT]")) continue;
+    initDurationMs = maxReading(initDurationMs, readUnsignedIntAfter(line, "initDurationMs="));
+    tweenCount = maxReading(tweenCount, readUnsignedIntAfter(line, "tweenCount="));
+    elementCount = maxReading(elementCount, readUnsignedIntAfter(line, "elementCount="));
+  }
+  if (initDurationMs === void 0 && tweenCount === void 0 && elementCount === void 0) {
+    return void 0;
+  }
+  return { initDurationMs, tweenCount, elementCount };
+}
+function summarizeBrowserDiagnostics(lines) {
+  let errors = 0;
+  let pageErrors = 0;
+  let requestFailed = 0;
+  let httpErrors = 0;
+  let navigationStarts = 0;
+  let navigationFailures = 0;
+  let consoleErrors = 0;
+  let consoleWarnings = 0;
+  for (const line of lines) {
+    const isPageError = line.includes("PAGEERROR");
+    const isRequestFailed = line.includes("REQUESTFAILED");
+    const isHttpError = isHttpErrorDiagnostic(line);
+    const isNavigationFailure = line.includes("[FrameCapture:ERROR] page.goto failed");
+    const isConsoleError = line.includes("[error]");
+    if (isPageError) pageErrors++;
+    if (isRequestFailed) requestFailed++;
+    if (isHttpError) httpErrors++;
+    if (line.includes("[FrameCapture:NAV] page.goto start")) navigationStarts++;
+    if (isNavigationFailure) navigationFailures++;
+    if (isConsoleError) consoleErrors++;
+    if (line.includes("[warn]")) consoleWarnings++;
+    if (line.includes("ERROR") && !isPageError && !isRequestFailed && !isHttpError && !isNavigationFailure && !isConsoleError) {
+      errors++;
+    }
+  }
+  return {
+    total: lines.length,
+    errors,
+    pageErrors,
+    requestFailed,
+    httpErrors,
+    navigationStarts,
+    navigationFailures,
+    consoleErrors,
+    consoleWarnings
+  };
+}
+var RenderObservabilityRecorder = class {
+  constructor(input) {
+    this.input = input;
+  }
+  input;
+  events = [];
+  eventCount = 0;
+  failedPhase;
+  checkpoint(phase, message, data) {
+    return this.record({
+      phase,
+      status: "checkpoint",
+      elapsedMs: Date.now() - this.input.pipelineStartMs,
+      message: sanitizeObservationMessage(message),
+      data: sanitizeObservationData(data)
+    });
+  }
+  stageStart(phase, data) {
+    this.record({
+      phase,
+      status: "start",
+      elapsedMs: Date.now() - this.input.pipelineStartMs,
+      data: sanitizeObservationData(data)
+    });
+    return Date.now();
+  }
+  stageEnd(phase, startedAtMs, data) {
+    this.record({
+      phase,
+      status: "end",
+      elapsedMs: Date.now() - this.input.pipelineStartMs,
+      durationMs: Date.now() - startedAtMs,
+      data: sanitizeObservationData(data)
+    });
+  }
+  stageError(phase, startedAtMs, error, data) {
+    this.failedPhase = phase;
+    this.record({
+      phase,
+      status: "error",
+      elapsedMs: Date.now() - this.input.pipelineStartMs,
+      durationMs: Date.now() - startedAtMs,
+      message: sanitizeObservationMessage(normalizeErrorMessage(error)),
+      data: sanitizeObservationData(data)
+    });
+  }
+  summary(input) {
+    const lastEvent = this.events[this.events.length - 1];
+    return {
+      renderJobId: this.input.renderJobId,
+      compositionHash: input.compositionHash,
+      events: this.events.slice(),
+      eventCount: this.eventCount,
+      lastEvent,
+      failedPhase: this.failedPhase,
+      browserDiagnostics: summarizeBrowserDiagnostics(input.lastBrowserConsole),
+      capture: { ...input.capture },
+      extraction: input.extraction ? { ...input.extraction } : void 0,
+      init: summarizeInitObservability(input.lastBrowserConsole, input.initFallback)
+    };
+  }
+  hasFailure() {
+    return this.failedPhase !== void 0;
+  }
+  /** A phase failure that was subsequently recovered (e.g. the drawElement
+   * self-verify fallback re-rendering via screenshot) should not brand the
+   * whole render as failed in the summary. */
+  clearFailure(phase) {
+    if (this.failedPhase === phase) this.failedPhase = void 0;
+  }
+  record(event) {
+    this.eventCount++;
+    const eventWithJob = { ...event, renderJobId: this.input.renderJobId };
+    this.events.push(eventWithJob);
+    if (this.events.length > MAX_EVENTS) {
+      this.events.shift();
+    }
+    this.input.log.info("[Render:trace]", {
+      renderJobId: eventWithJob.renderJobId,
+      phase: eventWithJob.phase,
+      status: eventWithJob.status,
+      elapsedMs: eventWithJob.elapsedMs,
+      durationMs: eventWithJob.durationMs,
+      message: eventWithJob.message,
+      ...eventWithJob.data
+    });
+    return eventWithJob;
+  }
+};
 var HEARTBEAT_RAMP_MS = [3e4, 6e4, 12e4];
 var HEARTBEAT_REPEAT_MS = 12e4;
 var HEARTBEAT_RAMP_END_MS = HEARTBEAT_RAMP_MS[HEARTBEAT_RAMP_MS.length - 1] ?? HEARTBEAT_REPEAT_MS;
+function heartbeatTargetMs(index) {
+  const rampTarget = HEARTBEAT_RAMP_MS[index];
+  if (rampTarget !== void 0) return rampTarget;
+  const overflow = index - HEARTBEAT_RAMP_MS.length + 1;
+  return HEARTBEAT_RAMP_END_MS + overflow * HEARTBEAT_REPEAT_MS;
+}
+var RenderStageStalledError = class extends CaptureFailure2 {
+  code = "RENDER_STAGE_STALLED";
+  owner = "system";
+  retryable = false;
+  constructor(phase, silentMs, progress) {
+    super({
+      kind: "protocol_timeout",
+      message: `[Render] ${phase} stalled: no progress for ${silentMs}ms (stuck at ${progress}).`
+    });
+    this.name = "RenderStageStalledError";
+  }
+};
+var STALL_POLL_MS = 5e3;
+function watchStall(phase, rule) {
+  let last = rule.progress();
+  let budgetMs = rule.firstProgressMs;
+  let lastChangeAt = Date.now();
+  let attempt;
+  let timer;
+  const restartWindow = (ms) => {
+    last = rule.progress();
+    budgetMs = ms;
+    lastChangeAt = Date.now();
+  };
+  const tripped = new Promise((_, reject) => {
+    timer = setInterval(
+      () => {
+        const current = rule.progress();
+        if (current !== last) return restartWindow(rule.noProgressMs);
+        if (Date.now() - lastChangeAt < budgetMs) return;
+        const error = new RenderStageStalledError(phase, budgetMs, current);
+        if (attempt && !attempt.signal.aborted) {
+          attempt.abort(error);
+          return restartWindow(rule.noProgressMs);
+        }
+        clearInterval(timer);
+        reject(error);
+      },
+      Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs)
+    );
+  });
+  const openAttempt = (parent) => {
+    attempt = new AbortController();
+    restartWindow(rule.firstProgressMs);
+    return parent ? AbortSignal.any([parent, attempt.signal]) : attempt.signal;
+  };
+  const stalledBy = () => attempt?.signal.aborted ? attempt.signal.reason : void 0;
+  return { tripped, openAttempt, stalledBy, stop: () => clearInterval(timer) };
+}
+var openUnwatchedAttempt = (parent) => parent ?? new AbortController().signal;
+async function observeRenderStage(recorder, phase, data, fn, options = {}) {
+  const heartbeatMessage = options.heartbeatMessage ?? "stage still running";
+  const startedAt = recorder.stageStart(phase, data);
+  let heartbeatCount = 0;
+  let lastFiredAtMs = 0;
+  let heartbeatTimer;
+  const scheduleNextHeartbeat = () => {
+    const targetMs = heartbeatTargetMs(heartbeatCount);
+    heartbeatTimer = setTimeout(() => {
+      lastFiredAtMs = targetMs;
+      heartbeatCount += 1;
+      recorder.checkpoint(phase, heartbeatMessage, {
+        ...data,
+        heartbeatIndex: heartbeatCount,
+        stageElapsedMs: Date.now() - startedAt
+      });
+      scheduleNextHeartbeat();
+    }, targetMs - lastFiredAtMs);
+    heartbeatTimer.unref?.();
+  };
+  scheduleNextHeartbeat();
+  const stall = options.stall ? watchStall(phase, options.stall) : void 0;
+  const clearHeartbeats = () => {
+    clearTimeout(heartbeatTimer);
+    stall?.stop();
+  };
+  try {
+    const run = fn(stall?.openAttempt ?? openUnwatchedAttempt);
+    const result = await (stall ? Promise.race([run, stall.tripped]) : run);
+    clearHeartbeats();
+    recorder.stageEnd(phase, startedAt, data);
+    return result;
+  } catch (caught) {
+    const error = stall?.stalledBy() ?? caught;
+    clearHeartbeats();
+    recorder.stageError(phase, startedAt, error, data);
+    throw error;
+  }
+}
 
 // src/services/render/videoFrameCoverage.ts
 import { parseHTML } from "linkedom";
@@ -3477,9 +3799,10 @@ async function discoverMediaFromBrowser(page) {
       const tagName = isImage ? "image" : htmlEl.tagName.toLowerCase() === "video" ? "video" : "audio";
       const id = htmlEl.getAttribute("data-hf-render-id") || htmlEl.id || (isImage ? autoImageIds.get(htmlEl) : void 0);
       if (!id) return;
-      const ownSrc = htmlEl.getAttribute("src");
-      const blankSrc = ownSrc !== null && ownSrc.trim() === "" && !htmlEl.querySelector("source[src]");
-      const src = blankSrc ? "" : htmlEl.currentSrc || htmlEl.src || ownSrc || "";
+      const resolvedSrc = htmlEl.currentSrc || htmlEl.src || htmlEl.getAttribute("src") || "";
+      const withoutQueryAndHash = (url) => url.replace(/[?#][\s\S]*$/, "");
+      const isPageItself = withoutQueryAndHash(resolvedSrc) === withoutQueryAndHash(location.href);
+      const src = isPageItself ? "" : resolvedSrc;
       const start = parseFloat(htmlEl.getAttribute("data-start") || "0");
       const endRaw = htmlEl.getAttribute("data-end");
       const durationRaw = htmlEl.getAttribute("data-duration");
@@ -5793,6 +6116,7 @@ async function runCaptureStage(input) {
       interleaveFrames: input.interleaveFrames,
       dedupPerfs,
       pageErrorLines: workerPageErrors,
+      openAttempt: input.openAttempt,
       onProgress: (progress) => {
         if (progress.latestWorkerPhase) {
           if (progress.capturedFrames === 0) reportWorkerStartup(job, progress, onProgress);
@@ -5864,6 +6188,11 @@ async function runSequentialDiskCapture(input, captureOptions, captureCfg, check
   let transientRetriesUsed = 0;
   let resumeFrom = 0;
   while (true) {
+    const attempt = input.openAttempt?.();
+    const assertAttemptActive = () => {
+      assertNotAborted();
+      attempt?.throwIfAborted();
+    };
     const videoInjector = createRenderVideoFrameInjector();
     const session = probeSession ?? await createCaptureSession5(
       fileServer.url,
@@ -5879,10 +6208,10 @@ async function runSequentialDiskCapture(input, captureOptions, captureCfg, check
         probeSession = null;
       }
       await ensureSessionReady(session);
-      assertNotAborted();
+      assertAttemptActive();
       lastBrowserConsole = session.browserConsoleBuffer;
       await captureSessionFrames(session, {
-        input,
+        input: { ...input, assertNotAborted: assertAttemptActive },
         rangeStart,
         rangeEnd,
         resumeFrom,
@@ -6566,7 +6895,17 @@ function isTransientCaptureRetryEligible(failureKind, missing, retriesUsed) {
 function captureAttemptMadeProgress(attemptTargetFrameCount, remainingFrameCount) {
   return remainingFrameCount < attemptTargetFrameCount;
 }
+var FIRST_FRAME_STALL_MS = 3e5;
+function frameStallRule(job) {
+  const noProgressMs = resolveCaptureStallTimeoutMs();
+  return {
+    progress: () => job.framesRendered ?? 0,
+    noProgressMs,
+    firstProgressMs: Math.max(FIRST_FRAME_STALL_MS, noProgressMs)
+  };
+}
 function isRecoverableParallelCaptureError(error) {
+  if (error instanceof RenderStageStalledError) return true;
   const message = normalizeErrorMessage(error);
   if (!message.includes("[Parallel] Capture failed")) return false;
   const kind = classifyCaptureFailure3(error).kind;
@@ -6618,8 +6957,10 @@ async function executeDiskCaptureWithAdaptiveRetry(options) {
       )
     ];
     options.dedupPerfs.length = 0;
+    let attemptSignal = options.abortSignal;
     try {
       for (const tasks of batches) {
+        attemptSignal = options.openAttempt?.(options.abortSignal) ?? options.abortSignal;
         const capturedBeforeBatch = countCapturedFrames(
           options.totalFrames,
           options.framesDir,
@@ -6632,7 +6973,7 @@ async function executeDiskCaptureWithAdaptiveRetry(options) {
             tasks,
             options.captureOptions,
             options.createBeforeCaptureHook,
-            options.abortSignal,
+            attemptSignal,
             options.onProgress ? (progress) => {
               options.onProgress?.({
                 ...progress,
@@ -6680,7 +7021,8 @@ async function executeDiskCaptureWithAdaptiveRetry(options) {
       currentWorkers = nextWorkers;
       missingRanges = remaining;
       attempt++;
-    } catch (error) {
+    } catch (caught) {
+      const error = attemptSignal?.aborted && !options.abortSignal?.aborted ? attemptSignal.reason : caught;
       const failure = classifyCaptureFailure3(error, { signal: options.abortSignal });
       if (failure.kind === "cancelled") {
         throw error;
@@ -11091,29 +11433,36 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath, options) {
           const captureSession = session;
           session = null;
           overlap.current = canEncodeWhileCapturing(encodeInput, chunkWorkerCount, cpus().length) ? encodeWhileCapturing(encodeInput, framesInChunk) : null;
-          await runCaptureStage({
-            fileServer,
-            workDir,
-            framesDir,
-            job,
-            totalFrames: framesInChunk,
-            cfg: captureCfg,
-            plan: capturePlan,
-            log,
-            probeSession: captureSession,
-            captureAttempts,
-            // This sink records each worker's effective capture mode so a
-            // fallback remains observable to adapters and smoke tests.
-            dedupPerfs: capturePerfs,
-            buildCaptureOptions: () => captureOptions,
-            createRenderVideoFrameInjector: createChunkVideoFrameInjector,
-            abortSignal: void 0,
-            assertNotAborted: () => {
-            },
-            frameRange: range,
-            onFrameLanded: (frame) => overlap.current?.onFrameLanded(frame),
-            interleaveFrames: interleavesCapture(overlap.current !== null, options)
-          }).catch(async (captureError) => {
+          await observeRenderStage(
+            new RenderObservabilityRecorder({ pipelineStartMs: start, log }),
+            "capture_chunk",
+            { chunkIndex },
+            (openAttempt) => runCaptureStage({
+              fileServer,
+              workDir,
+              framesDir,
+              job,
+              totalFrames: framesInChunk,
+              cfg: captureCfg,
+              plan: capturePlan,
+              log,
+              probeSession: captureSession,
+              captureAttempts,
+              // This sink records each worker's effective capture mode so a
+              // fallback remains observable to adapters and smoke tests.
+              dedupPerfs: capturePerfs,
+              buildCaptureOptions: () => captureOptions,
+              createRenderVideoFrameInjector: createChunkVideoFrameInjector,
+              abortSignal: void 0,
+              assertNotAborted: () => {
+              },
+              frameRange: range,
+              onFrameLanded: (frame) => overlap.current?.onFrameLanded(frame),
+              interleaveFrames: interleavesCapture(overlap.current !== null, options),
+              openAttempt
+            }),
+            { stall: frameStallRule(job) }
+          ).catch(async (captureError) => {
             await overlap.current?.abort();
             overlap.current = null;
             throw captureError;

package/dist/hyperframe.manifest.json

@@ -1,7 +1,7 @@
 {
   "version": "0.1.0",
   "buildId": "dev",
-  "sha256": "93402c78fff7cdcf247d971840896d2b0d3380fa1d0901f31618ba6f62160667",
+  "sha256": "7d7e26ef5f6c62ee5fd433020813c7253231a9ddaf89a3bcde67fd2d0ed33c66",
   "artifacts": {
     "iife": "hyperframe.runtime.iife.js",
     "esm": "hyperframe.runtime.mjs"