@hyperframes/producer 0.8.203 → 0.8.204

package/dist/index.js

@@ -1238,6 +1238,7 @@ function buildRenderErrorDetails(input) {
 
 // src/services/render/renderEventPublisher.ts
 import { cloneCaptureWarnings } from "@hyperframes/engine";
+var FLUSH_TIMEOUT_MS = 1e4;
 function snapshotJob(job) {
   return {
     ...job,
@@ -1267,7 +1268,15 @@ var OrderedRenderEventPublisher = class {
     });
   }
   async flush() {
-    await this.tail;
+    let timer;
+    const drained = await Promise.race([
+      this.tail.then(() => true),
+      new Promise((resolve12) => {
+        timer = setTimeout(() => resolve12(false), FLUSH_TIMEOUT_MS);
+      })
+    ]);
+    clearTimeout(timer);
+    if (!drained) this.log.warn("Render event sink did not drain; continuing without it");
   }
 };
 function publishRenderFailure(job, failure, onProgress) {
@@ -5100,6 +5109,7 @@ function shouldFallbackToScreenshotAfterCalibrationError(error) {
 // src/services/render/observability.ts
 import { createHash as createHash3 } from "node:crypto";
 import { redactTelemetryString } from "@hyperframes/core";
+import { CaptureFailure as CaptureFailure2 } from "@hyperframes/engine";
 var MAX_EVENTS = 160;
 var ALLOWED_STRING_DATA_KEYS = /* @__PURE__ */ new Set([
   "browserGpuMode",
@@ -5332,6 +5342,56 @@ function heartbeatTargetMs(index) {
   const overflow = index - HEARTBEAT_RAMP_MS.length + 1;
   return HEARTBEAT_RAMP_END_MS + overflow * HEARTBEAT_REPEAT_MS;
 }
+var RenderStageStalledError = class extends CaptureFailure2 {
+  code = "RENDER_STAGE_STALLED";
+  owner = "system";
+  retryable = false;
+  constructor(phase, silentMs, progress) {
+    super({
+      kind: "protocol_timeout",
+      message: `[Render] ${phase} stalled: no progress for ${silentMs}ms (stuck at ${progress}).`
+    });
+    this.name = "RenderStageStalledError";
+  }
+};
+var STALL_POLL_MS = 5e3;
+function watchStall(phase, rule) {
+  let last = rule.progress();
+  let budgetMs = rule.firstProgressMs;
+  let lastChangeAt = Date.now();
+  let attempt;
+  let timer;
+  const restartWindow = (ms) => {
+    last = rule.progress();
+    budgetMs = ms;
+    lastChangeAt = Date.now();
+  };
+  const tripped = new Promise((_, reject) => {
+    timer = setInterval(
+      () => {
+        const current = rule.progress();
+        if (current !== last) return restartWindow(rule.noProgressMs);
+        if (Date.now() - lastChangeAt < budgetMs) return;
+        const error = new RenderStageStalledError(phase, budgetMs, current);
+        if (attempt && !attempt.signal.aborted) {
+          attempt.abort(error);
+          return restartWindow(rule.noProgressMs);
+        }
+        clearInterval(timer);
+        reject(error);
+      },
+      Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs)
+    );
+  });
+  const openAttempt = (parent) => {
+    attempt = new AbortController();
+    restartWindow(rule.firstProgressMs);
+    return parent ? AbortSignal.any([parent, attempt.signal]) : attempt.signal;
+  };
+  const stalledBy = () => attempt?.signal.aborted ? attempt.signal.reason : void 0;
+  return { tripped, openAttempt, stalledBy, stop: () => clearInterval(timer) };
+}
+var openUnwatchedAttempt = (parent) => parent ?? new AbortController().signal;
 async function observeRenderStage(recorder, phase, data, fn, options = {}) {
   const heartbeatMessage = options.heartbeatMessage ?? "stage still running";
   const startedAt = recorder.stageStart(phase, data);
@@ -5353,15 +5413,19 @@ async function observeRenderStage(recorder, phase, data, fn, options = {}) {
     heartbeatTimer.unref?.();
   };
   scheduleNextHeartbeat();
+  const stall = options.stall ? watchStall(phase, options.stall) : void 0;
   const clearHeartbeats = () => {
     clearTimeout(heartbeatTimer);
+    stall?.stop();
   };
   try {
-    const result = await fn();
+    const run = fn(stall?.openAttempt ?? openUnwatchedAttempt);
+    const result = await (stall ? Promise.race([run, stall.tripped]) : run);
     clearHeartbeats();
     recorder.stageEnd(phase, startedAt, data);
     return result;
-  } catch (error) {
+  } catch (caught) {
+    const error = stall?.stalledBy() ?? caught;
     clearHeartbeats();
     recorder.stageError(phase, startedAt, error, data);
     throw error;
@@ -7516,9 +7580,10 @@ async function discoverMediaFromBrowser(page) {
       const tagName = isImage ? "image" : htmlEl.tagName.toLowerCase() === "video" ? "video" : "audio";
       const id = htmlEl.getAttribute("data-hf-render-id") || htmlEl.id || (isImage ? autoImageIds.get(htmlEl) : void 0);
       if (!id) return;
-      const ownSrc = htmlEl.getAttribute("src");
-      const blankSrc = ownSrc !== null && ownSrc.trim() === "" && !htmlEl.querySelector("source[src]");
-      const src = blankSrc ? "" : htmlEl.currentSrc || htmlEl.src || ownSrc || "";
+      const resolvedSrc = htmlEl.currentSrc || htmlEl.src || htmlEl.getAttribute("src") || "";
+      const withoutQueryAndHash = (url) => url.replace(/[?#][\s\S]*$/, "");
+      const isPageItself = withoutQueryAndHash(resolvedSrc) === withoutQueryAndHash(location.href);
+      const src = isPageItself ? "" : resolvedSrc;
       const start = parseFloat(htmlEl.getAttribute("data-start") || "0");
       const endRaw = htmlEl.getAttribute("data-end");
       const durationRaw = htmlEl.getAttribute("data-duration");
@@ -9824,6 +9889,7 @@ async function runCaptureStage(input) {
       interleaveFrames: input.interleaveFrames,
       dedupPerfs,
       pageErrorLines: workerPageErrors,
+      openAttempt: input.openAttempt,
       onProgress: (progress) => {
         if (progress.latestWorkerPhase) {
           if (progress.capturedFrames === 0) reportWorkerStartup(job, progress, onProgress);
@@ -9895,6 +9961,11 @@ async function runSequentialDiskCapture(input, captureOptions, captureCfg, check
   let transientRetriesUsed = 0;
   let resumeFrom = 0;
   while (true) {
+    const attempt = input.openAttempt?.();
+    const assertAttemptActive = () => {
+      assertNotAborted();
+      attempt?.throwIfAborted();
+    };
     const videoInjector = createRenderVideoFrameInjector();
     const session = probeSession ?? await createCaptureSession5(
       fileServer.url,
@@ -9910,10 +9981,10 @@ async function runSequentialDiskCapture(input, captureOptions, captureCfg, check
         probeSession = null;
       }
       await ensureSessionReady(session);
-      assertNotAborted();
+      assertAttemptActive();
       lastBrowserConsole = session.browserConsoleBuffer;
       await captureSessionFrames(session, {
-        input,
+        input: { ...input, assertNotAborted: assertAttemptActive },
         rangeStart,
         rangeEnd,
         resumeFrom,
@@ -11773,7 +11844,17 @@ function captureAttemptMadeProgress(attemptTargetFrameCount, remainingFrameCount
 function resetCaptureAttemptProgress(job) {
   job.framesRendered = 0;
 }
+var FIRST_FRAME_STALL_MS = 3e5;
+function frameStallRule(job) {
+  const noProgressMs = resolveCaptureStallTimeoutMs();
+  return {
+    progress: () => job.framesRendered ?? 0,
+    noProgressMs,
+    firstProgressMs: Math.max(FIRST_FRAME_STALL_MS, noProgressMs)
+  };
+}
 function isRecoverableParallelCaptureError(error) {
+  if (error instanceof RenderStageStalledError) return true;
   const message = normalizeErrorMessage(error);
   if (!message.includes("[Parallel] Capture failed")) return false;
   const kind = classifyCaptureFailure3(error).kind;
@@ -11836,8 +11917,10 @@ async function executeDiskCaptureWithAdaptiveRetry(options) {
       )
     ];
     options.dedupPerfs.length = 0;
+    let attemptSignal = options.abortSignal;
     try {
       for (const tasks of batches) {
+        attemptSignal = options.openAttempt?.(options.abortSignal) ?? options.abortSignal;
         const capturedBeforeBatch = countCapturedFrames(
           options.totalFrames,
           options.framesDir,
@@ -11850,7 +11933,7 @@ async function executeDiskCaptureWithAdaptiveRetry(options) {
             tasks,
             options.captureOptions,
             options.createBeforeCaptureHook,
-            options.abortSignal,
+            attemptSignal,
             options.onProgress ? (progress) => {
               options.onProgress?.({
                 ...progress,
@@ -11898,7 +11981,8 @@ async function executeDiskCaptureWithAdaptiveRetry(options) {
       currentWorkers = nextWorkers;
       missingRanges = remaining;
       attempt++;
-    } catch (error) {
+    } catch (caught) {
+      const error = attemptSignal?.aborted && !options.abortSignal?.aborted ? attemptSignal.reason : caught;
       const failure = classifyCaptureFailure3(error, { signal: options.abortSignal });
       if (failure.kind === "cancelled") {
         throw error;
@@ -13567,37 +13651,44 @@ async function executeRenderPipeline(input) {
         observability,
         "capture_hdr_layered",
         captureStageObservationData({ hasHdrContent }),
-        () => runCaptureHdrStage({
-          job,
-          cfg,
-          plan: layeredPlan,
-          log,
-          projectDir,
-          compiledDir,
-          framesDir,
-          videoOnlyPath,
-          width,
-          height,
-          totalFrames,
-          composition,
-          hasHdrContent,
-          effectiveHdr,
-          nativeHdrVideoIds,
-          nativeHdrImageIds,
-          videoTransfers,
-          imageTransfers,
-          hdrImageSrcPaths,
-          preset,
-          effectiveQuality,
-          effectiveBitrate,
-          fileServer: activeFileServer,
-          buildCaptureOptions,
-          createRenderVideoFrameInjector,
-          hdrDiagnostics,
-          abortSignal: executionSignal,
-          assertNotAborted,
-          onProgress
-        })
+        (openAttempt) => {
+          const attempt = openAttempt(executionSignal);
+          return runCaptureHdrStage({
+            job,
+            cfg,
+            plan: layeredPlan,
+            log,
+            projectDir,
+            compiledDir,
+            framesDir,
+            videoOnlyPath,
+            width,
+            height,
+            totalFrames,
+            composition,
+            hasHdrContent,
+            effectiveHdr,
+            nativeHdrVideoIds,
+            nativeHdrImageIds,
+            videoTransfers,
+            imageTransfers,
+            hdrImageSrcPaths,
+            preset,
+            effectiveQuality,
+            effectiveBitrate,
+            fileServer: activeFileServer,
+            buildCaptureOptions,
+            createRenderVideoFrameInjector,
+            hdrDiagnostics,
+            abortSignal: attempt,
+            assertNotAborted: () => {
+              assertNotAborted();
+              attempt.throwIfAborted();
+            },
+            onProgress
+          });
+        },
+        { stall: frameStallRule(job) }
       );
       lastBrowserConsole = hdrRes.lastBrowserConsole;
       layeredCaptureWarnings.push(...hdrRes.warnings);
@@ -13962,7 +14053,7 @@ async function executeRenderPipeline(input) {
           observability,
           "capture_disk",
           captureStageObservationData({ needsAlpha: diskPlan.needsAlpha }),
-          () => runCaptureStage({
+          (openAttempt) => runCaptureStage({
             fileServer: activeFileServer,
             workDir,
             framesDir,
@@ -13980,8 +14071,10 @@ async function executeRenderPipeline(input) {
             assertNotAborted,
             onProgress,
             onFrameLanded: (frame) => overlap?.onFrameLanded(frame),
-            interleaveFrames: overlap !== null
-          })
+            interleaveFrames: overlap !== null,
+            openAttempt
+          }),
+          { stall: frameStallRule(job) }
         );
         let captureRes;
         try {
@@ -17639,6 +17732,7 @@ import {
 } from "@hyperframes/core";
 import { parseEncoderOptions } from "@hyperframes/studio-server/encoder-options";
 var DEFAULT_SERVER_FPS = { num: 30, den: 1 };
+var SSE_KEEPALIVE_MS = 3e4;
 var SAFE_RENDER_ERROR_CODES = /* @__PURE__ */ new Set([
   "ASSET_MEDIA_TYPE_MISMATCH",
   "NOT_MEDIA_PAYLOAD",
@@ -17646,7 +17740,8 @@ var SAFE_RENDER_ERROR_CODES = /* @__PURE__ */ new Set([
   "VIDEO_SOURCE_UNRENDERABLE",
   "VIDEO_EXTRACTION_FAILED",
   "ENCODER_INTERRUPTED",
-  "RENDER_TOO_SLOW"
+  "RENDER_TOO_SLOW",
+  "RENDER_STAGE_STALLED"
 ]);
 function extractSafeRenderErrorCode(error) {
   if (typeof error !== "object" || error === null || !("code" in error)) return void 0;
@@ -18160,15 +18255,21 @@ function createRenderHandlers(options = {}) {
           })
         });
       }
+      const reportProgress = createSseProgressReporter(stream, requestId);
+      const keepalive = setInterval(() => {
+        Promise.resolve(reportProgress(job, job.currentStage)).catch(() => {
+        });
+      }, SSE_KEEPALIVE_MS);
       const release = await renderSemaphore.acquire();
       try {
         await executeRenderJob(
           job,
           input.projectDir,
           absoluteOutputPath,
-          createSseProgressReporter(stream, requestId),
+          reportProgress,
           abortController.signal
-        );
+        ).finally(release);
+        clearInterval(keepalive);
         const fileSize = outputFileSize(absoluteOutputPath);
         const outputToken = store.register(absoluteOutputPath);
         const outputUrl = `${outputUrlPrefix}/${outputToken}`;
@@ -18188,6 +18289,7 @@ function createRenderHandlers(options = {}) {
           })
         });
       } catch (error) {
+        clearInterval(keepalive);
         await writeRenderStreamFailure({
           error,
           job,
@@ -18197,7 +18299,6 @@ function createRenderHandlers(options = {}) {
           log
         });
       } finally {
-        release();
         c.req.raw.signal.removeEventListener("abort", onRequestAbort);
         cleanupTempDir(cleanupProjectDir, log);
       }
@@ -20014,29 +20115,36 @@ async function renderChunk(planDir, chunkIndex, outputChunkPath, options) {
           const captureSession = session;
           session = null;
           overlap.current = canEncodeWhileCapturing(encodeInput, chunkWorkerCount, cpus3().length) ? encodeWhileCapturing(encodeInput, framesInChunk) : null;
-          await runCaptureStage({
-            fileServer,
-            workDir,
-            framesDir,
-            job,
-            totalFrames: framesInChunk,
-            cfg: captureCfg,
-            plan: capturePlan,
-            log,
-            probeSession: captureSession,
-            captureAttempts,
-            // This sink records each worker's effective capture mode so a
-            // fallback remains observable to adapters and smoke tests.
-            dedupPerfs: capturePerfs,
-            buildCaptureOptions: () => captureOptions,
-            createRenderVideoFrameInjector: createChunkVideoFrameInjector,
-            abortSignal: void 0,
-            assertNotAborted: () => {
-            },
-            frameRange: range,
-            onFrameLanded: (frame) => overlap.current?.onFrameLanded(frame),
-            interleaveFrames: interleavesCapture(overlap.current !== null, options)
-          }).catch(async (captureError) => {
+          await observeRenderStage(
+            new RenderObservabilityRecorder({ pipelineStartMs: start, log }),
+            "capture_chunk",
+            { chunkIndex },
+            (openAttempt) => runCaptureStage({
+              fileServer,
+              workDir,
+              framesDir,
+              job,
+              totalFrames: framesInChunk,
+              cfg: captureCfg,
+              plan: capturePlan,
+              log,
+              probeSession: captureSession,
+              captureAttempts,
+              // This sink records each worker's effective capture mode so a
+              // fallback remains observable to adapters and smoke tests.
+              dedupPerfs: capturePerfs,
+              buildCaptureOptions: () => captureOptions,
+              createRenderVideoFrameInjector: createChunkVideoFrameInjector,
+              abortSignal: void 0,
+              assertNotAborted: () => {
+              },
+              frameRange: range,
+              onFrameLanded: (frame) => overlap.current?.onFrameLanded(frame),
+              interleaveFrames: interleavesCapture(overlap.current !== null, options),
+              openAttempt
+            }),
+            { stall: frameStallRule(job) }
+          ).catch(async (captureError) => {
             await overlap.current?.abort();
             overlap.current = null;
             throw captureError;