@hyperframes/producer 0.9.9 → 0.9.11

package/dist/distributed.js

@@ -1435,6 +1435,27 @@ import {
   prepareCaptureSessionForReuse,
   spawnStreamingEncoder
 } from "@hyperframes/engine";
+
+// src/services/render/stallClock.ts
+var SUSPEND_GAP_MS = 15e3;
+function createStallClock(now = () => Date.now()) {
+  let since = now();
+  let seen = since;
+  return {
+    reset() {
+      since = seen = now();
+    },
+    idle(expectedMs) {
+      const at = now();
+      const gap = at - seen;
+      seen = at;
+      if (gap > expectedMs + SUSPEND_GAP_MS) since += gap;
+      return at - since;
+    }
+  };
+}
+
+// src/services/render/stages/captureStreamingStage.ts
 var DEFAULT_CAPTURE_STALL_MS = 6e4;
 function resolveCaptureStallTimeoutMs() {
   const raw = process.env.HF_DE_STALL_MS ?? process.env.HF_DE_PARALLEL_STALL_MS;
@@ -1802,35 +1823,33 @@ function watchStall(phase, rule) {
   const detail = rule.detail ?? (() => void 0);
   let last = rule.progress();
   let budgetMs = rule.firstProgressMs;
-  let lastChangeAt = Date.now();
+  const silence = createStallClock();
+  const pollMs = Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs);
   let attempt;
   let timer;
   let inTail = false;
   const restartWindow = (ms) => {
     last = rule.progress();
     budgetMs = ms;
-    lastChangeAt = Date.now();
+    silence.reset();
   };
   const tripped = new Promise((_, reject) => {
-    timer = setInterval(
-      () => {
-        const current = rule.progress();
-        if (rule.complete?.() && !inTail) {
-          inTail = true;
-          return restartWindow(rule.tailMs ?? rule.noProgressMs);
-        }
-        if (current !== last) return restartWindow(rule.noProgressMs);
-        if (Date.now() - lastChangeAt < budgetMs) return;
-        const error = new RenderStageStalledError(phase, budgetMs, current, detail());
-        if (attempt && !attempt.signal.aborted) {
-          attempt.abort(error);
-          return restartWindow(rule.noProgressMs);
-        }
-        clearInterval(timer);
-        reject(error);
-      },
-      Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs)
-    );
+    timer = setInterval(() => {
+      const current = rule.progress();
+      if (rule.complete?.() && !inTail) {
+        inTail = true;
+        return restartWindow(rule.tailMs ?? rule.noProgressMs);
+      }
+      if (current !== last) return restartWindow(rule.noProgressMs);
+      if (silence.idle(pollMs) < budgetMs) return;
+      const error = new RenderStageStalledError(phase, budgetMs, current, detail());
+      if (attempt && !attempt.signal.aborted) {
+        attempt.abort(error);
+        return restartWindow(rule.noProgressMs);
+      }
+      clearInterval(timer);
+      reject(error);
+    }, pollMs);
   });
   const openAttempt = (parent) => {
     attempt = new AbortController();

package/dist/index.js

@@ -3721,6 +3721,27 @@ import {
   prepareCaptureSessionForReuse,
   spawnStreamingEncoder
 } from "@hyperframes/engine";
+
+// src/services/render/stallClock.ts
+var SUSPEND_GAP_MS = 15e3;
+function createStallClock(now = () => Date.now()) {
+  let since = now();
+  let seen = since;
+  return {
+    reset() {
+      since = seen = now();
+    },
+    idle(expectedMs) {
+      const at = now();
+      const gap = at - seen;
+      seen = at;
+      if (gap > expectedMs + SUSPEND_GAP_MS) since += gap;
+      return at - since;
+    }
+  };
+}
+
+// src/services/render/stages/captureStreamingStage.ts
 var DEFAULT_CAPTURE_STALL_MS = 6e4;
 var DE_STALL_POLL_MS = 5e3;
 var STALL_ADVICE = "Chrome stopped answering. If another app is keeping the GPU or CPU busy (a 3D render, a game, another export), let it finish or render one job at a time, then retry.";
@@ -3765,21 +3786,27 @@ var SequentialCaptureStallError = class extends Error {
 };
 function raceAgainstStall(promise, deadlineMs, input, signal) {
   return new Promise((resolve13, reject) => {
-    const timer = setTimeout(
-      () => {
-        reject(
-          signal?.aborted ? new Error(
-            `[Render] Sequential ${captureModeLabel(input.session.captureMode)} capture aborted while a capture call was in flight (frame ${input.frameIndex}/${input.totalFrames}).`
-          ) : new SequentialCaptureStallError(
-            input.session,
-            input.stallTimeoutMs,
-            input.frameIndex,
-            input.totalFrames
-          )
-        );
-      },
-      Math.max(0, deadlineMs)
+    const clock = createStallClock();
+    const trip = () => reject(
+      signal?.aborted ? new Error(
+        `[Render] Sequential ${captureModeLabel(input.session.captureMode)} capture aborted while a capture call was in flight (frame ${input.frameIndex}/${input.totalFrames}).`
+      ) : new SequentialCaptureStallError(
+        input.session,
+        input.stallTimeoutMs,
+        input.frameIndex,
+        input.totalFrames
+      )
     );
+    const deadline = Math.max(0, deadlineMs);
+    let wait = Math.min(deadline, DE_STALL_POLL_MS);
+    let timer;
+    const look = () => {
+      const idle = clock.idle(wait);
+      if (idle >= deadline) return trip();
+      wait = Math.min(deadline - idle, DE_STALL_POLL_MS);
+      timer = setTimeout(look, wait);
+    };
+    timer = setTimeout(look, wait);
     promise.then(
       (value) => {
         clearTimeout(timer);
@@ -3889,26 +3916,26 @@ function createDrainFrameGuard(args) {
     return buf;
   };
 }
-async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEncoder, reorderBuffer, assertNotAborted, onProgress, log, stats, abortSignal, risk) {
+async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEncoder, reorderBuffer, assertNotAborted, onProgress, log, stats, abortSignal, resume, risk) {
   let prev = null;
   const frameTime = (i) => i * job.config.fps.den / job.config.fps.num;
   const guard = createDrainFrameGuard({ log, stats, frameTime });
   const guardFrame = (item, buf) => item.trusted ? Promise.resolve(buf) : guard(session, item.idx, buf);
   const captureRisk = async (idx) => {
     await risk.open();
-    lastProgressAt = Date.now();
+    quiet.reset();
     return raceAgainstStall(
       risk.capture(idx, frameTime(idx)),
-      stallTimeoutMs - (Date.now() - lastProgressAt),
+      stallTimeoutMs - quiet.idle(0),
       { session, frameIndex: idx, totalFrames, stallTimeoutMs },
       abortSignal
     );
   };
   const stallTimeoutMs = resolveCaptureStallTimeoutMs();
-  let lastProgressAt = Date.now();
+  const quiet = createStallClock();
   const captureWithStallGuard = (idx, promise) => raceAgainstStall(
     promise,
-    stallTimeoutMs - (Date.now() - lastProgressAt),
+    stallTimeoutMs - quiet.idle(0),
     { session, frameIndex: idx, totalFrames, stallTimeoutMs },
     abortSignal
   );
@@ -3920,7 +3947,8 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
     ensureFrameWritten(await currentEncoder.writeFrame(buf), prev.idx, currentEncoder);
     reorderBuffer.advanceTo(prev.idx + 1);
     job.framesRendered = prev.idx + 1;
-    lastProgressAt = Date.now();
+    resume.written = prev.idx + 1;
+    quiet.reset();
     reportFrameProgress(
       job,
       `Streaming frame ${prev.idx + 1}/${totalFrames}`,
@@ -3940,7 +3968,8 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
       ensureFrameWritten(await currentEncoder.writeFrame(buf), item.idx, currentEncoder);
       reorderBuffer.advanceTo(item.idx + 1);
       job.framesRendered = item.idx + 1;
-      lastProgressAt = Date.now();
+      resume.written = item.idx + 1;
+      quiet.reset();
       reportFrameProgress(
         job,
         `Streaming frame ${item.idx + 1}/${totalFrames}`,
@@ -3955,7 +3984,7 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
     const boundarySS = process.env.HF_FAST_CAPTURE_BOUNDARY_SS === "true";
     const batchable = (f) => !session.staticFrames?.has(f) && !(boundarySS && session.clipBoundaryFrames?.has(f)) && !risk?.frames.has(f);
     let prevBatch = [];
-    let i = 0;
+    let i = resume.from;
     while (i < totalFrames) {
       assertNotAborted();
       if (batchable(i)) {
@@ -3988,7 +4017,7 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
     await drainBatch(prevBatch);
     return;
   }
-  for (let i = 0; i < totalFrames; i++) {
+  for (let i = resume.from; i < totalFrames; i++) {
     assertNotAborted();
     if (risk?.frames.has(i)) {
       const buffer = await captureRisk(i);
@@ -4006,6 +4035,10 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
   }
   await drainPrev();
 }
+var STALL_RESUMES_BEFORE_FALLBACK = 3;
+function resumesAfterStall(error, abortSignal, resume, resumes) {
+  return error instanceof SequentialCaptureStallError && abortSignal?.aborted !== true && resume.written > resume.from && resumes < STALL_RESUMES_BEFORE_FALLBACK;
+}
 async function runCaptureStreamingStage(input) {
   const {
     fileServer,
@@ -4079,7 +4112,7 @@ async function runCaptureStreamingStage(input) {
       let parallelGuardRan = false;
       let writtenFrames = 0;
       let resumeFrom = 0;
-      let resumedAfterStall = false;
+      let resumes = 0;
       let workerResults;
       for (; ; ) {
         const parallelReorder = createFrameReorderBuffer(resumeFrom, totalFrames);
@@ -4114,7 +4147,7 @@ async function runCaptureStreamingStage(input) {
         }
         const stallTimeoutMs = resolveCaptureStallTimeoutMs();
         let lastCapturedFrames = 0;
-        let lastProgressAt = Date.now();
+        const quiet = createStallClock();
         const workerPhases = /* @__PURE__ */ new Map();
         const phaseDetail = (phase) => `phase=${phase.phase} frame=${phase.frameIndex ?? "n/a"} step=${phase.readStep?.() ?? "n/a"} browser=${phase.browserExecutable} version=${phase.browserVersion} CanvasDrawElement=${phase.canvasDrawElement} gpu=${phase.gpuBackend}`;
         const phaseSummary = () => [...workerPhases.entries()].sort(([left], [right]) => left - right).map(([workerId, detail]) => `worker=${workerId} ${detail}`).join("; ");
@@ -4125,23 +4158,21 @@ async function runCaptureStreamingStage(input) {
           captureLabel === "BeginFrame",
           cfg
         );
-        const stallTimer = setInterval(
-          () => {
-            if (!watchdogArmed || Date.now() - lastProgressAt <= budget()) return;
-            stalled = true;
-            const stallErr = new ParallelCaptureStallError({
-              variant: "drain",
-              captureLabel,
-              stallTimeoutMs: budget(),
-              lastCapturedFrames: resumeFrom + lastCapturedFrames,
-              totalFrames,
-              phaseSummary: phaseSummary()
-            });
-            parallelReorder.abort(stallErr);
-            stallController.abort();
-          },
-          Math.min(budget(), DE_STALL_POLL_MS)
-        );
+        const pollMs = Math.min(budget(), DE_STALL_POLL_MS);
+        const stallTimer = setInterval(() => {
+          if (!watchdogArmed || quiet.idle(pollMs) <= budget()) return;
+          stalled = true;
+          const stallErr = new ParallelCaptureStallError({
+            variant: "drain",
+            captureLabel,
+            stallTimeoutMs: budget(),
+            lastCapturedFrames: resumeFrom + lastCapturedFrames,
+            totalFrames,
+            phaseSummary: phaseSummary()
+          });
+          parallelReorder.abort(stallErr);
+          stallController.abort();
+        }, pollMs);
         try {
           workerResults = await executeParallelCapture(
             fileServer.url,
@@ -4156,7 +4187,7 @@ async function runCaptureStreamingStage(input) {
                 workerPhases.set(phase.workerId, phaseDetail(phase));
                 if (phase.phase === "frame_capture" && !watchdogArmed) {
                   watchdogArmed = true;
-                  lastProgressAt = Date.now();
+                  quiet.reset();
                 }
                 log.info("[Render] Parallel capture worker phase", {
                   workerId: phase.workerId,
@@ -4172,7 +4203,7 @@ async function runCaptureStreamingStage(input) {
               }
               if (progress.capturedFrames > lastCapturedFrames) {
                 lastCapturedFrames = progress.capturedFrames;
-                lastProgressAt = Date.now();
+                quiet.reset();
                 watchdogArmed = true;
               }
               const doneFrames = resumeFrom + progress.capturedFrames;
@@ -4203,17 +4234,14 @@ async function runCaptureStreamingStage(input) {
             }
           );
         } catch (err) {
-          if (stalled && abortSignal?.aborted !== true && captureLabel === "BeginFrame" && !resumedAfterStall && writtenFrames > resumeFrom) {
-            resumedAfterStall = true;
-            log.warn(
-              "[Render] Parallel BeginFrame capture stalled; resuming from the first unwritten frame",
-              {
-                resumeFrom: writtenFrames,
-                totalFrames,
-                workerPhases: phaseSummary()
-              }
-            );
+          if (stalled && abortSignal?.aborted !== true && writtenFrames > resumeFrom && resumes < STALL_RESUMES_BEFORE_FALLBACK) {
+            log.warn("[Render] Parallel capture stalled; resuming from the first unwritten frame", {
+              resumeFrom: writtenFrames,
+              totalFrames,
+              workerPhases: phaseSummary()
+            });
             resumeFrom = writtenFrames;
+            resumes++;
             continue;
           }
           if (parallelDrainError) throw poolFailure(parallelDrainError, probeSession);
@@ -4249,116 +4277,133 @@ async function runCaptureStreamingStage(input) {
       }
       lastBrowserConsole = withWorkerPageErrors(lastBrowserConsole, workerPageErrors);
     } else {
-      const videoInjector = createRenderVideoFrameInjector();
-      const session = probeSession ?? await createCaptureSession(
-        fileServer.url,
-        framesDir,
-        buildCaptureOptions(),
-        videoInjector,
-        captureCfg
-      );
-      captureBeyondViewport = session.options.captureBeyondViewport;
-      if (probeSession) {
-        prepareCaptureSessionForReuse(session, framesDir, videoInjector);
-        probeSession = null;
-      }
-      let riskSession = null;
-      const openRiskSession = async () => {
-        const opened = await createCaptureSession(
+      const resume = { from: 0, written: 0 };
+      let resumes = 0;
+      for (; ; ) {
+        const videoInjector = createRenderVideoFrameInjector();
+        const session = probeSession ?? await createCaptureSession(
           fileServer.url,
           framesDir,
           buildCaptureOptions(),
-          createRenderVideoFrameInjector(),
-          {
-            ...captureCfg,
-            useDrawElement: false,
-            staticFrameDedup: false,
-            enableBrowserPool: false
-          }
+          videoInjector,
+          captureCfg
         );
-        riskSession = opened;
-        await initializeSession(opened);
-        return opened;
-      };
-      try {
-        if (!session.isInitialized) {
-          await initializeSession(session);
+        captureBeyondViewport = session.options.captureBeyondViewport;
+        if (probeSession) {
+          prepareCaptureSessionForReuse(session, framesDir, videoInjector);
+          probeSession = null;
         }
-        await completeDeferredDrawElementInit(session);
-        assertNotAborted();
-        lastBrowserConsole = session.browserConsoleBuffer;
-        if (session.workerEncodeEnabled) {
-          deDrainStats = {
-            verifyChecked: 0,
-            blankSuspects: 0,
-            blankDeterministicAccepts: 0,
-            blankRecaptures: 0
-          };
-          const riskFrames = session.deRiskFrames;
-          await runWorkerEncodePipelineLoop(
-            session,
-            totalFrames,
-            job,
-            currentEncoder,
-            reorderBuffer,
-            assertNotAborted,
-            onProgress,
-            log,
-            deDrainStats,
-            abortSignal,
-            riskFrames && riskFrames.size > 0 ? {
-              frames: riskFrames,
-              // Opened on the first risk frame, in a browser of its own: pages sharing a
-              // browser share a compositor, which starves the drawElement page's paint events.
-              open: async () => {
-                riskSession ??= await openRiskSession();
-              },
-              capture: async (frameIndex, time) => (await captureFrameToBuffer(riskSession, frameIndex, time)).buffer
-            } : void 0
+        let riskSession = null;
+        const openRiskSession = async () => {
+          const opened = await createCaptureSession(
+            fileServer.url,
+            framesDir,
+            buildCaptureOptions(),
+            createRenderVideoFrameInjector(),
+            {
+              ...captureCfg,
+              useDrawElement: false,
+              staticFrameDedup: false,
+              enableBrowserPool: false
+            }
           );
-        } else {
-          const stallTimeoutMs = resolveCaptureStallTimeoutMs();
-          let lastProgressAt = Date.now();
-          for (let i = 0; i < totalFrames; i++) {
-            assertNotAborted();
-            const time = i * job.config.fps.den / job.config.fps.num;
-            const budget = stallBudgetMs(
-              stallTimeoutMs,
-              i,
-              session.captureMode === "beginframe",
-              cfg
-            );
-            const { buffer } = await raceAgainstStall(
-              captureFrameToBuffer(session, i, time),
-              budget - (Date.now() - lastProgressAt),
-              { session, frameIndex: i, totalFrames, stallTimeoutMs: budget },
-              abortSignal
-            );
-            await reorderBuffer.waitForFrame(i);
-            ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
-            reorderBuffer.advanceTo(i + 1);
-            job.framesRendered = i + 1;
-            lastProgressAt = Date.now();
-            const frameProgress = (i + 1) / totalFrames;
-            const progress = 25 + frameProgress * 55;
-            reportFrameProgress(
+          riskSession = opened;
+          await initializeSession(opened);
+          return opened;
+        };
+        let resumed = false;
+        try {
+          if (!session.isInitialized) {
+            await initializeSession(session);
+          }
+          await completeDeferredDrawElementInit(session);
+          assertNotAborted();
+          lastBrowserConsole = session.browserConsoleBuffer;
+          if (session.workerEncodeEnabled) {
+            deDrainStats ??= {
+              verifyChecked: 0,
+              blankSuspects: 0,
+              blankDeterministicAccepts: 0,
+              blankRecaptures: 0
+            };
+            const riskFrames = session.deRiskFrames;
+            await runWorkerEncodePipelineLoop(
+              session,
+              totalFrames,
               job,
-              `Streaming frame ${i + 1}/${totalFrames}`,
-              Math.round(progress),
+              currentEncoder,
+              reorderBuffer,
+              assertNotAborted,
               onProgress,
-              i + 1,
-              totalFrames
+              log,
+              deDrainStats,
+              abortSignal,
+              resume,
+              riskFrames && riskFrames.size > 0 ? {
+                frames: riskFrames,
+                // Opened on the first risk frame, in a browser of its own: pages sharing a
+                // browser share a compositor, which starves the drawElement page's paint events.
+                open: async () => {
+                  riskSession ??= await openRiskSession();
+                },
+                capture: async (frameIndex, time) => (await captureFrameToBuffer(riskSession, frameIndex, time)).buffer
+              } : void 0
             );
+          } else {
+            const stallTimeoutMs = resolveCaptureStallTimeoutMs();
+            let lastProgressAt = Date.now();
+            for (let i = resume.from; i < totalFrames; i++) {
+              assertNotAborted();
+              const time = i * job.config.fps.den / job.config.fps.num;
+              const budget = stallBudgetMs(
+                stallTimeoutMs,
+                i,
+                session.captureMode === "beginframe",
+                cfg
+              );
+              const { buffer } = await raceAgainstStall(
+                captureFrameToBuffer(session, i, time),
+                budget - (Date.now() - lastProgressAt),
+                { session, frameIndex: i, totalFrames, stallTimeoutMs: budget },
+                abortSignal
+              );
+              await reorderBuffer.waitForFrame(i);
+              ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
+              reorderBuffer.advanceTo(i + 1);
+              job.framesRendered = i + 1;
+              resume.written = i + 1;
+              lastProgressAt = Date.now();
+              const frameProgress = (i + 1) / totalFrames;
+              const progress = 25 + frameProgress * 55;
+              reportFrameProgress(
+                job,
+                `Streaming frame ${i + 1}/${totalFrames}`,
+                Math.round(progress),
+                onProgress,
+                i + 1,
+                totalFrames
+              );
+            }
+          }
+          dedupPerfs.push(getCapturePerfSummary(session));
+        } catch (error) {
+          lastBrowserConsole = session.browserConsoleBuffer;
+          if (!resumesAfterStall(error, abortSignal, resume, resumes)) {
+            throw wrapCaptureStageError(error, lastBrowserConsole);
           }
+          log.warn("[Render] Sequential capture stalled; resuming from the first unwritten frame", {
+            resumeFrom: resume.written,
+            totalFrames
+          });
+          resume.from = resume.written;
+          resumes++;
+          resumed = true;
+        } finally {
+          lastBrowserConsole = session.browserConsoleBuffer;
+          await closeCaptureSession(session);
+          if (riskSession) await closeCaptureSession(riskSession);
         }
-        dedupPerfs.push(getCapturePerfSummary(session));
-      } catch (error) {
-        lastBrowserConsole = session.browserConsoleBuffer;
-        throw wrapCaptureStageError(error, lastBrowserConsole);
-      } finally {
-        lastBrowserConsole = session.browserConsoleBuffer;
-        await closeCaptureSession(session);
-        if (riskSession) await closeCaptureSession(riskSession);
+        if (!resumed) break;
       }
     }
     const encodeFrom = job.progress;
@@ -5569,35 +5614,33 @@ function watchStall(phase, rule) {
   const detail = rule.detail ?? (() => void 0);
   let last = rule.progress();
   let budgetMs = rule.firstProgressMs;
-  let lastChangeAt = Date.now();
+  const silence = createStallClock();
+  const pollMs = Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs);
   let attempt;
   let timer;
   let inTail = false;
   const restartWindow = (ms) => {
     last = rule.progress();
     budgetMs = ms;
-    lastChangeAt = Date.now();
+    silence.reset();
   };
   const tripped = new Promise((_, reject) => {
-    timer = setInterval(
-      () => {
-        const current = rule.progress();
-        if (rule.complete?.() && !inTail) {
-          inTail = true;
-          return restartWindow(rule.tailMs ?? rule.noProgressMs);
-        }
-        if (current !== last) return restartWindow(rule.noProgressMs);
-        if (Date.now() - lastChangeAt < budgetMs) return;
-        const error = new RenderStageStalledError(phase, budgetMs, current, detail());
-        if (attempt && !attempt.signal.aborted) {
-          attempt.abort(error);
-          return restartWindow(rule.noProgressMs);
-        }
-        clearInterval(timer);
-        reject(error);
-      },
-      Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs)
-    );
+    timer = setInterval(() => {
+      const current = rule.progress();
+      if (rule.complete?.() && !inTail) {
+        inTail = true;
+        return restartWindow(rule.tailMs ?? rule.noProgressMs);
+      }
+      if (current !== last) return restartWindow(rule.noProgressMs);
+      if (silence.idle(pollMs) < budgetMs) return;
+      const error = new RenderStageStalledError(phase, budgetMs, current, detail());
+      if (attempt && !attempt.signal.aborted) {
+        attempt.abort(error);
+        return restartWindow(rule.noProgressMs);
+      }
+      clearInterval(timer);
+      reject(error);
+    }, pollMs);
   });
   const openAttempt = (parent) => {
     attempt = new AbortController();

package/dist/public-server.js

@@ -3734,6 +3734,27 @@ import {
   prepareCaptureSessionForReuse,
   spawnStreamingEncoder
 } from "@hyperframes/engine";
+
+// src/services/render/stallClock.ts
+var SUSPEND_GAP_MS = 15e3;
+function createStallClock(now = () => Date.now()) {
+  let since = now();
+  let seen = since;
+  return {
+    reset() {
+      since = seen = now();
+    },
+    idle(expectedMs) {
+      const at = now();
+      const gap = at - seen;
+      seen = at;
+      if (gap > expectedMs + SUSPEND_GAP_MS) since += gap;
+      return at - since;
+    }
+  };
+}
+
+// src/services/render/stages/captureStreamingStage.ts
 var DEFAULT_CAPTURE_STALL_MS = 6e4;
 var DE_STALL_POLL_MS = 5e3;
 var STALL_ADVICE = "Chrome stopped answering. If another app is keeping the GPU or CPU busy (a 3D render, a game, another export), let it finish or render one job at a time, then retry.";
@@ -3778,21 +3799,27 @@ var SequentialCaptureStallError = class extends Error {
 };
 function raceAgainstStall(promise, deadlineMs, input, signal) {
   return new Promise((resolve11, reject) => {
-    const timer = setTimeout(
-      () => {
-        reject(
-          signal?.aborted ? new Error(
-            `[Render] Sequential ${captureModeLabel(input.session.captureMode)} capture aborted while a capture call was in flight (frame ${input.frameIndex}/${input.totalFrames}).`
-          ) : new SequentialCaptureStallError(
-            input.session,
-            input.stallTimeoutMs,
-            input.frameIndex,
-            input.totalFrames
-          )
-        );
-      },
-      Math.max(0, deadlineMs)
+    const clock = createStallClock();
+    const trip = () => reject(
+      signal?.aborted ? new Error(
+        `[Render] Sequential ${captureModeLabel(input.session.captureMode)} capture aborted while a capture call was in flight (frame ${input.frameIndex}/${input.totalFrames}).`
+      ) : new SequentialCaptureStallError(
+        input.session,
+        input.stallTimeoutMs,
+        input.frameIndex,
+        input.totalFrames
+      )
     );
+    const deadline = Math.max(0, deadlineMs);
+    let wait = Math.min(deadline, DE_STALL_POLL_MS);
+    let timer;
+    const look = () => {
+      const idle = clock.idle(wait);
+      if (idle >= deadline) return trip();
+      wait = Math.min(deadline - idle, DE_STALL_POLL_MS);
+      timer = setTimeout(look, wait);
+    };
+    timer = setTimeout(look, wait);
     promise.then(
       (value) => {
         clearTimeout(timer);
@@ -3902,26 +3929,26 @@ function createDrainFrameGuard(args) {
     return buf;
   };
 }
-async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEncoder, reorderBuffer, assertNotAborted, onProgress, log, stats, abortSignal, risk) {
+async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEncoder, reorderBuffer, assertNotAborted, onProgress, log, stats, abortSignal, resume, risk) {
   let prev = null;
   const frameTime = (i) => i * job.config.fps.den / job.config.fps.num;
   const guard = createDrainFrameGuard({ log, stats, frameTime });
   const guardFrame = (item, buf) => item.trusted ? Promise.resolve(buf) : guard(session, item.idx, buf);
   const captureRisk = async (idx) => {
     await risk.open();
-    lastProgressAt = Date.now();
+    quiet.reset();
     return raceAgainstStall(
       risk.capture(idx, frameTime(idx)),
-      stallTimeoutMs - (Date.now() - lastProgressAt),
+      stallTimeoutMs - quiet.idle(0),
       { session, frameIndex: idx, totalFrames, stallTimeoutMs },
       abortSignal
     );
   };
   const stallTimeoutMs = resolveCaptureStallTimeoutMs();
-  let lastProgressAt = Date.now();
+  const quiet = createStallClock();
   const captureWithStallGuard = (idx, promise) => raceAgainstStall(
     promise,
-    stallTimeoutMs - (Date.now() - lastProgressAt),
+    stallTimeoutMs - quiet.idle(0),
     { session, frameIndex: idx, totalFrames, stallTimeoutMs },
     abortSignal
   );
@@ -3933,7 +3960,8 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
     ensureFrameWritten(await currentEncoder.writeFrame(buf), prev.idx, currentEncoder);
     reorderBuffer.advanceTo(prev.idx + 1);
     job.framesRendered = prev.idx + 1;
-    lastProgressAt = Date.now();
+    resume.written = prev.idx + 1;
+    quiet.reset();
     reportFrameProgress(
       job,
       `Streaming frame ${prev.idx + 1}/${totalFrames}`,
@@ -3953,7 +3981,8 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
       ensureFrameWritten(await currentEncoder.writeFrame(buf), item.idx, currentEncoder);
       reorderBuffer.advanceTo(item.idx + 1);
       job.framesRendered = item.idx + 1;
-      lastProgressAt = Date.now();
+      resume.written = item.idx + 1;
+      quiet.reset();
       reportFrameProgress(
         job,
         `Streaming frame ${item.idx + 1}/${totalFrames}`,
@@ -3968,7 +3997,7 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
     const boundarySS = process.env.HF_FAST_CAPTURE_BOUNDARY_SS === "true";
     const batchable = (f) => !session.staticFrames?.has(f) && !(boundarySS && session.clipBoundaryFrames?.has(f)) && !risk?.frames.has(f);
     let prevBatch = [];
-    let i = 0;
+    let i = resume.from;
     while (i < totalFrames) {
       assertNotAborted();
       if (batchable(i)) {
@@ -4001,7 +4030,7 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
     await drainBatch(prevBatch);
     return;
   }
-  for (let i = 0; i < totalFrames; i++) {
+  for (let i = resume.from; i < totalFrames; i++) {
     assertNotAborted();
     if (risk?.frames.has(i)) {
       const buffer = await captureRisk(i);
@@ -4019,6 +4048,10 @@ async function runWorkerEncodePipelineLoop(session, totalFrames, job, currentEnc
   }
   await drainPrev();
 }
+var STALL_RESUMES_BEFORE_FALLBACK = 3;
+function resumesAfterStall(error, abortSignal, resume, resumes) {
+  return error instanceof SequentialCaptureStallError && abortSignal?.aborted !== true && resume.written > resume.from && resumes < STALL_RESUMES_BEFORE_FALLBACK;
+}
 async function runCaptureStreamingStage(input) {
   const {
     fileServer,
@@ -4092,7 +4125,7 @@ async function runCaptureStreamingStage(input) {
       let parallelGuardRan = false;
       let writtenFrames = 0;
       let resumeFrom = 0;
-      let resumedAfterStall = false;
+      let resumes = 0;
       let workerResults;
       for (; ; ) {
         const parallelReorder = createFrameReorderBuffer(resumeFrom, totalFrames);
@@ -4127,7 +4160,7 @@ async function runCaptureStreamingStage(input) {
         }
         const stallTimeoutMs = resolveCaptureStallTimeoutMs();
         let lastCapturedFrames = 0;
-        let lastProgressAt = Date.now();
+        const quiet = createStallClock();
         const workerPhases = /* @__PURE__ */ new Map();
         const phaseDetail = (phase) => `phase=${phase.phase} frame=${phase.frameIndex ?? "n/a"} step=${phase.readStep?.() ?? "n/a"} browser=${phase.browserExecutable} version=${phase.browserVersion} CanvasDrawElement=${phase.canvasDrawElement} gpu=${phase.gpuBackend}`;
         const phaseSummary = () => [...workerPhases.entries()].sort(([left], [right]) => left - right).map(([workerId, detail]) => `worker=${workerId} ${detail}`).join("; ");
@@ -4138,23 +4171,21 @@ async function runCaptureStreamingStage(input) {
           captureLabel === "BeginFrame",
           cfg
         );
-        const stallTimer = setInterval(
-          () => {
-            if (!watchdogArmed || Date.now() - lastProgressAt <= budget()) return;
-            stalled = true;
-            const stallErr = new ParallelCaptureStallError({
-              variant: "drain",
-              captureLabel,
-              stallTimeoutMs: budget(),
-              lastCapturedFrames: resumeFrom + lastCapturedFrames,
-              totalFrames,
-              phaseSummary: phaseSummary()
-            });
-            parallelReorder.abort(stallErr);
-            stallController.abort();
-          },
-          Math.min(budget(), DE_STALL_POLL_MS)
-        );
+        const pollMs = Math.min(budget(), DE_STALL_POLL_MS);
+        const stallTimer = setInterval(() => {
+          if (!watchdogArmed || quiet.idle(pollMs) <= budget()) return;
+          stalled = true;
+          const stallErr = new ParallelCaptureStallError({
+            variant: "drain",
+            captureLabel,
+            stallTimeoutMs: budget(),
+            lastCapturedFrames: resumeFrom + lastCapturedFrames,
+            totalFrames,
+            phaseSummary: phaseSummary()
+          });
+          parallelReorder.abort(stallErr);
+          stallController.abort();
+        }, pollMs);
         try {
           workerResults = await executeParallelCapture(
             fileServer.url,
@@ -4169,7 +4200,7 @@ async function runCaptureStreamingStage(input) {
                 workerPhases.set(phase.workerId, phaseDetail(phase));
                 if (phase.phase === "frame_capture" && !watchdogArmed) {
                   watchdogArmed = true;
-                  lastProgressAt = Date.now();
+                  quiet.reset();
                 }
                 log.info("[Render] Parallel capture worker phase", {
                   workerId: phase.workerId,
@@ -4185,7 +4216,7 @@ async function runCaptureStreamingStage(input) {
               }
               if (progress.capturedFrames > lastCapturedFrames) {
                 lastCapturedFrames = progress.capturedFrames;
-                lastProgressAt = Date.now();
+                quiet.reset();
                 watchdogArmed = true;
               }
               const doneFrames = resumeFrom + progress.capturedFrames;
@@ -4216,17 +4247,14 @@ async function runCaptureStreamingStage(input) {
             }
           );
         } catch (err) {
-          if (stalled && abortSignal?.aborted !== true && captureLabel === "BeginFrame" && !resumedAfterStall && writtenFrames > resumeFrom) {
-            resumedAfterStall = true;
-            log.warn(
-              "[Render] Parallel BeginFrame capture stalled; resuming from the first unwritten frame",
-              {
-                resumeFrom: writtenFrames,
-                totalFrames,
-                workerPhases: phaseSummary()
-              }
-            );
+          if (stalled && abortSignal?.aborted !== true && writtenFrames > resumeFrom && resumes < STALL_RESUMES_BEFORE_FALLBACK) {
+            log.warn("[Render] Parallel capture stalled; resuming from the first unwritten frame", {
+              resumeFrom: writtenFrames,
+              totalFrames,
+              workerPhases: phaseSummary()
+            });
             resumeFrom = writtenFrames;
+            resumes++;
             continue;
           }
           if (parallelDrainError) throw poolFailure(parallelDrainError, probeSession);
@@ -4262,116 +4290,133 @@ async function runCaptureStreamingStage(input) {
       }
       lastBrowserConsole = withWorkerPageErrors(lastBrowserConsole, workerPageErrors);
     } else {
-      const videoInjector = createRenderVideoFrameInjector();
-      const session = probeSession ?? await createCaptureSession(
-        fileServer.url,
-        framesDir,
-        buildCaptureOptions(),
-        videoInjector,
-        captureCfg
-      );
-      captureBeyondViewport = session.options.captureBeyondViewport;
-      if (probeSession) {
-        prepareCaptureSessionForReuse(session, framesDir, videoInjector);
-        probeSession = null;
-      }
-      let riskSession = null;
-      const openRiskSession = async () => {
-        const opened = await createCaptureSession(
+      const resume = { from: 0, written: 0 };
+      let resumes = 0;
+      for (; ; ) {
+        const videoInjector = createRenderVideoFrameInjector();
+        const session = probeSession ?? await createCaptureSession(
           fileServer.url,
           framesDir,
           buildCaptureOptions(),
-          createRenderVideoFrameInjector(),
-          {
-            ...captureCfg,
-            useDrawElement: false,
-            staticFrameDedup: false,
-            enableBrowserPool: false
-          }
+          videoInjector,
+          captureCfg
         );
-        riskSession = opened;
-        await initializeSession(opened);
-        return opened;
-      };
-      try {
-        if (!session.isInitialized) {
-          await initializeSession(session);
+        captureBeyondViewport = session.options.captureBeyondViewport;
+        if (probeSession) {
+          prepareCaptureSessionForReuse(session, framesDir, videoInjector);
+          probeSession = null;
         }
-        await completeDeferredDrawElementInit(session);
-        assertNotAborted();
-        lastBrowserConsole = session.browserConsoleBuffer;
-        if (session.workerEncodeEnabled) {
-          deDrainStats = {
-            verifyChecked: 0,
-            blankSuspects: 0,
-            blankDeterministicAccepts: 0,
-            blankRecaptures: 0
-          };
-          const riskFrames = session.deRiskFrames;
-          await runWorkerEncodePipelineLoop(
-            session,
-            totalFrames,
-            job,
-            currentEncoder,
-            reorderBuffer,
-            assertNotAborted,
-            onProgress,
-            log,
-            deDrainStats,
-            abortSignal,
-            riskFrames && riskFrames.size > 0 ? {
-              frames: riskFrames,
-              // Opened on the first risk frame, in a browser of its own: pages sharing a
-              // browser share a compositor, which starves the drawElement page's paint events.
-              open: async () => {
-                riskSession ??= await openRiskSession();
-              },
-              capture: async (frameIndex, time) => (await captureFrameToBuffer(riskSession, frameIndex, time)).buffer
-            } : void 0
+        let riskSession = null;
+        const openRiskSession = async () => {
+          const opened = await createCaptureSession(
+            fileServer.url,
+            framesDir,
+            buildCaptureOptions(),
+            createRenderVideoFrameInjector(),
+            {
+              ...captureCfg,
+              useDrawElement: false,
+              staticFrameDedup: false,
+              enableBrowserPool: false
+            }
           );
-        } else {
-          const stallTimeoutMs = resolveCaptureStallTimeoutMs();
-          let lastProgressAt = Date.now();
-          for (let i = 0; i < totalFrames; i++) {
-            assertNotAborted();
-            const time = i * job.config.fps.den / job.config.fps.num;
-            const budget = stallBudgetMs(
-              stallTimeoutMs,
-              i,
-              session.captureMode === "beginframe",
-              cfg
-            );
-            const { buffer } = await raceAgainstStall(
-              captureFrameToBuffer(session, i, time),
-              budget - (Date.now() - lastProgressAt),
-              { session, frameIndex: i, totalFrames, stallTimeoutMs: budget },
-              abortSignal
-            );
-            await reorderBuffer.waitForFrame(i);
-            ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
-            reorderBuffer.advanceTo(i + 1);
-            job.framesRendered = i + 1;
-            lastProgressAt = Date.now();
-            const frameProgress = (i + 1) / totalFrames;
-            const progress = 25 + frameProgress * 55;
-            reportFrameProgress(
+          riskSession = opened;
+          await initializeSession(opened);
+          return opened;
+        };
+        let resumed = false;
+        try {
+          if (!session.isInitialized) {
+            await initializeSession(session);
+          }
+          await completeDeferredDrawElementInit(session);
+          assertNotAborted();
+          lastBrowserConsole = session.browserConsoleBuffer;
+          if (session.workerEncodeEnabled) {
+            deDrainStats ??= {
+              verifyChecked: 0,
+              blankSuspects: 0,
+              blankDeterministicAccepts: 0,
+              blankRecaptures: 0
+            };
+            const riskFrames = session.deRiskFrames;
+            await runWorkerEncodePipelineLoop(
+              session,
+              totalFrames,
               job,
-              `Streaming frame ${i + 1}/${totalFrames}`,
-              Math.round(progress),
+              currentEncoder,
+              reorderBuffer,
+              assertNotAborted,
               onProgress,
-              i + 1,
-              totalFrames
+              log,
+              deDrainStats,
+              abortSignal,
+              resume,
+              riskFrames && riskFrames.size > 0 ? {
+                frames: riskFrames,
+                // Opened on the first risk frame, in a browser of its own: pages sharing a
+                // browser share a compositor, which starves the drawElement page's paint events.
+                open: async () => {
+                  riskSession ??= await openRiskSession();
+                },
+                capture: async (frameIndex, time) => (await captureFrameToBuffer(riskSession, frameIndex, time)).buffer
+              } : void 0
             );
+          } else {
+            const stallTimeoutMs = resolveCaptureStallTimeoutMs();
+            let lastProgressAt = Date.now();
+            for (let i = resume.from; i < totalFrames; i++) {
+              assertNotAborted();
+              const time = i * job.config.fps.den / job.config.fps.num;
+              const budget = stallBudgetMs(
+                stallTimeoutMs,
+                i,
+                session.captureMode === "beginframe",
+                cfg
+              );
+              const { buffer } = await raceAgainstStall(
+                captureFrameToBuffer(session, i, time),
+                budget - (Date.now() - lastProgressAt),
+                { session, frameIndex: i, totalFrames, stallTimeoutMs: budget },
+                abortSignal
+              );
+              await reorderBuffer.waitForFrame(i);
+              ensureFrameWritten(await currentEncoder.writeFrame(buffer), i, currentEncoder);
+              reorderBuffer.advanceTo(i + 1);
+              job.framesRendered = i + 1;
+              resume.written = i + 1;
+              lastProgressAt = Date.now();
+              const frameProgress = (i + 1) / totalFrames;
+              const progress = 25 + frameProgress * 55;
+              reportFrameProgress(
+                job,
+                `Streaming frame ${i + 1}/${totalFrames}`,
+                Math.round(progress),
+                onProgress,
+                i + 1,
+                totalFrames
+              );
+            }
+          }
+          dedupPerfs.push(getCapturePerfSummary(session));
+        } catch (error) {
+          lastBrowserConsole = session.browserConsoleBuffer;
+          if (!resumesAfterStall(error, abortSignal, resume, resumes)) {
+            throw wrapCaptureStageError(error, lastBrowserConsole);
           }
+          log.warn("[Render] Sequential capture stalled; resuming from the first unwritten frame", {
+            resumeFrom: resume.written,
+            totalFrames
+          });
+          resume.from = resume.written;
+          resumes++;
+          resumed = true;
+        } finally {
+          lastBrowserConsole = session.browserConsoleBuffer;
+          await closeCaptureSession(session);
+          if (riskSession) await closeCaptureSession(riskSession);
         }
-        dedupPerfs.push(getCapturePerfSummary(session));
-      } catch (error) {
-        lastBrowserConsole = session.browserConsoleBuffer;
-        throw wrapCaptureStageError(error, lastBrowserConsole);
-      } finally {
-        lastBrowserConsole = session.browserConsoleBuffer;
-        await closeCaptureSession(session);
-        if (riskSession) await closeCaptureSession(riskSession);
+        if (!resumed) break;
       }
     }
     const encodeFrom = job.progress;
@@ -5582,35 +5627,33 @@ function watchStall(phase, rule) {
   const detail = rule.detail ?? (() => void 0);
   let last = rule.progress();
   let budgetMs = rule.firstProgressMs;
-  let lastChangeAt = Date.now();
+  const silence = createStallClock();
+  const pollMs = Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs);
   let attempt;
   let timer;
   let inTail = false;
   const restartWindow = (ms) => {
     last = rule.progress();
     budgetMs = ms;
-    lastChangeAt = Date.now();
+    silence.reset();
   };
   const tripped = new Promise((_, reject) => {
-    timer = setInterval(
-      () => {
-        const current = rule.progress();
-        if (rule.complete?.() && !inTail) {
-          inTail = true;
-          return restartWindow(rule.tailMs ?? rule.noProgressMs);
-        }
-        if (current !== last) return restartWindow(rule.noProgressMs);
-        if (Date.now() - lastChangeAt < budgetMs) return;
-        const error = new RenderStageStalledError(phase, budgetMs, current, detail());
-        if (attempt && !attempt.signal.aborted) {
-          attempt.abort(error);
-          return restartWindow(rule.noProgressMs);
-        }
-        clearInterval(timer);
-        reject(error);
-      },
-      Math.min(STALL_POLL_MS, rule.noProgressMs, rule.firstProgressMs)
-    );
+    timer = setInterval(() => {
+      const current = rule.progress();
+      if (rule.complete?.() && !inTail) {
+        inTail = true;
+        return restartWindow(rule.tailMs ?? rule.noProgressMs);
+      }
+      if (current !== last) return restartWindow(rule.noProgressMs);
+      if (silence.idle(pollMs) < budgetMs) return;
+      const error = new RenderStageStalledError(phase, budgetMs, current, detail());
+      if (attempt && !attempt.signal.aborted) {
+        attempt.abort(error);
+        return restartWindow(rule.noProgressMs);
+      }
+      clearInterval(timer);
+      reject(error);
+    }, pollMs);
   });
   const openAttempt = (parent) => {
     attempt = new AbortController();

package/dist/services/render/stallClock.d.ts

@@ -0,0 +1,7 @@
+export interface StallClock {
+    /** Progress landed: the silence starts again. */
+    reset(): void;
+    /** Silent ms, not counting a suspended stretch; `expectedMs` is how long after the last look this one was due. */
+    idle(expectedMs: number): number;
+}
+export declare function createStallClock(now?: () => number): StallClock;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperframes/producer",
-  "version": "0.9.9",
+  "version": "0.9.11",
   "description": "HTML-to-video rendering engine using Chrome's BeginFrame API",
   "repository": {
     "type": "git",
@@ -35,11 +35,11 @@
     "postcss": "^8.4.0",
     "puppeteer": "~25.11.0",
     "puppeteer-core": "^25.11.0",
-    "@hyperframes/core": "0.9.9",
-    "@hyperframes/engine": "0.9.9",
-    "@hyperframes/lint": "0.9.9",
-    "@hyperframes/studio-server": "0.9.9",
-    "@hyperframes/parsers": "0.9.9"
+    "@hyperframes/core": "0.9.11",
+    "@hyperframes/engine": "0.9.11",
+    "@hyperframes/lint": "0.9.11",
+    "@hyperframes/parsers": "0.9.11",
+    "@hyperframes/studio-server": "0.9.11"
   },
   "devDependencies": {
     "@types/node": "^25.0.10",
@@ -48,7 +48,7 @@
     "tsx": "^4.21.0",
     "typescript": "^5.7.2",
     "vitest": "^4.1.11",
-    "@hyperframes/aws-lambda": "0.9.9"
+    "@hyperframes/aws-lambda": "0.9.11"
   },
   "engines": {
     "node": ">=22"