autokap 1.9.3 → 1.9.4

package/dist/opcode-actions.d.ts

@@ -54,5 +54,15 @@ export interface OpcodeActionResult {
      * click instead of when the cursor was still travelling.
      */
     clickTimestampsMs?: number[];
+    /**
+     * For CAPTURE_SCREENSHOT: index in the variant's `artifacts` array of the
+     * screenshot this action just delivered. The deterministic capture pushes the
+     * artifact immediately; LLM enrichment (quality verification + alt text) runs
+     * afterwards in `executeOpcode` and mutates this artifact in place, so it can
+     * never void an already-captured screenshot or trip the action timeout.
+     */
+    captureArtifactIndex?: number;
+    /** For CAPTURE_SCREENSHOT: page URL at capture time, reused by enrichment. */
+    captureUrl?: string;
 }
 export declare function executeOpcodeCoreAction(opcode: ExecutionOpcode, adapter: RuntimeAdapter, context?: OpcodeActionContext): Promise<OpcodeActionResult>;

package/dist/opcode-runner.js

@@ -55,6 +55,15 @@ export class NoOpRecoveryChain {
 }
 const MIN_CLIP_FINALIZATION_TIMEOUT_MS = 30000;
 const DEFAULT_VIDEO_RECORDING_RESOLUTION = { width: 1920, height: 1080 };
+/**
+ * The compiled per-opcode action budget. For CAPTURE_SCREENSHOT this governs
+ * ONLY the deterministic capture (visual stabilize + screenshot + favicon/title
+ * + artifact push); LLM enrichment (quality verification + alt text) runs AFTER
+ * the action under the global wait deadline (see `enrichCaptureArtifact`) and
+ * must never be folded back under this timeout — that is the regression this
+ * separation prevents. END_CLIP finalization gets a floor since muxing a
+ * recording is inherently slow.
+ */
 function resolveOpcodeTimeoutMs(opcode) {
     if (opcode.kind === 'END_CLIP') {
         return Math.max(opcode.timeoutMs, MIN_CLIP_FINALIZATION_TIMEOUT_MS);
@@ -543,6 +552,18 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
             await sleep(VIDEO_POST_ACTION_SETTLE_MS);
         }
         breaker.recordSuccess(index);
+        // Enrich the just-captured screenshot OFF the timed action path: quality
+        // verification + alt text run here, under the global wait deadline, and
+        // mutate the artifact in place — so slow LLM calls can never void a
+        // delivered screenshot (the failure this fixes). Recaptured screenshots from
+        // the recovery path are intentionally left un-enriched (recovery stays
+        // deterministic and fast).
+        if (opcode.kind === 'CAPTURE_SCREENSHOT'
+            && options.llmConfig
+            && result.captureArtifactIndex !== undefined
+            && result.captureUrl !== undefined) {
+            await enrichCaptureArtifact(artifacts[result.captureArtifactIndex], opcode, adapter, result.captureUrl, currentVariant, options, telemetry, globalDeadlineMs);
+        }
         return {
             opcodeIndex: index,
             kind: opcode.kind,
@@ -760,13 +781,7 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
                 // semantic loaders/DOM-quiet + bounded pixel fallback); fall back to the
                 // legacy smart-wait for adapters that don't implement it. A page that
                 // never fully settles (e.g. a perpetual animation) is captured anyway.
-                const stabilize = (maxWaitMs) => adapter.waitForVisuallyStable
-                    ? adapter.waitForVisuallyStable({ maxWaitMs })
-                    : smartWaitForStability(adapter, { maxWaitMs }).then((r) => ({
-                        stable: r.stable,
-                        reason: r.waitedFor.join(', ') || 'unknown',
-                    }));
-                const stability = await stabilize(5000);
+                const stability = await stabilizeForCapture(adapter, 5000);
                 if (!stability.stable) {
                     logger.debug(`[opcode ${opcodeIndex}] capturing despite unstable page: ${stability.reason}`);
                 }
@@ -784,85 +799,7 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
                 }
                 const captureLowConfidenceReason = lowConfidenceReasons.join('; ') || undefined;
                 const captureUrl = await adapter.getCurrentUrl();
-                const takeBuffer = async () => {
-                    if (opcode.elementSelector && adapter.takeElementScreenshot) {
-                        return adapter.takeElementScreenshot(opcode.elementSelector, opcode.outscale);
-                    }
-                    if (opcode.elementSelector) {
-                        throw new Error(`element capture requires adapter support for selector "${opcode.elementSelector}"`);
-                    }
-                    return adapter.takeScreenshot();
-                };
-                let buffer = await takeBuffer();
-                if (runOptions?.llmConfig) {
-                    const verification = await verifyCaptureQuality(buffer, {
-                        expectedDescription: opcode.description,
-                        url: captureUrl,
-                        locale: currentVariant?.locale,
-                        theme: currentVariant?.theme,
-                    }, runOptions.llmConfig);
-                    if (verification.llmResult) {
-                        telemetry.llmCallCount++;
-                        telemetry.llmCostEur += verification.llmResult.costEur;
-                        telemetry.llmStepUsages.push({
-                            stepType: 'capture_verification',
-                            generationId: verification.llmResult.generationId,
-                            model: verification.llmResult.model,
-                            promptTokens: verification.llmResult.promptTokens,
-                            completionTokens: verification.llmResult.completionTokens,
-                        });
-                    }
-                    if (!verification.passed) {
-                        await stabilize(8000);
-                        const retryBuffer = await takeBuffer();
-                        const retryVerification = await verifyCaptureQuality(retryBuffer, {
-                            expectedDescription: opcode.description,
-                            url: captureUrl,
-                            locale: currentVariant?.locale,
-                            theme: currentVariant?.theme,
-                        }, runOptions.llmConfig);
-                        if (retryVerification.llmResult) {
-                            telemetry.llmCallCount++;
-                            telemetry.llmCostEur += retryVerification.llmResult.costEur;
-                            telemetry.llmStepUsages.push({
-                                stepType: 'capture_verification',
-                                generationId: retryVerification.llmResult.generationId,
-                                model: retryVerification.llmResult.model,
-                                promptTokens: retryVerification.llmResult.promptTokens,
-                                completionTokens: retryVerification.llmResult.completionTokens,
-                            });
-                        }
-                        if (retryVerification.passed) {
-                            buffer = retryBuffer;
-                        }
-                    }
-                }
-                let altText;
-                if (runOptions?.llmConfig) {
-                    try {
-                        const altResult = await generateAltText(buffer, {
-                            description: opcode.description,
-                            url: captureUrl,
-                            locale: currentVariant?.locale,
-                            presetName: runOptions.presetName,
-                        }, runOptions.llmConfig);
-                        altText = altResult.altText;
-                        if (altResult.llmResult) {
-                            telemetry.llmCallCount++;
-                            telemetry.llmCostEur += altResult.llmResult.costEur;
-                            telemetry.llmStepUsages.push({
-                                stepType: 'alt_text_generation',
-                                generationId: altResult.llmResult.generationId,
-                                model: altResult.llmResult.model,
-                                promptTokens: altResult.llmResult.promptTokens,
-                                completionTokens: altResult.llmResult.completionTokens,
-                            });
-                        }
-                    }
-                    catch {
-                        // Alt text generation failed — non-fatal
-                    }
-                }
+                const buffer = await takeCaptureBuffer(adapter, opcode);
                 // Extract page favicon for browser bar mockup
                 let tabIconData;
                 let tabIconMimeType;
@@ -892,7 +829,7 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
                     captureId: opcode.captureId,
                     captureName: opcode.captureName ?? opcode.description,
                     elementSelector: opcode.elementSelector,
-                    altText,
+                    altText: undefined,
                     stepDescription: opcode.description,
                     stepIndex: opcodeIndex,
                     variantId: currentVariant?.id,
@@ -901,7 +838,11 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
                     lowConfidence: captureLowConfidence || undefined,
                     lowConfidenceReason: captureLowConfidence ? captureLowConfidenceReason : undefined,
                 });
-                break;
+                // Deliver the screenshot artifact NOW. LLM enrichment (quality
+                // verification + alt text) runs afterwards in `executeOpcode`, off this
+                // timed action path, and mutates the artifact in place — it can never
+                // void a captured screenshot or push the action past its timeout.
+                return { success: true, captureArtifactIndex: artifacts.length - 1, captureUrl };
             }
             case 'BEGIN_CLIP': {
                 if (executionState.activeClip) {
@@ -967,6 +908,127 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
         };
     }
 }
+// ── Capture: deterministic capture + best-effort enrichment ─────────
+/**
+ * Re-stabilize budget for the verification retry. Sized for the post-action
+ * enrichment budget (under the ~30s global deadline), not the tight per-opcode
+ * action timeout — it only needs to clear transient skeletons/spinners before a
+ * second look. Was 8000ms when this ran inside the 10s action; that starved the
+ * capture and is exactly what this separation removes.
+ */
+const VERIFY_RETRY_STABILIZE_MS = 3000;
+/** Visual stabilization shared by the capture action and the verify retry.
+ * Never fails the capture — a page that never settles is captured anyway. */
+function stabilizeForCapture(adapter, maxWaitMs) {
+    return adapter.waitForVisuallyStable
+        ? adapter.waitForVisuallyStable({ maxWaitMs })
+        : smartWaitForStability(adapter, { maxWaitMs }).then((r) => ({
+            stable: r.stable,
+            reason: r.waitedFor.join(', ') || 'unknown',
+        }));
+}
+/** Take the screenshot buffer for a CAPTURE_SCREENSHOT opcode (element clip or
+ * full page). Shared by the capture action and the verify retry. */
+function takeCaptureBuffer(adapter, opcode) {
+    if (opcode.elementSelector && adapter.takeElementScreenshot) {
+        return adapter.takeElementScreenshot(opcode.elementSelector, opcode.outscale);
+    }
+    if (opcode.elementSelector) {
+        throw new Error(`element capture requires adapter support for selector "${opcode.elementSelector}"`);
+    }
+    return adapter.takeScreenshot();
+}
+function recordCaptureVerificationTelemetry(telemetry, llmResult) {
+    if (!llmResult)
+        return;
+    telemetry.llmCallCount++;
+    telemetry.llmCostEur += llmResult.costEur;
+    telemetry.llmStepUsages.push({
+        stepType: 'capture_verification',
+        generationId: llmResult.generationId,
+        model: llmResult.model,
+        promptTokens: llmResult.promptTokens,
+        completionTokens: llmResult.completionTokens,
+    });
+}
+/**
+ * Best-effort LLM enrichment of an already-delivered screenshot artifact:
+ * quality verification (with one budget-bounded re-stabilize + re-shoot) and
+ * alt text. Runs AFTER the deterministic capture action, off its timed path,
+ * and mutates the artifact in place. By construction it never changes
+ * `artifacts.length`, so it can never turn a captured screenshot into a
+ * "no artifact" failure — the regression this fixes.
+ *
+ * Bounded by the per-opcode global wait deadline. If the budget elapses
+ * mid-call the in-flight LLM promise is orphaned; `settled` makes any late
+ * in-place write a no-op so it cannot mutate an artifact the runner moved past.
+ */
+async function enrichCaptureArtifact(artifact, opcode, adapter, captureUrl, currentVariant, runOptions, telemetry, globalDeadlineMs) {
+    const llmConfig = runOptions.llmConfig;
+    if (!llmConfig)
+        return;
+    const budgetMs = getRemainingTimeMs(globalDeadlineMs);
+    if (budgetMs <= 0)
+        return;
+    let settled = false;
+    const verificationContext = {
+        expectedDescription: opcode.description,
+        url: captureUrl,
+        locale: currentVariant?.locale,
+        theme: currentVariant?.theme,
+    };
+    try {
+        await withTimeout(async () => {
+            const verification = await verifyCaptureQuality(artifact.buffer, verificationContext, llmConfig);
+            recordCaptureVerificationTelemetry(telemetry, verification.llmResult);
+            // On a failed verdict, give the page a brief settle and re-shoot once,
+            // bounded by the remaining global budget. Swap the buffer in place only if
+            // the second shot verifies clean.
+            if (!verification.passed) {
+                const retryStabilizeMs = Math.min(VERIFY_RETRY_STABILIZE_MS, getRemainingTimeMs(globalDeadlineMs));
+                if (retryStabilizeMs > 0) {
+                    await stabilizeForCapture(adapter, retryStabilizeMs);
+                    const retryBuffer = await takeCaptureBuffer(adapter, opcode);
+                    const retryVerification = await verifyCaptureQuality(retryBuffer, verificationContext, llmConfig);
+                    recordCaptureVerificationTelemetry(telemetry, retryVerification.llmResult);
+                    if (retryVerification.passed && !settled) {
+                        artifact.buffer = retryBuffer;
+                    }
+                }
+            }
+            try {
+                const altResult = await generateAltText(artifact.buffer, {
+                    description: opcode.description,
+                    url: captureUrl,
+                    locale: currentVariant?.locale,
+                    presetName: runOptions.presetName,
+                }, llmConfig);
+                if (!settled)
+                    artifact.altText = altResult.altText;
+                if (altResult.llmResult) {
+                    telemetry.llmCallCount++;
+                    telemetry.llmCostEur += altResult.llmResult.costEur;
+                    telemetry.llmStepUsages.push({
+                        stepType: 'alt_text_generation',
+                        generationId: altResult.llmResult.generationId,
+                        model: altResult.llmResult.model,
+                        promptTokens: altResult.llmResult.promptTokens,
+                        completionTokens: altResult.llmResult.completionTokens,
+                    });
+                }
+            }
+            catch {
+                // Alt text generation failed — non-fatal.
+            }
+        }, budgetMs);
+    }
+    catch (err) {
+        logger.debug(`[opcode CAPTURE_SCREENSHOT] enrichment skipped: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    finally {
+        settled = true;
+    }
+}
 /**
  * Snapshot per-opcode timing + element bbox before the action runs. Returns
  * null when no timing should be emitted (mediaMode != video, or no active

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autokap",
-  "version": "1.9.3",
+  "version": "1.9.4",
   "description": "AI-powered CLI tool for capturing clean screenshots of websites",
   "type": "module",
   "main": "./dist/index.js",