autokap 1.8.5 → 1.8.7

package/dist/action-verifier.d.ts

@@ -12,6 +12,12 @@ export interface ActionVerification {
     changes: ActionChange[];
     /** Summary for logging */
     summary: string;
+    /**
+     * AUT-240 (decision 2): the after-state was unreadable even after a settle +
+     * retry, so the effect was assumed as a last resort. The capture this taints
+     * is flagged low-confidence rather than failed.
+     */
+    lowConfidence?: boolean;
 }
 export type ActionChangeKind = 'url_changed' | 'tree_structure_changed' | 'node_appeared' | 'node_disappeared' | 'node_state_changed' | 'scroll_changed' | 'overlay_changed' | 'no_change';
 export interface ActionChange {

package/dist/action-verifier.js

@@ -32,23 +32,36 @@ export class ActionVerifier {
                 summary: 'no before state captured, assuming action had effect',
             };
         }
-        let afterUrl;
-        let afterTree;
+        // AUT-240 (decision 2): "assume OK, but smart". A first `page.evaluate`
+        // hiccup (typically mid-navigation) is no longer assumed-OK immediately —
+        // settle the page and retry once. Only if it still throws do we assume the
+        // effect, and then we flag the result low-confidence for downstream scrutiny.
+        let after = null;
         try {
-            [afterUrl, afterTree] = await Promise.all([
-                adapter.getCurrentUrl(),
-                adapter.getAKTree(),
-            ]);
+            const [url, tree] = await Promise.all([adapter.getCurrentUrl(), adapter.getAKTree()]);
+            after = { url, tree };
         }
         catch {
-            this.beforeTree = null;
-            this.beforeUrl = null;
-            return {
-                hadEffect: true,
-                changes: [],
-                summary: 'after state unavailable, skipping AKTree effect verification',
-            };
+            try {
+                if (adapter.waitForVisuallyStable) {
+                    await adapter.waitForVisuallyStable({ maxWaitMs: 2000 });
+                }
+                const [url, tree] = await Promise.all([adapter.getCurrentUrl(), adapter.getAKTree()]);
+                after = { url, tree };
+            }
+            catch {
+                this.beforeTree = null;
+                this.beforeUrl = null;
+                return {
+                    hadEffect: true,
+                    changes: [],
+                    summary: 'after state unavailable after settle, assuming effect (low-confidence)',
+                    lowConfidence: true,
+                };
+            }
         }
+        const afterUrl = after.url;
+        const afterTree = after.tree;
         const changes = [];
         // 1. URL change
         if (afterUrl !== this.beforeUrl) {
@@ -105,11 +118,11 @@ export class ActionVerifier {
         const beforeState = collectVisibleNodeState(this.beforeTree.root);
         const afterState = collectVisibleNodeState(afterTree.root);
         const changedStates = [];
-        for (const [key, before] of beforeState) {
-            const after = afterState.get(key);
-            if (!after)
+        for (const [key, beforeNodeState] of beforeState) {
+            const afterNodeState = afterState.get(key);
+            if (!afterNodeState)
                 continue;
-            if (before !== after) {
+            if (beforeNodeState !== afterNodeState) {
                 changedStates.push(key);
                 if (changedStates.length >= 5)
                     break;

package/dist/browser.d.ts

@@ -1,5 +1,6 @@
 import { type BrowserContext, type Page } from 'playwright';
 import type { AKNode, AKNodeRuntimeIndexEntry, AKTree, BrowserOptions, BrowserSessionStorageState, BrowserStorageState, InteractiveElement, ObservedRequest, OutscaleConfig, PageState, PageStateLite, ResolvedMock, SelectorValidationResult, VideoPageSignals } from './types.js';
+import type { ProgressSnapshot, VisualStabilityResult } from './execution-types.js';
 export interface SelectorCaptureError {
     error: 'no_match' | 'ambiguous' | 'invisible' | 'zero_size';
     errorMessage: string;
@@ -103,6 +104,18 @@ export declare class Browser {
     private poolContext;
     private persistentContext;
     private ownedChromiumProfileDir;
+    /**
+     * Network-progress counters (AUT-240, Layer C). Maintained by request
+     * lifecycle listeners attached lazily and idempotently per page via
+     * `ensureProgressListeners()`, so every page-creation path is covered without
+     * touching each one. Count FIRST-PARTY traffic only (same site as the live
+     * main-frame origin) so background third-party telemetry does not read as
+     * progress. Read cheaply by `getProgressSnapshot()`.
+     */
+    private progressListenersPage;
+    private inflightRequests;
+    private networkEventCount;
+    private lastNetworkActivityAtMs;
     /**
      * Xvfb instance backing the headed Chromium used by clip capture on Cloud
      * Run with NVIDIA L4. Set when forClipCapture spawns Xvfb; null otherwise
@@ -168,6 +181,52 @@ export declare class Browser {
      */
     private waitForDomStability;
     private waitForFontsBeforeScreenshot;
+    /**
+     * Attach network-progress listeners to `page` once. Idempotent per page: a
+     * no-op if already attached, and resets counters only when the `Page` object
+     * itself changes (recreated context, pooled re-acquire). A `page.reload()`
+     * keeps the same object, so counters stay monotonic across it — harmless, as
+     * the watchdog only ever compares deltas via `hasProgress`. Covers every
+     * page-creation path without editing each one.
+     */
+    private ensureProgressListeners;
+    /**
+     * Cheap snapshot of page activity for the runner's progress watchdog
+     * (AUT-240, Layer C). Network counters are free (maintained by listeners);
+     * `readyState`/`domNodeCount` come from one light `evaluate`. Never rejects:
+     * an `evaluate` failure (navigation in flight) is itself progress, surfaced
+     * as `navigating: true`.
+     */
+    getProgressSnapshot(): Promise<ProgressSnapshot>;
+    /**
+     * Wait until the page is visually stable enough to screenshot (AUT-240,
+     * Layer B). Best-effort and non-blocking: composes light semantic signals —
+     * fonts ready, images loaded, no visible `[aria-busy]`/`[role=progressbar]`,
+     * DOM quiet — and only falls back to a bounded pixel-convergence check when
+     * loaders never clear. Returns `stable: false` with a reason instead of
+     * throwing or blocking; the caller captures anyway (a perpetual animation is
+     * cosmetic, not a reason to fail the capture — decision 1).
+     */
+    waitForVisuallyStable(options?: {
+        maxWaitMs?: number;
+    }): Promise<VisualStabilityResult>;
+    private remainingMs;
+    /** Poll until in-DOM images have finished loading, or the budget elapses. */
+    private waitForImagesSettled;
+    /**
+     * Poll until no semantically-marked loader is visible, or the deadline hits.
+     * Condition: zero visible elements matching
+     * `[aria-busy="true"], [role="progressbar"]:not([hidden])` — covers shadcn,
+     * Radix, MUI, etc. without a hardcoded class list. Returns true if cleared.
+     */
+    private waitForNoVisibleLoaders;
+    /**
+     * Bounded pixel-convergence fallback (AUT-240 decision 1): take up to
+     * `PIXEL_FALLBACK_MAX_PASSES` raw frames; if two consecutive frames are within
+     * `PIXEL_FALLBACK_DIFF_THRESHOLD`, the page is settled. Never blocks past the
+     * deadline; returns false (capture anyway) for perpetual animations.
+     */
+    private pixelConvergenceFallback;
     takeScreenshot(): Promise<Buffer>;
     takeScreenshotForAI(options?: {
         timeoutMs?: number;

package/dist/browser.js

@@ -3,7 +3,9 @@ import sharp from 'sharp';
 import { createHash } from 'crypto';
 import { cp, mkdir, readFile, rm, writeFile } from 'fs/promises';
 import { join } from 'path';
+import { DOM_QUIET_WINDOW_MS, GLOBAL_WAIT_CAP_MS, PIXEL_FALLBACK_DIFF_THRESHOLD, PIXEL_FALLBACK_MAX_PASSES, } from './wait-contract.js';
 import { buildAKNodeRuntimeIndex, deriveInteractiveElementsFromAKTree, disambiguateFingerprint, focusAKTree, fingerprintAKNode, serializeAKTree, } from './ak-tree.js';
+import { isFirstPartyUrl } from './security.js';
 /**
  * Set-of-Marks (SoM) annotation: overlays colored [N] badges on each visible
  * interactive element so the vision model can reference elements by their badge index.
@@ -876,6 +878,29 @@ export function describeObservationChange(before, after) {
             : 'No visible state change detected after the action.',
     };
 }
+function delayMs(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Sampled byte-diff ratio between two screenshot buffers (AUT-240, Layer B
+ * pixel fallback). Cheap O(sampleCount) approximation — good enough to tell a
+ * settled frame from a moving one without decoding the PNG.
+ */
+function sampledBufferDiffRatio(a, b) {
+    const maxLength = Math.max(a.length, b.length);
+    if (maxLength === 0)
+        return 0;
+    const sampleCount = Math.min(10000, maxLength);
+    const step = Math.max(1, Math.floor(maxLength / sampleCount));
+    let checked = 0;
+    let changed = 0;
+    for (let i = 0; i < maxLength; i += step) {
+        checked++;
+        if (a[i] !== b[i])
+            changed++;
+    }
+    return checked === 0 ? 0 : changed / checked;
+}
 export class Browser {
     options;
     browser = null;
@@ -886,6 +911,18 @@ export class Browser {
     poolContext = false;
     persistentContext = false;
     ownedChromiumProfileDir = null;
+    /**
+     * Network-progress counters (AUT-240, Layer C). Maintained by request
+     * lifecycle listeners attached lazily and idempotently per page via
+     * `ensureProgressListeners()`, so every page-creation path is covered without
+     * touching each one. Count FIRST-PARTY traffic only (same site as the live
+     * main-frame origin) so background third-party telemetry does not read as
+     * progress. Read cheaply by `getProgressSnapshot()`.
+     */
+    progressListenersPage = null;
+    inflightRequests = 0;
+    networkEventCount = 0;
+    lastNetworkActivityAtMs = 0;
     /**
      * Xvfb instance backing the headed Chromium used by clip capture on Cloud
      * Run with NVIDIA L4. Set when forClipCapture spawns Xvfb; null otherwise
@@ -1554,6 +1591,228 @@ export class Browser {
             logger.debug(`[capture] font paint gate resolved via=${gate.via} elapsedMs=${gate.elapsedMs} totalMs=${Date.now() - startedAt}`);
         }
     }
+    /**
+     * Attach network-progress listeners to `page` once. Idempotent per page: a
+     * no-op if already attached, and resets counters only when the `Page` object
+     * itself changes (recreated context, pooled re-acquire). A `page.reload()`
+     * keeps the same object, so counters stay monotonic across it — harmless, as
+     * the watchdog only ever compares deltas via `hasProgress`. Covers every
+     * page-creation path without editing each one.
+     */
+    ensureProgressListeners(page) {
+        if (this.progressListenersPage === page)
+            return;
+        this.progressListenersPage = page;
+        this.inflightRequests = 0;
+        this.networkEventCount = 0;
+        this.lastNetworkActivityAtMs = Date.now();
+        const bump = () => {
+            this.networkEventCount++;
+            this.lastNetworkActivityAtMs = Date.now();
+        };
+        // Only the app's OWN traffic counts as progress (AUT-240). Third-party
+        // telemetry (PostHog beacons, analytics/ad pixels, Sentry, polling/ws-fallback
+        // XHR to other origins) chatters indefinitely on a real app and would
+        // otherwise keep `hasProgress` true forever, so the stuck watchdog could never
+        // cut a wait whose condition will never be met. First-party-ness is decided
+        // once, at request time, against the live main-frame origin; the same request
+        // object is remembered so finished/failed decrement symmetrically even if the
+        // page navigates mid-flight (a foreign decrement is simply skipped).
+        const firstPartyInflight = new WeakSet();
+        page.on('request', (request) => {
+            if (!isFirstPartyUrl(page.url(), request.url()))
+                return;
+            firstPartyInflight.add(request);
+            this.inflightRequests++;
+            bump();
+        });
+        page.on('requestfinished', (request) => {
+            if (!firstPartyInflight.has(request))
+                return;
+            this.inflightRequests = Math.max(0, this.inflightRequests - 1);
+            bump();
+        });
+        page.on('requestfailed', (request) => {
+            if (!firstPartyInflight.has(request))
+                return;
+            this.inflightRequests = Math.max(0, this.inflightRequests - 1);
+            bump();
+        });
+    }
+    /**
+     * Cheap snapshot of page activity for the runner's progress watchdog
+     * (AUT-240, Layer C). Network counters are free (maintained by listeners);
+     * `readyState`/`domNodeCount` come from one light `evaluate`. Never rejects:
+     * an `evaluate` failure (navigation in flight) is itself progress, surfaced
+     * as `navigating: true`.
+     */
+    async getProgressSnapshot() {
+        const page = this.ensurePage();
+        this.ensureProgressListeners(page);
+        let readyState = 'unknown';
+        let domNodeCount = -1;
+        let navigating = false;
+        try {
+            const probe = await page.evaluate(() => ({
+                rs: document.readyState,
+                n: document.getElementsByTagName('*').length,
+            }));
+            readyState = probe.rs;
+            domNodeCount = probe.n;
+        }
+        catch {
+            navigating = true;
+        }
+        return {
+            networkEventCount: this.networkEventCount,
+            inflightRequests: this.inflightRequests,
+            lastNetworkActivityAtMs: this.lastNetworkActivityAtMs,
+            readyState,
+            domNodeCount,
+            navigating,
+        };
+    }
+    /**
+     * Wait until the page is visually stable enough to screenshot (AUT-240,
+     * Layer B). Best-effort and non-blocking: composes light semantic signals —
+     * fonts ready, images loaded, no visible `[aria-busy]`/`[role=progressbar]`,
+     * DOM quiet — and only falls back to a bounded pixel-convergence check when
+     * loaders never clear. Returns `stable: false` with a reason instead of
+     * throwing or blocking; the caller captures anyway (a perpetual animation is
+     * cosmetic, not a reason to fail the capture — decision 1).
+     */
+    async waitForVisuallyStable(options) {
+        const page = this.ensurePage();
+        const startedAt = Date.now();
+        const maxWait = Math.max(1000, options?.maxWaitMs ?? GLOBAL_WAIT_CAP_MS.screenshot);
+        const deadline = startedAt + maxWait;
+        // 1. Fonts + images painted (avoid FOUT / broken images).
+        await this.waitForFontsBeforeScreenshot(page).catch(() => { });
+        await this.waitForImagesSettled(page, Math.min(3000, this.remainingMs(deadline)));
+        // 2. Semantic loaders cleared (replaces the legacy 8 hardcoded CSS classes).
+        const loadersCleared = await this.waitForNoVisibleLoaders(page, deadline);
+        // 3. DOM quiet.
+        const domBudget = Math.min(3000, this.remainingMs(deadline));
+        if (domBudget > 0) {
+            await this.waitForDomStability(page, DOM_QUIET_WINDOW_MS, domBudget);
+        }
+        if (loadersCleared) {
+            return { stable: true, reason: 'page visually stable', waitedMs: Date.now() - startedAt };
+        }
+        // 4. Loaders persisted ⇒ bounded pixel-convergence fallback (decision 1).
+        // Two converging frames ⇒ the page is effectively static (e.g. a non-ARIA
+        // spinner that finished); otherwise it is a perpetual animation — capture
+        // anyway rather than block.
+        const converged = await this.pixelConvergenceFallback(page, deadline);
+        return converged
+            ? {
+                stable: true,
+                reason: 'stable via pixel convergence (loaders not ARIA-marked)',
+                waitedMs: Date.now() - startedAt,
+            }
+            : {
+                stable: false,
+                reason: 'loaders or animation still active at deadline; capturing anyway',
+                waitedMs: Date.now() - startedAt,
+            };
+    }
+    remainingMs(deadline) {
+        return Math.max(0, deadline - Date.now());
+    }
+    /** Poll until in-DOM images have finished loading, or the budget elapses. */
+    async waitForImagesSettled(page, budgetMs) {
+        if (budgetMs <= 0)
+            return;
+        try {
+            await page.evaluate((maxWait) => {
+                return new Promise((resolve) => {
+                    const deadline = Date.now() + maxWait;
+                    const check = () => {
+                        const imgs = document.querySelectorAll('img');
+                        for (const img of imgs) {
+                            if (!img.src || img.src.startsWith('data:'))
+                                continue;
+                            if (img.width <= 1 && img.height <= 1)
+                                continue;
+                            if (!img.complete || img.naturalWidth === 0)
+                                return false;
+                        }
+                        return true;
+                    };
+                    if (check()) {
+                        resolve();
+                        return;
+                    }
+                    const iv = setInterval(() => {
+                        if (check() || Date.now() >= deadline) {
+                            clearInterval(iv);
+                            resolve();
+                        }
+                    }, 100);
+                });
+            }, budgetMs);
+        }
+        catch {
+            // Page navigated during the wait — not stable yet, but best-effort.
+        }
+    }
+    /**
+     * Poll until no semantically-marked loader is visible, or the deadline hits.
+     * Condition: zero visible elements matching
+     * `[aria-busy="true"], [role="progressbar"]:not([hidden])` — covers shadcn,
+     * Radix, MUI, etc. without a hardcoded class list. Returns true if cleared.
+     */
+    async waitForNoVisibleLoaders(page, deadline) {
+        const selector = '[aria-busy="true"], [role="progressbar"]:not([hidden])';
+        while (Date.now() < deadline) {
+            let count = -1;
+            try {
+                count = await page.evaluate((sel) => {
+                    const isVisible = (el) => {
+                        const rect = el.getBoundingClientRect();
+                        if (rect.width === 0 || rect.height === 0)
+                            return false;
+                        const style = getComputedStyle(el);
+                        return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0';
+                    };
+                    return Array.from(document.querySelectorAll(sel)).filter(isVisible).length;
+                }, selector);
+            }
+            catch {
+                count = -1; // navigation in flight — not settled, keep waiting
+            }
+            if (count === 0)
+                return true;
+            await delayMs(Math.min(150, this.remainingMs(deadline)));
+        }
+        return false;
+    }
+    /**
+     * Bounded pixel-convergence fallback (AUT-240 decision 1): take up to
+     * `PIXEL_FALLBACK_MAX_PASSES` raw frames; if two consecutive frames are within
+     * `PIXEL_FALLBACK_DIFF_THRESHOLD`, the page is settled. Never blocks past the
+     * deadline; returns false (capture anyway) for perpetual animations.
+     */
+    async pixelConvergenceFallback(page, deadline) {
+        let prev = null;
+        for (let pass = 0; pass < PIXEL_FALLBACK_MAX_PASSES; pass++) {
+            if (Date.now() >= deadline)
+                break;
+            let frame;
+            try {
+                frame = await page.screenshot();
+            }
+            catch {
+                return false;
+            }
+            if (prev && sampledBufferDiffRatio(prev, frame) <= PIXEL_FALLBACK_DIFF_THRESHOLD) {
+                return true;
+            }
+            prev = frame;
+            await delayMs(Math.min(250, this.remainingMs(deadline)));
+        }
+        return false;
+    }
     async takeScreenshot() {
         const page = this.ensurePage();
         // Move cursor off-screen to avoid hover effects in screenshots

package/dist/cli-contract.d.ts

@@ -131,6 +131,11 @@ export interface ArtifactUploadMetadata {
     targetLabel?: string | null;
     programVersion?: number | null;
     compileFingerprint?: string | null;
+    engineVersion?: number | null;
+    programSchemaVersion?: number | null;
+    programSchemaVersionOrigin?: number | null;
+    cliVersion?: string | null;
+    programHash?: string | null;
     mediaMode: "screenshot" | "clip" | "video";
     mimeType: string;
     captureType: "fullpage" | "element";

package/dist/cli-runner.d.ts

@@ -19,7 +19,6 @@ export interface RecordableBrowserSettings {
     runtimeDeviceScaleFactor: number;
 }
 export declare function resolveRecordableBrowserSettings(program: ExecutionProgram, variant: VariantSpec): RecordableBrowserSettings;
-export declare function normalizeVideoCaptureProgram(program: ExecutionProgram): ExecutionProgram;
 export interface CLIRunnerOptions {
     /** Preset ID to run */
     presetId: string;