autokap 1.5.2 → 1.5.3

package/dist/auth-capture.js

@@ -1,5 +1,6 @@
 import { chromium } from 'playwright';
 import { logger } from './logger.js';
+import { ensureChromiumInstalled } from './playwright-installer.js';
 /**
  * Opens a headed Chromium window, lets the user log in, and uploads the
  * resulting storageState (cookies + localStorage, HttpOnly included) to the
@@ -11,6 +12,7 @@ import { logger } from './logger.js';
  */
 export async function captureAuthSession(options) {
     const { apiBaseUrl, apiKey, projectId, accountId, startUrl } = options;
+    await ensureChromiumInstalled();
     logger.info('[auth] Launching Chromium…');
     const browser = await chromium.launch({ headless: false });
     let context = null;

package/dist/cli-contract.d.ts

@@ -58,6 +58,13 @@ export interface VideoClipMetadata {
          * SFX in the video compositor.
          */
         keystrokeOffsetsMs?: number[];
+        /**
+         * For CLICK / DOUBLE_CLICK / CHECK opcodes captured in clipCursor mode:
+         * clip-relative ms timestamp of each actual click dispatched by Playwright
+         * (measured AFTER the cursor animation settled). Drives mouse SFX in
+         * sync with the visible click.
+         */
+        clickOffsetsMs?: number[];
     }>;
 }
 export interface VideoAudioAsset {

package/dist/cli-runner.js

@@ -19,6 +19,7 @@ import { Browser } from './browser.js';
 import { API_BASE_URL_ENV_VAR, requireConfig } from './cli-config.js';
 import { WebPlaywrightLocal } from './web-playwright-local.js';
 import { executeProgram } from './opcode-runner.js';
+import { ensureChromiumInstalled } from './playwright-installer.js';
 import { RecoveryChainImpl } from './recovery-chain.js';
 import { parseProgram } from './execution-schema.js';
 import { buildCursorOverlayScript } from './cursor-overlay-script.js';
@@ -108,6 +109,18 @@ function normalizeNumericScale(value) {
 const HEALER_SYSTEM_PROMPT = 'You repair failed deterministic browser opcodes. Respond only with JSON.';
 // ── Main entry point ────────────────────────────────────────────────
 export async function runCapture(options) {
+    // Self-heal a missing Playwright Chromium binary BEFORE anything else.
+    // Skipped postinstalls or downstream Playwright version bumps would
+    // otherwise surface as a cryptic launch failure mid-capture.
+    try {
+        await ensureChromiumInstalled();
+    }
+    catch (err) {
+        return {
+            success: false,
+            error: `playwright chromium install failed: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
     const config = await requireConfig();
     // Step 1: Get the compiled program
     let resolvedProgram;
@@ -751,6 +764,9 @@ export function buildVideoClipMetadata(videoId, result, program, runId) {
                 ...(t.keystrokeOffsetsMs && t.keystrokeOffsetsMs.length > 0
                     ? { keystrokeOffsetsMs: t.keystrokeOffsetsMs }
                     : {}),
+                ...(t.clickOffsetsMs && t.clickOffsetsMs.length > 0
+                    ? { clickOffsetsMs: t.clickOffsetsMs }
+                    : {}),
             }));
             clipsByKey.set(`${variantId}:${artifact.clipId}`, {
                 variantId,
@@ -954,8 +970,8 @@ async function prepareDirectArtifactUpload(params) {
         clipName: artifact.clipName ?? null,
         stepDescription: artifact.stepDescription ?? null,
         stepIndex: typeof artifact.stepIndex === 'number' ? artifact.stepIndex : null,
-        durationMs: typeof artifact.durationMs === 'number' ? artifact.durationMs : null,
-        trimStartMs: typeof artifact.trimStartMs === 'number' ? artifact.trimStartMs : null,
+        durationMs: typeof artifact.durationMs === 'number' ? Math.round(artifact.durationMs) : null,
+        trimStartMs: typeof artifact.trimStartMs === 'number' ? Math.round(artifact.trimStartMs) : null,
         artifactPlan: program.artifactPlan,
         tabIconMimeType: artifact.tabIconData ? (artifact.tabIconMimeType ?? 'image/png') : null,
         tabIconSha256,

package/dist/execution-types.d.ts

@@ -700,6 +700,14 @@ export interface OpcodeTiming {
      * for non-TYPE opcodes and for typing paths that bypass humanType.
      */
     keystrokeOffsetsMs?: number[];
+    /**
+     * For CLICK / DOUBLE_CLICK / CHECK opcodes captured in clipCursor mode:
+     * timestamp (ms relative to the active clip start) at which Playwright
+     * dispatched each actual click — measured AFTER the cursor animation
+     * settled on the target. Drives mouse SFX in sync with the visible click.
+     * Empty/undefined for opcodes whose adapter doesn't surface the timestamp.
+     */
+    clickOffsetsMs?: number[];
 }
 export interface RunResult {
     programId: string;
@@ -736,6 +744,25 @@ export interface ClickOptions {
     };
     /** Mouse button. Default: 'left' */
     button?: 'left' | 'right' | 'middle';
+    /**
+     * Fired with `Date.now()` right before Playwright dispatches the actual
+     * click — i.e. AFTER the visible cursor animation has settled on the
+     * target. The runner converts the wall-clock to clip-relative offsets so
+     * the video compositor can fire mouse SFX in lock-step with the visible
+     * click (instead of when the cursor was still travelling).
+     */
+    onClick?: (timestampMs: number) => void;
+}
+export interface ClickByTargetOptions {
+    selector?: string;
+    target?: SemanticTarget;
+    selectorAlternates?: string[];
+    onClick?: (timestampMs: number) => void;
+}
+export interface MouseActionOptions {
+    /** Same semantics as `ClickOptions.onClick` — fires right before the
+     *  actual click is dispatched (CHECK / DOUBLE_CLICK). */
+    onClick?: (timestampMs: number) => void;
 }
 export interface RecordingOptions {
     mediaMode: 'clip' | 'video';
@@ -815,11 +842,7 @@ export interface RuntimeAdapter {
     } | null>;
     close(): Promise<void>;
     /** Click an element by semantic target. Falls back to selector if target not found. */
-    clickByTarget?(opts: {
-        selector?: string;
-        target?: SemanticTarget;
-        selectorAlternates?: string[];
-    }): Promise<void>;
+    clickByTarget?(opts: ClickByTargetOptions): Promise<void>;
     /** Type into an element by semantic target. */
     typeByTarget?(opts: {
         selector?: string;
@@ -849,8 +872,8 @@ export interface RuntimeAdapter {
         value?: string;
         index?: number;
     }): Promise<void>;
-    check?(selector: string, checked: boolean): Promise<void>;
-    doubleClick?(selector: string): Promise<void>;
+    check?(selector: string, checked: boolean, opts?: MouseActionOptions): Promise<void>;
+    doubleClick?(selector: string, opts?: MouseActionOptions): Promise<void>;
     /**
      * Drag the source element from point A to point B with an animated cursor
      * when a clip is recording. Destination is either another element

package/dist/opcode-actions.d.ts

@@ -46,5 +46,13 @@ export interface OpcodeActionResult {
      * clip-relative offsets so the video compositor can fire per-keystroke SFX.
      */
     keystrokeTimestampsMs?: number[];
+    /**
+     * For CLICK / DOUBLE_CLICK / CHECK opcodes: absolute wall-clock timestamps
+     * captured INSIDE the adapter, just before Playwright dispatches the
+     * actual click — i.e. AFTER the cursor animation has settled on the
+     * target. Lets the compositor place the mouse SFX in sync with the visible
+     * click instead of when the cursor was still travelling.
+     */
+    clickTimestampsMs?: number[];
 }
 export declare function executeOpcodeCoreAction(opcode: ExecutionOpcode, adapter: RuntimeAdapter, context?: OpcodeActionContext): Promise<OpcodeActionResult>;

package/dist/opcode-actions.js

@@ -62,9 +62,13 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
             case 'DISMISS_OVERLAYS':
                 await dismissAllOverlays(adapter);
                 break;
-            case 'CLICK':
+            case 'CLICK': {
+                const clickTimestampsMs = [];
+                const onClick = (timestampMs) => {
+                    clickTimestampsMs.push(timestampMs);
+                };
                 try {
-                    await adapter.click(opcode.selector, opcode.button ? { button: opcode.button } : undefined);
+                    await adapter.click(opcode.selector, { ...(opcode.button ? { button: opcode.button } : {}), onClick });
                 }
                 catch (error) {
                     if (!opcode.target || !adapter.clickByTarget)
@@ -73,9 +77,11 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
                         selector: opcode.selector,
                         target: opcode.target,
                         selectorAlternates: opcode.selectorAlternates,
+                        onClick,
                     });
                 }
-                break;
+                return { success: true, clickTimestampsMs };
+            }
             case 'TYPE': {
                 const rawText = (opcode.textByLocale && context.currentVariant?.locale
                     ? opcode.textByLocale[context.currentVariant.locale] ?? opcode.text
@@ -174,16 +180,24 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
                     index: opcode.optionIndex,
                 });
                 break;
-            case 'CHECK':
+            case 'CHECK': {
                 if (!adapter.check)
                     return { success: false, error: 'adapter does not support CHECK' };
-                await adapter.check(opcode.selector, opcode.checked);
-                break;
-            case 'DOUBLE_CLICK':
+                const clickTimestampsMs = [];
+                await adapter.check(opcode.selector, opcode.checked, {
+                    onClick: (timestampMs) => clickTimestampsMs.push(timestampMs),
+                });
+                return { success: true, clickTimestampsMs };
+            }
+            case 'DOUBLE_CLICK': {
                 if (!adapter.doubleClick)
                     return { success: false, error: 'adapter does not support DOUBLE_CLICK' };
-                await adapter.doubleClick(opcode.selector);
-                break;
+                const clickTimestampsMs = [];
+                await adapter.doubleClick(opcode.selector, {
+                    onClick: (timestampMs) => clickTimestampsMs.push(timestampMs),
+                });
+                return { success: true, clickTimestampsMs };
+            }
             case 'DRAG':
                 if (!adapter.drag)
                     return { success: false, error: 'adapter does not support DRAG' };

package/dist/opcode-runner.js

@@ -290,6 +290,9 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
             const keystrokeOffsetsMs = result.keystrokeTimestampsMs && result.keystrokeTimestampsMs.length > 0
                 ? result.keystrokeTimestampsMs.map((t) => Math.max(0, t - preTiming.clipStartedAt))
                 : undefined;
+            const clickOffsetsMs = result.clickTimestampsMs && result.clickTimestampsMs.length > 0
+                ? result.clickTimestampsMs.map((t) => Math.max(0, t - preTiming.clipStartedAt))
+                : undefined;
             opcodeTimings.push({
                 stepIndex: index,
                 stepId: opcode.stepId,
@@ -300,6 +303,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
                 timecodeEndMs: Math.max(0, Date.now() - preTiming.clipStartedAt),
                 bbox: preTiming.bbox,
                 ...(keystrokeOffsetsMs ? { keystrokeOffsetsMs } : {}),
+                ...(clickOffsetsMs ? { clickOffsetsMs } : {}),
             });
         }
         if (!result.success) {
@@ -666,6 +670,12 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
             }
             case 'END_CLIP': {
                 const clipIdentity = resolveClipIdentity(executionState.activeClip, opcode);
+                // Capture the URL BEFORE endRecording(): the local CDP branch of the
+                // adapter closes the browser context inside endRecording() to release
+                // the CDP session, which makes any subsequent browser operation throw
+                // "Browser not launched". The clip ends immediately before this call,
+                // so the URL is still accurate.
+                const captureUrl = await adapter.getCurrentUrl();
                 const recording = await adapter.endRecording();
                 executionState.activeClip = undefined;
                 // Match the artifact's mediaMode to the program's so the upload route
@@ -680,7 +690,7 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
                     trimStartMs: recording.trimStartMs,
                     dimensions: undefined,
                     captureType: 'fullpage',
-                    captureUrl: await adapter.getCurrentUrl(),
+                    captureUrl,
                     clipId: clipIdentity.clipId,
                     clipName: clipIdentity.clipName,
                     stepDescription: opcode.description,

package/dist/playwright-installer.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Preflight Playwright Chromium binary install.
+ *
+ * Run before any `chromium.launch(...)` to make sure the Playwright Chromium
+ * binary exists on disk. The package ships a `postinstall` hook that fetches
+ * Chromium at install time, but that step can be silently skipped on some
+ * setups (CI caches, monorepos, npm scripts) — and any Playwright version
+ * bump downstream invalidates the existing binary too. Catching this once at
+ * run start avoids the famously confusing "Executable doesn't exist at …"
+ * launch failure mid-capture.
+ *
+ * Cross-platform: uses `npx playwright install chromium` (works on macOS,
+ * Linux, Windows). Output is streamed inherit so the user sees the
+ * download progress.
+ */
+export declare function ensureChromiumInstalled(): Promise<void>;

package/dist/playwright-installer.js

@@ -0,0 +1,80 @@
+/**
+ * Preflight Playwright Chromium binary install.
+ *
+ * Run before any `chromium.launch(...)` to make sure the Playwright Chromium
+ * binary exists on disk. The package ships a `postinstall` hook that fetches
+ * Chromium at install time, but that step can be silently skipped on some
+ * setups (CI caches, monorepos, npm scripts) — and any Playwright version
+ * bump downstream invalidates the existing binary too. Catching this once at
+ * run start avoids the famously confusing "Executable doesn't exist at …"
+ * launch failure mid-capture.
+ *
+ * Cross-platform: uses `npx playwright install chromium` (works on macOS,
+ * Linux, Windows). Output is streamed inherit so the user sees the
+ * download progress.
+ */
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { chromium } from 'playwright';
+import { logger } from './logger.js';
+let cachedCheckResult = null;
+export async function ensureChromiumInstalled() {
+    // One-shot per process: once we've verified (or installed) the binary,
+    // skip every subsequent call. `chromium.executablePath()` is cheap but
+    // `existsSync` adds up across recovery retries.
+    if (cachedCheckResult === 'ok')
+        return;
+    let execPath = '';
+    try {
+        execPath = chromium.executablePath();
+    }
+    catch {
+        // executablePath() throws if Playwright has no path resolved at all.
+        // Treat the same as "not installed" so we run the install.
+        execPath = '';
+    }
+    if (execPath && existsSync(execPath)) {
+        cachedCheckResult = 'ok';
+        return;
+    }
+    logger.info('[playwright] Chromium browser is missing — installing it now (one-time, takes ~30s)…');
+    await runPlaywrightInstall();
+    // Re-check; if the install genuinely succeeded `executablePath()` now
+    // resolves to a real file. If it still doesn't, surface a clearer error
+    // than the raw launch failure.
+    try {
+        const refreshed = chromium.executablePath();
+        if (existsSync(refreshed)) {
+            cachedCheckResult = 'ok';
+            logger.info('[playwright] Chromium installed.');
+            return;
+        }
+    }
+    catch {
+        // fallthrough to error below
+    }
+    throw new Error('Playwright Chromium install completed but the binary is still missing. ' +
+        'Run `npx playwright install chromium` manually to diagnose.');
+}
+async function runPlaywrightInstall() {
+    return new Promise((resolve, reject) => {
+        // Use `npx` so we hit whichever Playwright version this CLI depends on,
+        // regardless of whether the user has a global `playwright` binary.
+        const child = spawn('npx', ['--yes', 'playwright', 'install', 'chromium'], {
+            stdio: 'inherit',
+        });
+        child.on('error', (err) => {
+            reject(new Error(`Failed to spawn \`npx playwright install chromium\`: ${err.message}. ` +
+                'Make sure Node.js + npm are installed and on PATH.'));
+        });
+        child.on('close', (code) => {
+            if (code === 0) {
+                resolve();
+                return;
+            }
+            reject(new Error(`\`npx playwright install chromium\` exited with code ${code}. ` +
+                'Run it manually for full diagnostics.'));
+        });
+    });
+}
+//# sourceMappingURL=playwright-installer.js.map

package/dist/web-playwright-local.d.ts

@@ -34,7 +34,9 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
      * Click an element using semantic target resolution.
      * Tries CSS selector first, falls back to Playwright semantic locators.
      */
-    clickByTarget(opts: ResolveOptions): Promise<void>;
+    clickByTarget(opts: ResolveOptions & {
+        onClick?: (timestampMs: number) => void;
+    }): Promise<void>;
     /**
      * Type into an element using semantic target resolution.
      */
@@ -87,8 +89,12 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
         value?: string;
         index?: number;
     }): Promise<void>;
-    check(selector: string, checked: boolean): Promise<void>;
-    doubleClick(selector: string): Promise<void>;
+    check(selector: string, checked: boolean, actionOpts?: {
+        onClick?: (timestampMs: number) => void;
+    }): Promise<void>;
+    doubleClick(selector: string, actionOpts?: {
+        onClick?: (timestampMs: number) => void;
+    }): Promise<void>;
     drag(opts: {
         selector?: string;
         target?: SemanticTarget;

package/dist/web-playwright-local.js

@@ -77,9 +77,11 @@ export class WebPlaywrightLocal {
         const page = await this.browser.currentPage;
         const t0 = Date.now();
         logger.debug(`[click] start selector="${selector}"${options?.useKeyboard ? ' mode=keyboard' : ''}${options?.useJsDispatch ? ' mode=js_dispatch' : ''}${options?.coordinates ? ` mode=coords(${options.coordinates.x},${options.coordinates.y})` : ''}`);
+        const fireClickSfx = () => options?.onClick?.(Date.now());
         try {
             if (options?.coordinates) {
                 await this.moveClipCursorToPoint(options.coordinates);
+                fireClickSfx();
                 await this.browser.clickByCoordinates(options.coordinates.x, options.coordinates.y);
                 logger.debug(`[click] done coords took ${Date.now() - t0}ms`);
                 return;
@@ -88,11 +90,13 @@ export class WebPlaywrightLocal {
             const animatedTarget = await this.moveClipCursorToLocator(locator);
             if (options?.useKeyboard) {
                 await locator.focus();
+                fireClickSfx();
                 await page.keyboard.press('Enter');
                 logger.debug(`[click] done keyboard took ${Date.now() - t0}ms`);
                 return;
             }
             if (options?.useJsDispatch) {
+                fireClickSfx();
                 await locator.dispatchEvent('click');
                 logger.debug(`[click] done js_dispatch took ${Date.now() - t0}ms`);
                 return;
@@ -103,6 +107,7 @@ export class WebPlaywrightLocal {
                 ? await this.relativeClickPosition(locator, animatedTarget)
                 : null;
             if (options?.button && options.button !== 'left') {
+                fireClickSfx();
                 await locator.click({
                     button: options.button,
                     timeout: 5000,
@@ -113,6 +118,7 @@ export class WebPlaywrightLocal {
                 return;
             }
             if (clickPosition) {
+                fireClickSfx();
                 await locator.click({
                     timeout: 5000,
                     force: options?.force,
@@ -120,6 +126,7 @@ export class WebPlaywrightLocal {
                 });
             }
             else {
+                fireClickSfx();
                 await this.browser.clickBySelector(selector, { force: options?.force });
             }
             await this.emitClipClickPulse();
@@ -144,6 +151,7 @@ export class WebPlaywrightLocal {
         const position = target
             ? await this.relativeClickPosition(resolved.locator, target)
             : null;
+        opts.onClick?.(Date.now());
         await resolved.locator.click({
             timeout: 5000,
             ...(position ? { position } : {}),
@@ -633,7 +641,7 @@ export class WebPlaywrightLocal {
             optionIndex: option.index,
         });
     }
-    async check(selector, checked) {
+    async check(selector, checked, actionOpts) {
         const page = await this.browser.currentPage;
         const locator = page.locator(selector).first();
         const target = await this.moveClipCursorToLocator(locator);
@@ -641,6 +649,7 @@ export class WebPlaywrightLocal {
             ? await this.relativeClickPosition(locator, target)
             : null;
         const opts = { timeout: 5000, ...(position ? { position } : {}) };
+        actionOpts?.onClick?.(Date.now());
         if (checked) {
             await locator.check(opts);
         }
@@ -648,13 +657,14 @@ export class WebPlaywrightLocal {
             await locator.uncheck(opts);
         }
     }
-    async doubleClick(selector) {
+    async doubleClick(selector, actionOpts) {
         const page = await this.browser.currentPage;
         const locator = page.locator(selector).first();
         const target = await this.moveClipCursorToLocator(locator);
         const position = target
             ? await this.relativeClickPosition(locator, target)
             : null;
+        actionOpts?.onClick?.(Date.now());
         await locator.dblclick({
             timeout: 5000,
             ...(position ? { position } : {}),
@@ -895,7 +905,10 @@ export class WebPlaywrightLocal {
         }
         await page.waitForTimeout(70);
         await humanType(page, text, this.clipCursor
-            ? { minDelayMs: 20, maxDelayMs: 45, onKeystroke }
+            // Demo-video cadence: ~80-180ms between keys (≈ 80-100 WPM with
+            // natural variation). Faster than that reads as robotic in the
+            // mixed video + keyboard SFX track.
+            ? { minDelayMs: 80, maxDelayMs: 180, onKeystroke }
             : { onKeystroke });
     }
     async seedClipCursor(position) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autokap",
-  "version": "1.5.2",
+  "version": "1.5.3",
   "description": "AI-powered CLI tool for capturing clean screenshots of websites",
   "type": "module",
   "main": "./dist/index.js",