autokap 1.2.0 → 1.3.1

package/assets/skill/OPCODE-REFERENCE.md

@@ -1,6 +1,6 @@
 # AutoKap Opcode Reference
 
-Detailed parameter documentation for all 23 opcodes. For workflow, rules, and examples, see [SKILL.md](SKILL.md).
+Detailed parameter documentation for all 24 opcodes. For workflow, rules, and examples, see [SKILL.md](SKILL.md).
 
 ## Common Fields (all opcodes)
 
@@ -150,6 +150,23 @@ Wait for an element to appear.
 { "kind": "WAIT_FOR", "selector": "[data-ak=\"dashboard\"]", "state": "visible", "postcondition": { "type": "element_visible", "selector": "[data-ak=\"dashboard\"]", "waitMs": 10000 } }
 ```
 
+## SLEEP
+
+Pause execution for a fixed duration. For demo videos, use SLEEP as an explicit narration anchor with `stepId`, `narrationTextByLocale`, and `durationMs: 1`. `autokap run` generates TTS once per app-locale voice row, measures the audio, and rewrites `durationMs` to the maximum duration for that `stepId` before executing the opcodes, so one runtime program works for every language/theme variant.
+
+| Param | Type | Required | Default | Description |
+|-------|------|----------|---------|-------------|
+| `durationMs` | integer (1..60000) | yes | — | How long to sleep, in milliseconds |
+| `narrationTextByLocale` | object `{ [locale]: string }` | video narration | — | Spoken text for every TTS locale used by `narration_by_app_locale`. Current TTS catalogue supports `en` and `fr`. |
+| `narrationText` | string | legacy single-row only | — | Backward-compatible fallback when only one TTS row is requested. Prefer `narrationTextByLocale`. |
+
+**Can:** Hold the runtime for a precise wall-clock duration without DOM polling.
+**Cannot:** React to a selector / postcondition — use `WAIT_FOR` for that.
+
+```json
+{ "kind": "SLEEP", "description": "Hold for narration on 'open-pricing'", "stepId": "open-pricing", "durationMs": 1, "narrationTextByLocale": { "en": "Let's open pricing and look at the plan structure.", "fr": "On ouvre les tarifs et on regarde la structure des offres." }, "postcondition": { "type": "always" }, "recovery": { "retries": 0, "useSelectorMemory": false, "useAltInteraction": false, "allowReload": false, "allowHealer": false }, "timeoutMs": 65000, "maxFailures": 1 }
+```
+
 ## SCROLL
 
 Scroll the page or scroll an element into view.

package/assets/skill/SKILL.md

@@ -90,7 +90,7 @@ For every element you interact with in the opcodes, add a `data-ak="descriptive-
 interface ExecutionProgram {
   presetId: string;         // Unique slug (e.g. "homepage-hero")
   programVersion: number;   // Always 1 for new programs
-  mediaMode: 'screenshot' | 'clip';
+  mediaMode: 'screenshot' | 'clip' | 'video';
   baseUrl: string;          // Root URL of the application
   variants: VariantSpec[];  // Viewport/locale/theme combinations
   preconditions: {
@@ -104,11 +104,17 @@ interface ExecutionProgram {
   };
   steps: ExecutionOpcode[];
   artifactPlan: {
-    mediaMode: 'screenshot' | 'clip';
-    cursorTheme?: 'minimal' | 'macos' | 'windows'; // Clip only. Default: 'minimal'
+    mediaMode: 'screenshot' | 'clip' | 'video';
+    cursorTheme?: 'minimal' | 'macos' | 'windows'; // Clip/video recordings. Default: 'minimal'
     format?: {
       clipFormat?: 'gif' | 'mp4' | 'both';      // Default: 'gif'
       screenshotFormat?: 'png' | 'jpeg';          // Default: 'png'
+      // Video-only — required when mediaMode='video'.
+      // captureResolution must be 1920×1080; legacy 2560×1440 is normalized by AutoKap.
+      // captureFps defaults to 30; deliveryResolution defaults to 1920×1080.
+      captureResolution?: { width: number; height: number };
+      captureFps?: number;
+      deliveryResolution?: { width: number; height: number };
     };
     applyMockup?: boolean;     // Apply device frame mockup
     applyStatusBar?: boolean;  // Add status bar to mockup
@@ -133,7 +139,7 @@ interface VariantSpec {
 
 ## Opcode Quick Reference
 
-23 opcodes available. For full parameter documentation, see [OPCODE-REFERENCE.md](OPCODE-REFERENCE.md).
+24 opcodes available. For full parameter documentation, see [OPCODE-REFERENCE.md](OPCODE-REFERENCE.md).
 
 | Kind | Selector? | Key Params | Typical Postcondition | Notes |
 |------|-----------|-----------|----------------------|-------|
@@ -143,6 +149,7 @@ interface VariantSpec {
 | `TYPE` | yes | `text`, `clearFirst` | `any_change` | `{{email}}` / `{{password}}` for creds |
 | `PRESS_KEY` | no | `key` | `any_change` | `"Enter"`, `"Escape"`, `"Tab"`, etc. |
 | `WAIT_FOR` | yes* | `state` | `element_visible` | `"visible"` or `"attached"` (DOM only) |
+| `SLEEP` | no | `durationMs` (1..60000), `narrationTextByLocale?` | `always` | Pause N ms. Reserved for video narration anchors. Use `durationMs: 1` as a placeholder; AutoKap rewrites it during `autokap run` after generating TTS. |
 | `SCROLL` | no | `direction`, `targetSelector?`, `amount?` | `element_visible` | Use `targetSelector` for precise scroll |
 | `HOVER` | yes | — | `element_visible` | Capture immediately after for hover state |
 | `SELECT_OPTION` | yes | `optionLabel` / `optionValue` / `optionIndex` | `text_contains` | **Native `<select>` only.** Custom dropdowns: use CLICK sequence |
@@ -217,7 +224,7 @@ interface VariantSpec {
 | Tell the user to save a local `program.json` file | Persist the preset via CLI; only output full JSON as a fallback if the CLI/server write fails |
 | Guess CSS selectors | Add `data-ak` attributes to the code first |
 | Skip WAIT_FOR after page transitions | Add WAIT_FOR with `waitMs: 10000` after login, navigation, modal open |
-| Use NAVIGATE or SCROLL inside a clip to reach a clickable target | CLICK the link/button — cursor animates to it naturally |
+| Use NAVIGATE inside a clip (between BEGIN_CLIP and END_CLIP) | Always CLICK the in-app element (sidebar Link, modal trigger button) or PRESS_KEY Escape to close modals. NAVIGATE is `page.goto` = full reload + white flash + cursor lost. See "Designing human-like clip programs" below for the patterns |
 | Skip `cursorTheme` in clip `artifactPlan` | Set `cursorTheme: "macos"` or `"windows"` for polished recordings |
 
 ## Clip Workflow
@@ -243,17 +250,53 @@ Because the cursor is visible during recording, **how** you reach a target matte
 | Do | Don't | Why |
 |---|---|---|
 | **CLICK** on a nav link / anchor to scroll to a section | SCROLL blindly then capture | The cursor animates to the link before clicking — the viewer sees intentional navigation |
-| **CLICK** on an in-page link to change route | NAVIGATE to the new URL mid-clip | NAVIGATE is an instant browser jump with no cursor motion — it looks like a cut, not a flow |
+| **CLICK** on an in-page `<Link>` (sidebar, breadcrumb, card) to change route | NAVIGATE to the new URL mid-clip | NAVIGATE always calls Playwright's `page.goto` — a full HTTP navigation that destroys the document, wipes the cursor overlay, and shows a white flash. Even SPAs are forced into a hard reload. CLICK lets Next.js / React Router intercept and use prefetched `router.push` instead — no flash, cursor preserved |
+| **CLICK** on the modal's trigger button (e.g. `[data-ak-interact="new-preset-btn"]`) to open a modal | NAVIGATE to the modal's deep-link route | Most modals open via local React state, not URL navigation. The deep-link route exists for direct entry but forces a real reload — using the trigger button keeps the recording in one fluid take |
+| **PRESS_KEY** `"Escape"` to close a modal (or CLICK its close button) | NAVIGATE back to the previous URL | Radix/shadcn dialogs (the AutoKap dashboard convention) close on Escape and run their `onClose` handler — which usually does the SPA route change internally. NAVIGATE back triggers another full reload |
 | **HOVER** → pause → **CLICK** for important interactions | CLICK without HOVER | HOVER gives the viewer time to see where the cursor is heading and creates anticipation |
 | Keep the clip short and focused (5–15 interactions) | Record an entire user journey in one clip | Long clips lose the viewer. Split into multiple clips if needed |
 | Place WAIT_FOR after CLICK that triggers a route change | Immediately interact after navigation | Gives the page time to render and the viewer time to register the new state |
 | Add a `screenshot_stable` WAIT_FOR before BEGIN_CLIP when DISMISS_OVERLAYS precedes it | Start recording immediately after DISMISS_OVERLAYS | Overlay dismiss animations (fade-out, slide) bleed into the first frames of the clip |
 
-**Rule of thumb:** inside a clip, never use NAVIGATE or SCROLL to reach content that the user would normally reach by clicking a link or button. Use CLICK on that element instead — the cursor animation makes the transition visible and human-like.
+**Rule of thumb — NEVER `NAVIGATE` inside a clip.** NAVIGATE = `page.goto()` = full document reload, white flash, cursor overlay destroyed. There is no exception, even when the destination is on the same origin or the framework supports SPA routing — Playwright doesn't use the framework router. Reach every destination through the actual UI element a real user would interact with:
 
-SCROLL is still appropriate for:
-- Scrolling down a long page to reveal below-the-fold content when there is no clickable anchor
-- Scrolling inside a container or list
+- **Page change** → `CLICK` on a `<Link>` (sidebar item, breadcrumb, card). If no Link points where you need to go, ask the user where the user-facing entry point is — don't invent a NAVIGATE workaround.
+- **Open a modal** → `CLICK` on its trigger button (annotate it with `data-ak-interact="..."`). Even if the modal also has a deep-link URL, the trigger is the smooth path.
+- **Close a modal** → `PRESS_KEY "Escape"` (or CLICK the close button if present). The modal's `onClose` handler does any required SPA route change for you.
+- **Reveal off-screen content** → `SCROLL` is still fine when there's no clickable anchor (long content sections, lists inside a container).
+
+If you find yourself wanting to write `NAVIGATE` between `BEGIN_CLIP` and `END_CLIP`, stop and reconsider — there is almost always an annotated UI element to CLICK instead. Add a `data-ak-interact` to the source if it's missing.
+
+## Demo Video Workflow
+
+For narrated demo videos (`mediaMode: "video"`), AutoKap captures a language/theme matrix:
+
+1. Use **one desktop target** at 1920x1080. Do not multiply demo videos by device.
+2. Generate one variant for every app locale x theme pair. Variant ids must be stable, for example `desktop-1920x1080-fr-dark`.
+3. Add `SET_LOCALE` and `SET_THEME` with `"$variant"` during setup when the app supports locale/theme switching.
+4. Put all recorded content inside `BEGIN_CLIP` / `END_CLIP`.
+5. Every narrated pause is a `SLEEP` with `stepId`, `durationMs: 1`, and `narrationTextByLocale`.
+
+Voice-over settings are row-based: each app locale maps to a spoken TTS locale and a voice in `narration_by_app_locale`. `narrationTextByLocale` is the source of truth for TTS text. It must include one entry for every spoken TTS locale used by those rows. AutoKap currently supports TTS for `en` and `fr`; unsupported TTS locales are rejected explicitly. Keep `narrationText` only as a legacy fallback for single-row demos.
+
+```json
+{
+  "kind": "SLEEP",
+  "stepId": "opening-hello",
+  "durationMs": 1,
+  "narrationTextByLocale": {
+    "en": "Hey, welcome — I'll show you the dashboard quickly.",
+    "fr": "Salut, bienvenue — je te montre rapidement le dashboard."
+  },
+  "description": "Narration anchor: opening greeting",
+  "postcondition": { "type": "always" },
+  "recovery": { "retries": 0, "useSelectorMemory": false, "useAltInteraction": false, "allowReload": false, "allowHealer": false },
+  "timeoutMs": 65000,
+  "maxFailures": 1
+}
+```
+
+The server synthesizes TTS once per app-locale voice row, not once per theme. It rewrites every narrated `SLEEP.durationMs` to the maximum audio duration for that `stepId` across rows so the same runtime program can capture every variant.
 
 ### Clean start: wait for visual stability before recording
 
@@ -291,7 +334,7 @@ When an opcode fails, AutoKap tries 5 recovery strategies in order:
 4. **Targeted reload** — Reloads the page and retries. **Loses UI state** (open modals, form data).
 5. **LLM Healer** — AI analyzes the page screenshot and AKTree, then rewrites the failing opcode. Max 3 invocations per run.
 
-**Guidance:** Keep `allowReload: false` for most steps (it loses state). Set `allowReload: true` only for the first NAVIGATE or after full page transitions where UI state is expendable.
+**Guidance:** Keep `allowReload: false` for most steps (it loses state). Set `allowReload: true` only for the first NAVIGATE or after full page transitions where UI state is expendable. Never rely on reload recovery inside a `BEGIN_CLIP` / `END_CLIP` recording window.
 
 ## SET_LOCALE / SET_THEME: Method Selection
 

package/dist/browser.js

@@ -96,7 +96,7 @@ function resolveEffectivePadding(config, bbox) {
         left = config.paddingLeft;
     return { top, right, bottom, left };
 }
-import { dismissCookiesAndWidgets, ensureCaptureHideStyles } from './cookie-dismiss.js';
+import { CAPTURE_HIDE_STYLE_ID, dismissCookiesAndWidgets, ensureCaptureHideStyles, getCaptureHideCSS, } from './cookie-dismiss.js';
 import { CHROMIUM_ARGS, browserPool } from './browser-pool.js';
 import { isDebugEnabled, logger } from './logger.js';
 async function withHelperTimeout(label, timeoutMs, work) {
@@ -841,6 +841,28 @@ export class Browser {
         });
         // Inject cursor overlay at context level — survives all navigations in this session
         await instance.context.addInitScript(cursorScript);
+        // Also hide dev/prototype chrome on every document, including navigations
+        // that happen after cookie dismissal ran.
+        await instance.context.addInitScript(({ styleId, css }) => {
+            const install = () => {
+                const parent = document.head ?? document.documentElement;
+                if (!parent)
+                    return;
+                let style = document.getElementById(styleId);
+                if (!style) {
+                    style = document.createElement('style');
+                    style.id = styleId;
+                    parent.appendChild(style);
+                }
+                if (style.textContent !== css) {
+                    style.textContent = css;
+                }
+            };
+            install();
+            if (document.readyState === 'loading') {
+                document.addEventListener('DOMContentLoaded', install, { once: true });
+            }
+        }, { styleId: CAPTURE_HIDE_STYLE_ID, css: getCaptureHideCSS() });
         instance.page = await instance.context.newPage();
         return instance;
     }

package/dist/capture-strategy.d.ts

@@ -27,4 +27,18 @@ export declare class ClipStrategy implements CaptureStrategy {
     capture(adapter: RuntimeAdapter, _spec: ArtifactSpec): Promise<ArtifactResult>;
     postProcess(artifact: ArtifactResult, spec: ArtifactSpec): Promise<ArtifactResult>;
 }
+/**
+ * Long-form MP4 capture for `mediaMode='video'`. Shares the clip pipeline
+ * (frame loop + ffmpeg assembly) but pins capture resolution to the delivery
+ * frame (1920×1080 by default) at 30fps and removes the clip duration cap.
+ * The bifurcation actually lives in
+ * `WebPlaywrightLocal.beginRecording()` — this strategy is the typed surface
+ * the orphan capture-strategy factory exposes.
+ */
+export declare class VideoStrategy implements CaptureStrategy {
+    readonly mediaMode: "video";
+    prepare(_adapter: RuntimeAdapter, _spec: ArtifactSpec): Promise<void>;
+    capture(adapter: RuntimeAdapter, _spec: ArtifactSpec): Promise<ArtifactResult>;
+    postProcess(artifact: ArtifactResult, _spec: ArtifactSpec): Promise<ArtifactResult>;
+}
 export declare function createCaptureStrategy(mediaMode: MediaMode): CaptureStrategy;

package/dist/capture-strategy.js

@@ -58,11 +58,39 @@ export class ClipStrategy {
         return artifact;
     }
 }
+// ── Video strategy ──────────────────────────────────────────────────
+/**
+ * Long-form MP4 capture for `mediaMode='video'`. Shares the clip pipeline
+ * (frame loop + ffmpeg assembly) but pins capture resolution to the delivery
+ * frame (1920×1080 by default) at 30fps and removes the clip duration cap.
+ * The bifurcation actually lives in
+ * `WebPlaywrightLocal.beginRecording()` — this strategy is the typed surface
+ * the orphan capture-strategy factory exposes.
+ */
+export class VideoStrategy {
+    mediaMode = 'video';
+    async prepare(_adapter, _spec) {
+        // Resolution + fps are forced inside adapter.beginRecording when the
+        // BEGIN_CLIP opcode runs in video mode. Nothing to do here.
+    }
+    async capture(adapter, _spec) {
+        const recording = await adapter.endRecording();
+        return {
+            mediaMode: 'video',
+            buffer: recording.buffer,
+            mimeType: recording.mimeType,
+        };
+    }
+    async postProcess(artifact, _spec) {
+        return artifact;
+    }
+}
 // ── Factory ─────────────────────────────────────────────────────────
 export function createCaptureStrategy(mediaMode) {
     switch (mediaMode) {
         case 'screenshot': return new ScreenshotStrategy();
         case 'clip': return new ClipStrategy();
+        case 'video': return new VideoStrategy();
     }
 }
 //# sourceMappingURL=capture-strategy.js.map

package/dist/cli-contract.d.ts

@@ -18,3 +18,64 @@ export declare function buildCliRunCommand(presetId: string, options?: {
 }): string;
 export declare function buildCliInstalledSetupCommand(cliKey: string): string;
 export declare const CLI_PUBLIC_COMMANDS: CliPublicCommandDescriptor[];
+/**
+ * Per-clip metadata uploaded by the CLI alongside the raw MP4. The compositor
+ * worker consumes these to drive the merge step in clipId order. Opcode
+ * timings are retained for diagnostics and future compositor effects.
+ */
+export interface VideoClipMetadata {
+    /** Variant that produced this raw clip. Required for video-demo matrices. */
+    variantId: string;
+    lang: string;
+    theme: 'light' | 'dark';
+    clipId: string;
+    /** Storage path within the `videos` bucket (e.g. `raw/{video_id}/{clip_id}.mp4`). */
+    mp4StoragePath: string;
+    /** Measured duration of the captured MP4 in milliseconds. */
+    durationMs: number;
+    /**
+     * Per-opcode timing entries that landed within this clip, taken verbatim
+     * from `RunResult.opcodeTimings` (see execution-types). The compositor
+     * filters by `clipId === clip.clipId`.
+     */
+    opcodeTimings: Array<{
+        stepIndex: number;
+        stepId?: string;
+        opcodeKind: string;
+        timecodeStartMs: number;
+        timecodeEndMs: number;
+        bbox?: {
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        } | null;
+    }>;
+}
+export interface VideoAudioAsset {
+    stepId: string;
+    url: string;
+    duration_ms: number;
+    word_timings?: Array<{
+        word: string;
+        start_ms: number;
+        end_ms: number;
+    }>;
+}
+export type VideoAudioAssetsByLocale = Record<string, VideoAudioAsset[]>;
+/** Body of POST /api/cli/video-complete. */
+export interface VideoCompletePayload {
+    videoId: string;
+    /** `runId` the CLI generated for this run (already used for telemetry). */
+    runId: string;
+    clips: VideoClipMetadata[];
+    /** Per-run TTS assets generated by /api/cli/video-prepare. */
+    audioAssets?: VideoAudioAsset[];
+    /** Per-locale TTS assets generated by /api/cli/video-prepare. */
+    audioAssetsByLocale?: VideoAudioAssetsByLocale;
+}
+/** Server response. */
+export interface VideoCompleteResponse {
+    composeRunId: string;
+    composeRunIds?: string[];
+}

package/dist/cli-runner.d.ts

@@ -11,7 +11,15 @@
  * 6. Upload artifacts + telemetry to server
  */
 import { type ProgressEvent } from './opcode-runner.js';
-import type { ExecutionProgram, RunResult } from './execution-types.js';
+import { type VideoClipMetadata } from './cli-contract.js';
+import type { ExecutionProgram, VariantSpec, RunResult } from './execution-types.js';
+export interface RecordableBrowserSettings {
+    viewport: VariantSpec['viewport'];
+    requestedDeviceScaleFactor: number;
+    runtimeDeviceScaleFactor: number;
+}
+export declare function resolveRecordableBrowserSettings(program: ExecutionProgram, variant: VariantSpec): RecordableBrowserSettings;
+export declare function normalizeVideoCaptureProgram(program: ExecutionProgram): ExecutionProgram;
 export interface CLIRunnerOptions {
     /** Preset ID to run */
     presetId: string;
@@ -36,3 +44,4 @@ export interface CLIRunResult {
     error?: string;
 }
 export declare function runCapture(options: CLIRunnerOptions): Promise<CLIRunResult>;
+export declare function buildVideoClipMetadata(videoId: string, result: RunResult, program?: ExecutionProgram, runId?: string): VideoClipMetadata[];