autokap 1.2.0 → 1.3.1

package/dist/video-narration-schema.js

@@ -0,0 +1,137 @@
+/**
+ * AUT-57 — Video narration overlay schema.
+ *
+ * The opcode-based video demo feature stores a regular `ExecutionProgram`
+ * in a preset config. Narration is authored on `SLEEP` opcodes via stable
+ * `stepId` anchors plus `narrationTextByLocale`; `autokap run` extracts those
+ * anchors into this overlay shape, renders TTS, then persists the overlay and
+ * audio assets under `presets.config.videoDemo`.
+ */
+import { z } from 'zod';
+import { ExecutionProgramSchema } from './execution-schema.js';
+export const SUPPORTED_TTS_LOCALES = ['en', 'fr'];
+export function normalizeLocaleTag(locale) {
+    return locale.trim().replace(/_/g, '-');
+}
+export function primaryLocaleSubtag(locale) {
+    return normalizeLocaleTag(locale).split('-')[0]?.toLowerCase() ?? '';
+}
+export function resolveSupportedTtsLocale(locale) {
+    const primary = primaryLocaleSubtag(locale);
+    return SUPPORTED_TTS_LOCALES.includes(primary)
+        ? primary
+        : null;
+}
+/** `stepId` constraint shared with `OpcodeBase` — kebab-case, 1..64 chars. */
+const NarrationStepIdSchema = z.string()
+    .regex(/^[a-z0-9-]+$/, {
+    message: 'stepId must match /^[a-z0-9-]+$/ (lowercase kebab-case)',
+})
+    .min(1)
+    .max(64);
+/**
+ * One narration segment anchored to a single opcode `stepId`. The TTS provider
+ * receives `text` verbatim; `estimated_duration_ms` is the assistant's guess
+ * (used for diagnostics, not authoritative — the run-time TTS step
+ * (PR #4) measures the real audio length via ffprobe).
+ */
+export const NarrationSegmentSchema = z.object({
+    stepId: NarrationStepIdSchema,
+    text: z.string().min(1).max(2000),
+    estimated_duration_ms: z.number().int().positive().max(60_000),
+}).strict();
+/**
+ * Locale-aware narration overlay. `voice` is an opaque provider-side handle
+ * (e.g. `openai/gpt-4o-mini-tts:nova`); the backend validates it against the
+ * cached OpenRouter voice list (PR #3+).
+ *
+ * TTS support is intentionally catalog-gated: app locales can be arbitrary,
+ * but narration locales must currently resolve to `en` or `fr`.
+ */
+export const VideoNarrationOverlaySchema = z.object({
+    voice: z.string().min(1).max(128),
+    locale: z.string().min(2).max(16),
+    segments: z.array(NarrationSegmentSchema).min(1).max(200),
+}).strict().superRefine((value, ctx) => {
+    if (!resolveSupportedTtsLocale(value.locale)) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ['locale'],
+            message: `unsupported TTS locale "${value.locale}" — supported locales: ${SUPPORTED_TTS_LOCALES.join(', ')}`,
+        });
+    }
+    const seen = new Set();
+    for (const [index, seg] of value.segments.entries()) {
+        if (seen.has(seg.stepId)) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ['segments', index, 'stepId'],
+                message: `duplicate stepId in narration: "${seg.stepId}"`,
+            });
+        }
+        seen.add(seg.stepId);
+    }
+});
+/**
+ * Legacy ingest payload shape retained for compatibility tests. The current
+ * demo-video flow persists the program as `presets.config.program`; narration
+ * is extracted from `SLEEP.narrationTextByLocale` during `autokap run`.
+ *
+ * Cross-validation:
+ *   1. `program.mediaMode === 'video'`.
+ *   2. When `narration` is present, every `narration.segments[*].stepId` must
+ *      match a `program.steps[*].stepId`.
+ */
+export const VideoIngestPayloadSchema = z.object({
+    program: ExecutionProgramSchema,
+    narration: VideoNarrationOverlaySchema.optional(),
+}).strict().superRefine((value, ctx) => {
+    if (value.program.mediaMode !== 'video') {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ['program', 'mediaMode'],
+            message: `ingest payload requires program.mediaMode='video', got '${value.program.mediaMode}'`,
+        });
+        return;
+    }
+    if (!value.narration)
+        return;
+    const programStepIds = new Set();
+    // value.program here is `unknown`-shaped from Zod's perspective at refine
+    // time, but the ExecutionProgramSchema has already passed by the time we
+    // run — narrowing via cast is safe.
+    for (const op of value.program.steps) {
+        if (typeof op.stepId === 'string') {
+            programStepIds.add(op.stepId);
+        }
+    }
+    for (const [index, seg] of value.narration.segments.entries()) {
+        if (!programStepIds.has(seg.stepId)) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ['narration', 'segments', index, 'stepId'],
+                message: `narration references unknown stepId "${seg.stepId}" — no opcode in program.steps carries that stepId`,
+            });
+        }
+    }
+});
+/** Parse an ingest payload, throwing a ZodError on invalid input. */
+export function parseVideoIngestPayload(data) {
+    return VideoIngestPayloadSchema.parse(data);
+}
+/** Status state machine values for `videos.status` (opcode flow). */
+export const VIDEO_STATUSES = [
+    'draft',
+    'preparing',
+    'ready_to_run',
+    'running',
+    'uploading',
+    'compositing',
+    'done',
+    'failed',
+];
+/** Discriminator value for opcode-flow rows in the `videos` table. */
+export const VIDEO_FLOW_KIND_OPCODE = 'opcode_video';
+/** Discriminator value for legacy planner rows (migration 020 flow). */
+export const VIDEO_FLOW_KIND_LEGACY = 'legacy_planner';
+//# sourceMappingURL=video-narration-schema.js.map

package/dist/web-playwright-local.d.ts

@@ -14,6 +14,16 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
     private readonly sessionStartedAt;
     private recording;
     private clipCursor;
+    /**
+     * Flags / hooks that surface mid-recording navigations in CLI logs. When a
+     * navigation (NAVIGATE opcode, SET_LOCALE storage reload) is initiated by
+     * the runner, `expectedNavigationCount` is bumped before the call and
+     * decremented when the framenavigated handler observes it — so any nav that
+     * fires WITHOUT a pending count is logged as UNEXPECTED (likely page-side
+     * `location.reload()`, session redirect, or HMR). `detach()` removes the
+     * listeners when recording ends.
+     */
+    private recordingNavWatcher;
     constructor(browser: Browser, recordingDir?: string | undefined);
     navigate(url: string): Promise<void>;
     getCurrentUrl(): Promise<string>;
@@ -50,6 +60,12 @@ export declare class WebPlaywrightLocal implements RuntimeAdapter {
     takeElementScreenshot(selector: string): Promise<Buffer>;
     takeCleanScreenshot(): Promise<Buffer>;
     beginRecording(options: RecordingOptions): Promise<void>;
+    getElementBoundingBox(selector: string): Promise<{
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    } | null>;
     endRecording(): Promise<RecordingResult>;
     setLocale(locale: string): Promise<void>;
     setColorScheme(scheme: 'light' | 'dark'): Promise<void>;

package/dist/web-playwright-local.js

@@ -7,23 +7,60 @@
 import fs from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
-import { humanType, moveMouse } from './mouse-animation.js';
+import { humanType, moveMouse, } from './mouse-animation.js';
 import { resolveTarget } from './semantic-resolver.js';
 import { logger } from './logger.js';
 import { ClipCaptureLoop } from './clip-capture-loop.js';
-import { assembleMp4FromFrames } from './clip-postprocess.js';
+import { assembleMp4FromFrames, getMediaDurationMs } from './clip-postprocess.js';
 export class WebPlaywrightLocal {
     browser;
     recordingDir;
     sessionStartedAt = Date.now();
     recording = null;
     clipCursor = null;
+    /**
+     * Flags / hooks that surface mid-recording navigations in CLI logs. When a
+     * navigation (NAVIGATE opcode, SET_LOCALE storage reload) is initiated by
+     * the runner, `expectedNavigationCount` is bumped before the call and
+     * decremented when the framenavigated handler observes it — so any nav that
+     * fires WITHOUT a pending count is logged as UNEXPECTED (likely page-side
+     * `location.reload()`, session redirect, or HMR). `detach()` removes the
+     * listeners when recording ends.
+     */
+    recordingNavWatcher = null;
     constructor(browser, recordingDir) {
         this.browser = browser;
         this.recordingDir = recordingDir;
     }
     async navigate(url) {
-        await this.browser.navigateTo(url);
+        // Snapshot the cursor position BEFORE navigating — `page.goto` replaces
+        // the document and wipes the `<div id="__ak_cursor__">` overlay element.
+        // The cursor init script is re-run on the new document but the cursor
+        // starts off-screen until the next interaction. During a long SLEEP
+        // narration that follows a NAVIGATE, no cursor would be visible at all.
+        const lastCursorPosition = this.recording && this.clipCursor
+            ? this.clipCursor.currentPosition
+            : null;
+        if (this.recordingNavWatcher)
+            this.recordingNavWatcher.expectedNavigationCount += 1;
+        try {
+            await this.browser.navigateTo(url);
+        }
+        finally {
+            // Drop the budget after a brief settle window — Playwright's framenavigated
+            // for the load can fire after navigateTo's promise resolves on slow apps.
+            if (this.recordingNavWatcher) {
+                const watcher = this.recordingNavWatcher;
+                setTimeout(() => {
+                    if (watcher === this.recordingNavWatcher) {
+                        watcher.expectedNavigationCount = Math.max(0, watcher.expectedNavigationCount - 1);
+                    }
+                }, 500);
+            }
+        }
+        if (this.recording && this.clipCursor) {
+            await this.seedClipCursor(lastCursorPosition ?? undefined);
+        }
     }
     async getCurrentUrl() {
         const page = await this.browser.currentPage;
@@ -273,16 +310,55 @@ export class WebPlaywrightLocal {
         return this.browser.takeScreenshot();
     }
     async beginRecording(options) {
-        const page = await this.browser.currentPage;
+        let page = await this.browser.currentPage;
+        if (options.captureResolution) {
+            const alreadyMatchesCaptureSurface = options.mediaMode === 'video'
+                ? await captureSurfaceMatches(page, options.captureResolution)
+                : false;
+            if (!alreadyMatchesCaptureSurface) {
+                await this.browser.resizeViewport(options.captureResolution.width, options.captureResolution.height);
+                page = await this.browser.currentPage;
+            }
+        }
+        // Surface diagnostic — captures what frames will actually be at, so a
+        // mismatch between the configured 1920×1080 and the rendered surface is
+        // visible in CLI logs without instrumenting the user's machine.
+        try {
+            const surfaceCheck = await page.evaluate(() => ({
+                innerWidth: window.innerWidth,
+                innerHeight: window.innerHeight,
+                dpr: window.devicePixelRatio,
+            }));
+            const capturedW = Math.round(surfaceCheck.innerWidth * surfaceCheck.dpr);
+            const capturedH = Math.round(surfaceCheck.innerHeight * surfaceCheck.dpr);
+            logger.info(`[capture] Recording surface: CSS ${surfaceCheck.innerWidth}×${surfaceCheck.innerHeight} ` +
+                `@DPR=${surfaceCheck.dpr} → captured frames ${capturedW}×${capturedH} ` +
+                `(requested ${options.captureResolution?.width ?? 'auto'}×${options.captureResolution?.height ?? 'auto'})`);
+        }
+        catch {
+            // Best-effort diagnostic — non-fatal.
+        }
         const baseDir = this.recordingDir
             ?? await fs.mkdtemp(path.join(os.tmpdir(), 'autokap-recording-'));
         const framesDir = path.join(baseDir, 'frames');
         await fs.mkdir(framesDir, { recursive: true });
+        // Linux defaults are conservative because GitHub Actions free runners
+        // (2 vCPU) can't sustain 30 fps. AUTOKAP_CLOUD_RUNNER=1 signals that the
+        // process is running on managed cloud infra (Fly.io machines, ≥4 vCPU)
+        // where the cap can safely lift to 30 fps for clips too. Set by the
+        // `--cloud` flag and by the cloud-runner Docker image.
+        const isCloudRunner = process.env.AUTOKAP_CLOUD_RUNNER === '1';
+        const defaultFps = process.platform === 'linux' && !isCloudRunner ? 8 : 15;
+        const cloudClipFps = isCloudRunner ? 30 : defaultFps;
+        const targetFps = options.captureFps
+            ?? (options.mediaMode === 'video' ? 30 : cloudClipFps);
         const loop = new ClipCaptureLoop({
             page,
             framesDir,
-            targetFps: process.platform === 'linux' ? 8 : 15,
-            minRestMs: process.platform === 'linux' ? 50 : 16,
+            targetFps,
+            // Cloud runners have CPU headroom — drop the Linux 50 ms idle cushion
+            // (sized for tight CI runners) to let the loop stay close to its target.
+            minRestMs: process.platform === 'linux' && !isCloudRunner ? 50 : 16,
         });
         await loop.start();
         this.recording = {
@@ -293,22 +369,97 @@ export class WebPlaywrightLocal {
             loop,
             finalized: false,
         };
-        this.clipCursor = { currentPosition: null };
+        this.clipCursor = {
+            currentPosition: null,
+            pace: options.mediaMode === 'video' ? 'natural' : 'fast',
+        };
+        // Attach navigation/error watchers for the recording window. These surface
+        // the cause of mid-recording refreshes (cursor disappears, "robot" effect)
+        // that the existing reload guards don't catch — page-side `location.reload()`,
+        // session redirects, Next.js HMR, soft router refreshes.
+        const recordingStartedAt = Date.now();
+        let lastUrl = page.url();
+        const navHandler = (frame) => {
+            if (frame !== page.mainFrame())
+                return;
+            const elapsedMs = Date.now() - recordingStartedAt;
+            const url = frame.url();
+            const sameUrl = url === lastUrl;
+            lastUrl = url;
+            const watcher = this.recordingNavWatcher;
+            const expected = watcher && watcher.expectedNavigationCount > 0;
+            if (expected && watcher) {
+                watcher.expectedNavigationCount = Math.max(0, watcher.expectedNavigationCount - 1);
+                logger.info(`[capture] [recording nav +${elapsedMs}ms expected] → ${url}`);
+            }
+            else {
+                logger.warn(`[capture] [recording nav +${elapsedMs}ms framenavigated ${sameUrl ? 'same-url' : 'cross-url'}] → ${url}`);
+            }
+        };
+        // `load` ONLY fires on full document loads (page.goto, location.reload).
+        // SPA URL changes via pushState/popstate do NOT fire load. So if we see
+        // load events during recording that aren't tied to an opcode in flight,
+        // the document is being replaced — that's the white-flash + cursor-loss
+        // pattern. Surface it loudly so the cause (page-side reload, session
+        // redirect, HMR, mis-shaped NAVIGATE opcode) can be diagnosed quickly.
+        const loadHandler = () => {
+            const elapsedMs = Date.now() - recordingStartedAt;
+            logger.warn(`[capture] [recording document LOAD +${elapsedMs}ms] ${page.url()} — full document load (white flash + cursor loss)`);
+        };
+        const errorHandler = (err) => {
+            logger.warn(`[capture] [recording page error] ${err.message.split('\n')[0]}`);
+        };
+        page.on('framenavigated', navHandler);
+        page.on('load', loadHandler);
+        page.on('pageerror', errorHandler);
+        this.recordingNavWatcher = {
+            startedAt: recordingStartedAt,
+            expectedNavigationCount: 0,
+            detach: () => {
+                try {
+                    page.off('framenavigated', navHandler);
+                }
+                catch { /* ignore */ }
+                try {
+                    page.off('load', loadHandler);
+                }
+                catch { /* ignore */ }
+                try {
+                    page.off('pageerror', errorHandler);
+                }
+                catch { /* ignore */ }
+            },
+        };
         await this.seedClipCursor();
     }
+    async getElementBoundingBox(selector) {
+        const page = await this.browser.currentPage;
+        try {
+            return await page.locator(selector).first().boundingBox();
+        }
+        catch {
+            return null;
+        }
+    }
     async endRecording() {
         if (!this.recording) {
             throw new Error('recording was not started');
         }
         if (this.recording.finalized) {
             const buffer = await fs.readFile(this.recording.mp4Path);
+            const durationMs = this.recording.encodedDurationMs ?? await getMediaDurationMs(this.recording.mp4Path);
+            this.recording.encodedDurationMs = durationMs;
             return {
                 buffer,
-                durationMs: Date.now() - this.recording.startedAt,
+                durationMs,
                 mimeType: 'video/mp4',
                 trimStartMs: this.recording.result?.trimStartMs ?? 0,
             };
         }
+        if (this.recordingNavWatcher) {
+            this.recordingNavWatcher.detach();
+            this.recordingNavWatcher = null;
+        }
         const result = await this.recording.loop.stop();
         logger.info(`[capture] Clip frame capture: ${result.frameCount} frame(s), ` +
             `${result.measuredFps.toFixed(1)} fps over ${(result.actualDurationMs / 1000).toFixed(2)}s ` +
@@ -324,11 +475,12 @@ export class WebPlaywrightLocal {
         });
         this.recording.finalized = true;
         this.recording.result = result;
+        this.recording.encodedDurationMs = await getMediaDurationMs(this.recording.mp4Path);
         this.clipCursor = null;
         const buffer = await fs.readFile(this.recording.mp4Path);
         return {
             buffer,
-            durationMs: Date.now() - this.recording.startedAt,
+            durationMs: this.recording.encodedDurationMs,
             mimeType: 'video/mp4',
             trimStartMs: result.trimStartMs,
         };
@@ -340,7 +492,21 @@ export class WebPlaywrightLocal {
         await this.browser.setColorScheme(scheme);
     }
     async reloadPage() {
-        await this.browser.reloadCurrentPage();
+        if (this.recordingNavWatcher)
+            this.recordingNavWatcher.expectedNavigationCount += 1;
+        try {
+            await this.browser.reloadCurrentPage();
+        }
+        finally {
+            if (this.recordingNavWatcher) {
+                const watcher = this.recordingNavWatcher;
+                setTimeout(() => {
+                    if (watcher === this.recordingNavWatcher) {
+                        watcher.expectedNavigationCount = Math.max(0, watcher.expectedNavigationCount - 1);
+                    }
+                }, 500);
+            }
+        }
     }
     async writeStorageHint(params) {
         if (params.storage === 'cookie') {
@@ -494,7 +660,10 @@ export class WebPlaywrightLocal {
                 }
             }).catch(() => { });
         }
-        await moveMouse(page, sourcePoint, destPoint, { durationMs: dragDurationMs });
+        await moveMouse(page, sourcePoint, destPoint, {
+            durationMs: dragDurationMs,
+            ...(this.clipCursor ? { pace: this.clipCursor.pace } : {}),
+        });
         await page.waitForTimeout(70 + Math.random() * 60);
         await page.mouse.up();
         if (this.clipCursor) {
@@ -663,21 +832,25 @@ export class WebPlaywrightLocal {
             ? { minDelayMs: 20, maxDelayMs: 45 }
             : undefined);
     }
-    async seedClipCursor() {
+    async seedClipCursor(position) {
         if (!this.clipCursor)
             return;
         const page = await this.browser.currentPage;
         const viewport = page.viewportSize();
         if (!viewport)
             return;
-        const startX = Math.round(viewport.width * (0.3 + Math.random() * 0.4));
-        const startY = Math.round(viewport.height * (0.3 + Math.random() * 0.4));
-        await page.mouse.move(startX, startY);
+        const targetX = position
+            ? Math.max(0, Math.min(viewport.width, Math.round(position.x)))
+            : Math.round(viewport.width * (0.3 + Math.random() * 0.4));
+        const targetY = position
+            ? Math.max(0, Math.min(viewport.height, Math.round(position.y)))
+            : Math.round(viewport.height * (0.3 + Math.random() * 0.4));
+        await page.mouse.move(targetX, targetY);
         await page.evaluate(({ x, y }) => {
             if (typeof window.__akMoveCursor === 'function')
                 window.__akMoveCursor(x, y);
-        }, { x: startX, y: startY }).catch(() => { });
-        this.clipCursor.currentPosition = { x: startX, y: startY };
+        }, { x: targetX, y: targetY }).catch(() => { });
+        this.clipCursor.currentPosition = { x: targetX, y: targetY };
         await page.waitForTimeout(60);
     }
     async moveClipCursorToViewportCenter() {
@@ -748,7 +921,7 @@ export class WebPlaywrightLocal {
         const page = await this.browser.currentPage;
         const from = this.clipCursor.currentPosition;
         if (from) {
-            await moveMouse(page, from, point, options);
+            await moveMouse(page, from, point, { ...options, pace: this.clipCursor.pace });
         }
         else {
             await page.mouse.move(point.x, point.y);
@@ -784,6 +957,19 @@ function describeResolveOptions(opts) {
         parts.push(`placeholder="${opts.target.placeholder}"`);
     return parts.join(', ') || 'no target specified';
 }
+async function captureSurfaceMatches(page, expected) {
+    try {
+        const actual = await page.evaluate(() => ({
+            width: Math.round(window.innerWidth * window.devicePixelRatio),
+            height: Math.round(window.innerHeight * window.devicePixelRatio),
+        }));
+        return Math.abs(actual.width - expected.width) <= 2
+            && Math.abs(actual.height - expected.height) <= 2;
+    }
+    catch {
+        return false;
+    }
+}
 function getHumanPointInBox(box, viewport) {
     const insetX = Math.min(Math.max(box.width * 0.2, 6), Math.max(6, box.width / 2));
     const insetY = Math.min(Math.max(box.height * 0.2, 6), Math.max(6, box.height / 2));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autokap",
-  "version": "1.2.0",
+  "version": "1.3.1",
   "description": "AI-powered CLI tool for capturing clean screenshots of websites",
   "type": "module",
   "main": "./dist/index.js",
@@ -34,6 +34,14 @@
       "types": "./dist/execution-types.d.ts",
       "default": "./dist/execution-types.js"
     },
+    "./video-narration-schema": {
+      "types": "./dist/video-narration-schema.d.ts",
+      "default": "./dist/video-narration-schema.js"
+    },
+    "./openrouter-tts": {
+      "types": "./dist/openrouter-tts.d.ts",
+      "default": "./dist/openrouter-tts.js"
+    },
     "./alt-text": {
       "types": "./dist/alt-text.d.ts",
       "default": "./dist/alt-text.js"