autokap 1.0.6 → 1.0.8

package/dist/v2/web-playwright-local.js

@@ -0,0 +1,392 @@
+/**
+ * Capture Agent — WebPlaywrightLocal RuntimeAdapter
+ *
+ * Thin adapter delegating to the existing Browser class from src/browser.ts.
+ * This is the first (and for now only) RuntimeAdapter implementation.
+ */
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { humanType, moveMouse } from '../mouse-animation.js';
+import { resolveTarget } from './semantic-resolver.js';
+export class WebPlaywrightLocal {
+    browser;
+    recordingDir;
+    sessionStartedAt = Date.now();
+    recording = null;
+    clipCursor = null;
+    constructor(browser, recordingDir) {
+        this.browser = browser;
+        this.recordingDir = recordingDir;
+    }
+    async navigate(url) {
+        await this.browser.navigateTo(url);
+    }
+    async getCurrentUrl() {
+        const page = await this.browser.currentPage;
+        return page.url();
+    }
+    async getAKTree() {
+        return this.browser.getAKTree();
+    }
+    async getPageSignals() {
+        return this.browser.capturePageSignals();
+    }
+    async click(selector, options) {
+        const page = await this.browser.currentPage;
+        if (options?.coordinates) {
+            await this.moveClipCursorToPoint(options.coordinates);
+            await this.browser.clickByCoordinates(options.coordinates.x, options.coordinates.y);
+            return;
+        }
+        const locator = page.locator(selector).first();
+        await this.moveClipCursorToLocator(locator);
+        if (options?.useKeyboard) {
+            await locator.focus();
+            await page.keyboard.press('Enter');
+            return;
+        }
+        if (options?.useJsDispatch) {
+            await locator.dispatchEvent('click');
+            return;
+        }
+        await this.browser.clickBySelector(selector, { force: options?.force });
+    }
+    /**
+     * Click an element using semantic target resolution.
+     * Tries CSS selector first, falls back to Playwright semantic locators.
+     */
+    async clickByTarget(opts) {
+        const page = await this.browser.currentPage;
+        const resolved = await resolveTarget(page, opts);
+        if (!resolved) {
+            throw new Error(`cannot find target: ${describeResolveOptions(opts)}`);
+        }
+        await this.moveClipCursorToLocator(resolved.locator);
+        await resolved.locator.click({ timeout: 5000 });
+    }
+    /**
+     * Type into an element using semantic target resolution.
+     */
+    async typeByTarget(opts, text, clearFirst = true) {
+        const page = await this.browser.currentPage;
+        const resolved = await resolveTarget(page, opts);
+        if (!resolved) {
+            throw new Error(`cannot find target for typing: ${describeResolveOptions(opts)}`);
+        }
+        if (this.clipCursor) {
+            await this.typeIntoLocator(resolved.locator, text, clearFirst);
+            return;
+        }
+        if (clearFirst) {
+            await resolved.locator.fill(text);
+            return;
+        }
+        await resolved.locator.click();
+        await resolved.locator.pressSequentially(text);
+    }
+    /**
+     * Wait for an element using semantic target resolution.
+     */
+    async waitForTarget(opts, timeoutMs = 10000) {
+        const page = await this.browser.currentPage;
+        const resolved = await resolveTarget(page, { ...opts, timeoutMs });
+        return resolved !== null;
+    }
+    /**
+     * Scroll an element into view using semantic target resolution.
+     */
+    async scrollIntoViewByTarget(opts) {
+        const page = await this.browser.currentPage;
+        const resolved = await resolveTarget(page, opts);
+        if (!resolved) {
+            throw new Error(`cannot find target to scroll into view: ${describeResolveOptions(opts)}`);
+        }
+        if (this.clipCursor) {
+            await this.moveClipCursorToViewportCenter();
+            await resolved.locator.evaluate((node) => {
+                node.scrollIntoView({ block: 'center', behavior: 'smooth' });
+            });
+            await page.waitForTimeout(350);
+            return;
+        }
+        await resolved.locator.scrollIntoViewIfNeeded({ timeout: 5000 });
+    }
+    async type(selector, text, clearFirst = true) {
+        if (this.clipCursor) {
+            const page = await this.browser.currentPage;
+            await this.typeIntoLocator(page.locator(selector).first(), text, clearFirst);
+            return;
+        }
+        await this.browser.typeText(text, { selector, clearFirst });
+    }
+    async pressKey(key) {
+        const page = await this.browser.currentPage;
+        if (this.clipCursor) {
+            await page.waitForTimeout(90);
+            await page.keyboard.press(key);
+            return;
+        }
+        await this.browser.pressKey(key);
+    }
+    async scroll(direction, amount) {
+        if (this.clipCursor) {
+            const page = await this.browser.currentPage;
+            await this.moveClipCursorToViewportCenter();
+            const dx = direction === 'right' ? (amount ?? 500) : direction === 'left' ? -(amount ?? 500) : 0;
+            const dy = direction === 'down' ? (amount ?? 500) : direction === 'up' ? -(amount ?? 500) : 0;
+            await page.evaluate(({ deltaX, deltaY }) => {
+                window.scrollBy({ left: deltaX, top: deltaY, behavior: 'smooth' });
+            }, { deltaX: dx, deltaY: dy });
+            await page.waitForTimeout(420);
+            return;
+        }
+        await this.browser.scroll(direction, amount);
+    }
+    async scrollIntoView(selector) {
+        const page = await this.browser.currentPage;
+        if (this.clipCursor) {
+            await this.moveClipCursorToViewportCenter();
+            await page.locator(selector).first().evaluate((node) => {
+                node.scrollIntoView({ block: 'center', behavior: 'smooth' });
+            });
+            await page.waitForTimeout(350);
+            return;
+        }
+        await page.locator(selector).first().scrollIntoViewIfNeeded({ timeout: 5000 });
+    }
+    async waitFor(condition) {
+        try {
+            const page = await this.browser.currentPage;
+            const stateMap = { visible: 'visible', attached: 'attached' };
+            await page.locator(condition.selector).waitFor({
+                state: stateMap[condition.state],
+                timeout: condition.timeoutMs,
+            });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async dismissOverlays() {
+        // Pass 1: Built-in cookie/widget dismissal (cookie-dismiss.ts)
+        const result = await this.browser.dismissOverlays();
+        if (result.dismissed)
+            return result;
+        // Pass 2: Playwright-level sweep for common accept/close buttons
+        // in cookie-like containers that the built-in module might miss
+        const page = await this.browser.currentPage;
+        const acceptPatterns = [
+            'button:has-text("Accept")',
+            'button:has-text("Accept all")',
+            'button:has-text("Accepter")',
+            'button:has-text("Tout accepter")',
+            'button:has-text("Got it")',
+            'button:has-text("I agree")',
+            'button:has-text("OK")',
+            '[role="button"]:has-text("Accept")',
+            '[role="button"]:has-text("Accept all")',
+        ];
+        for (const pattern of acceptPatterns) {
+            try {
+                const btn = page.locator(pattern).first();
+                if (await btn.isVisible({ timeout: 500 })) {
+                    // Check if the button is near the bottom of the viewport (likely a banner)
+                    const box = await btn.boundingBox();
+                    const viewport = page.viewportSize();
+                    if (box && viewport && box.y > viewport.height * 0.6) {
+                        await btn.click({ timeout: 2000 });
+                        await page.waitForTimeout(500);
+                        return { dismissed: true, method: `v2-sweep:${pattern}` };
+                    }
+                }
+            }
+            catch {
+                // Try next pattern
+            }
+        }
+        // Pass 3: Try Escape key for modal overlays
+        try {
+            await page.keyboard.press('Escape');
+            await page.waitForTimeout(300);
+        }
+        catch {
+            // Non-fatal
+        }
+        return result;
+    }
+    async takeScreenshot() {
+        return this.browser.takeScreenshot();
+    }
+    async takeElementScreenshot(selector) {
+        const { buffer } = await this.browser.screenshotBySelector(selector);
+        return buffer;
+    }
+    async takeCleanScreenshot() {
+        return this.browser.takeScreenshot();
+    }
+    async beginRecording(options) {
+        const page = await this.browser.currentPage;
+        if (!page.video()) {
+            throw new Error(`recording is not enabled for ${options.mediaMode} mode`);
+        }
+        const recordingDir = this.recordingDir
+            ?? await fs.mkdtemp(path.join(os.tmpdir(), 'autokap-v2-recording-'));
+        this.recording = {
+            mediaMode: options.mediaMode,
+            startedAt: Date.now(),
+            trimStartMs: Math.max(0, Date.now() - this.sessionStartedAt),
+            outputPath: path.join(recordingDir, `${options.mediaMode}.webm`),
+            finalized: false,
+        };
+        this.clipCursor = { currentPosition: null };
+        await this.seedClipCursor();
+    }
+    async endRecording() {
+        if (!this.recording) {
+            throw new Error('recording was not started');
+        }
+        if (this.recording.finalized) {
+            const buffer = await fs.readFile(this.recording.outputPath);
+            return {
+                buffer,
+                durationMs: Date.now() - this.recording.startedAt,
+                mimeType: 'video/webm',
+                trimStartMs: this.recording.trimStartMs,
+            };
+        }
+        const videoRef = this.browser.currentPage.video();
+        if (!videoRef) {
+            throw new Error('recording finalization failed: no Playwright video handle found');
+        }
+        await this.browser.closeContext();
+        await videoRef.saveAs(this.recording.outputPath);
+        this.recording.finalized = true;
+        this.clipCursor = null;
+        const buffer = await fs.readFile(this.recording.outputPath);
+        return {
+            buffer,
+            durationMs: Date.now() - this.recording.startedAt,
+            mimeType: 'video/webm',
+            trimStartMs: this.recording.trimStartMs,
+        };
+    }
+    async setLocale(locale) {
+        await this.browser.setLanguage(locale);
+    }
+    async setColorScheme(scheme) {
+        await this.browser.setColorScheme(scheme);
+    }
+    async reloadPage() {
+        await this.browser.reloadCurrentPage();
+    }
+    async writeStorageHint(params) {
+        return this.browser.writeStorageHintCandidate({
+            storageName: params.storage,
+            key: params.key,
+            candidate: params.value,
+            kind: params.kind,
+        });
+    }
+    async close() {
+        await this.browser.close();
+    }
+    async typeIntoLocator(locator, text, clearFirst) {
+        const page = await this.browser.currentPage;
+        await locator.waitFor({ state: 'visible', timeout: 5000 });
+        await locator.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => undefined);
+        await this.moveClipCursorToLocator(locator);
+        await locator.click({ timeout: 5000 });
+        if (clearFirst) {
+            await page.keyboard.press('Control+A');
+        }
+        await page.waitForTimeout(70);
+        await humanType(page, text);
+    }
+    async seedClipCursor() {
+        if (!this.clipCursor)
+            return;
+        const page = await this.browser.currentPage;
+        const viewport = page.viewportSize();
+        if (!viewport)
+            return;
+        const startX = Math.round(viewport.width * (0.3 + Math.random() * 0.4));
+        const startY = Math.round(viewport.height * (0.3 + Math.random() * 0.4));
+        await page.mouse.move(startX, startY);
+        this.clipCursor.currentPosition = { x: startX, y: startY };
+        await page.waitForTimeout(60);
+    }
+    async moveClipCursorToViewportCenter() {
+        if (!this.clipCursor)
+            return;
+        const page = await this.browser.currentPage;
+        const viewport = page.viewportSize();
+        if (!viewport)
+            return;
+        await this.moveClipCursorToPoint({
+            x: Math.round(viewport.width / 2),
+            y: Math.round(viewport.height / 2),
+        }, { durationMs: 280, steps: 18 });
+    }
+    async moveClipCursorToLocator(locator) {
+        if (!this.clipCursor)
+            return;
+        await locator.waitFor({ state: 'visible', timeout: 5000 });
+        await locator.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => undefined);
+        const page = await this.browser.currentPage;
+        const viewport = page.viewportSize();
+        const box = await locator.boundingBox();
+        if (!box || !viewport)
+            return;
+        await this.moveClipCursorToPoint(getHumanPointInBox(box, viewport));
+    }
+    async moveClipCursorToPoint(point, options) {
+        if (!this.clipCursor)
+            return;
+        const page = await this.browser.currentPage;
+        const from = this.clipCursor.currentPosition;
+        if (from) {
+            await moveMouse(page, from, point, options);
+        }
+        else {
+            await page.mouse.move(point.x, point.y);
+        }
+        this.clipCursor.currentPosition = point;
+    }
+}
+function describeResolveOptions(opts) {
+    const parts = [];
+    if (opts.selector)
+        parts.push(`selector="${opts.selector}"`);
+    if (opts.target?.text)
+        parts.push(`text="${opts.target.text}"`);
+    if (opts.target?.role)
+        parts.push(`role="${opts.target.role}"`);
+    if (opts.target?.label)
+        parts.push(`label="${opts.target.label}"`);
+    if (opts.target?.placeholder)
+        parts.push(`placeholder="${opts.target.placeholder}"`);
+    return parts.join(', ') || 'no target specified';
+}
+function getHumanPointInBox(box, viewport) {
+    const insetX = Math.min(Math.max(box.width * 0.2, 6), Math.max(6, box.width / 2));
+    const insetY = Math.min(Math.max(box.height * 0.2, 6), Math.max(6, box.height / 2));
+    const minX = box.x + insetX;
+    const maxX = box.x + Math.max(insetX, box.width - insetX);
+    const minY = box.y + insetY;
+    const maxY = box.y + Math.max(insetY, box.height - insetY);
+    const targetX = maxX <= minX ? box.x + box.width / 2 : randomBetween(minX, maxX);
+    const targetY = maxY <= minY ? box.y + box.height / 2 : randomBetween(minY, maxY);
+    return {
+        x: clampPoint(targetX, 4, viewport.width - 4),
+        y: clampPoint(targetY, 4, viewport.height - 4),
+    };
+}
+function randomBetween(min, max) {
+    return min + Math.random() * Math.max(0, max - min);
+}
+function clampPoint(value, min, max) {
+    return Math.round(Math.max(min, Math.min(max, value)));
+}
+//# sourceMappingURL=web-playwright-local.js.map

package/dist/version.d.ts

@@ -0,0 +1 @@
+export declare const APP_VERSION: string;

package/dist/version.js

@@ -0,0 +1,5 @@
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const pkg = require('../package.json');
+export const APP_VERSION = pkg.version;
+//# sourceMappingURL=version.js.map

package/dist/video-agent.d.ts

@@ -0,0 +1,143 @@
+import { Browser } from './browser.js';
+import type { VideoAgentConfig, VideoAgentResult, VideoPhase, InteractiveElement, VideoStep, VideoPlan, StepUsage, VideoStepAttemptTrace, VideoStepArtifact, VideoSelectorMemoryUpdate, VideoPageSignals, VideoObservationSnapshot, BrowserSessionStorageState, BrowserStorageState } from './types.js';
+export type VideoLogCallback = (entry: {
+    level: 'info' | 'success' | 'error' | 'ai';
+    message: string;
+    timestamp: number;
+}) => void;
+export type VideoPhaseCallback = (phase: VideoPhase) => void;
+export type VideoStepCallback = (stepIndex: number, total: number, description: string, phase: 'dry_run' | 'recording') => void;
+export type VideoStepResultCallback = (stepIndex: number, ok: boolean, reason?: string) => void;
+export type VideoVariantPhase = 'preflight' | 'planning' | 'dry_run' | 'recording' | 'done' | 'failed';
+export interface VideoPreflightResult {
+    ok: boolean;
+    reason?: string;
+    code?: string;
+    detectedLang?: string | null;
+    detectedTheme?: 'light' | 'dark' | null;
+    finalUrl?: string;
+    storageState?: BrowserStorageState;
+    sessionStorage?: BrowserSessionStorageState;
+    pageSignals?: VideoPageSignals;
+    observationSummary?: string;
+    observationSnapshot?: VideoObservationSnapshot;
+    usage?: StepUsage[];
+}
+export type VideoPreflightCallback = (result: VideoPreflightResult) => void;
+export type VideoVariantPhaseCallback = (phase: VideoVariantPhase) => void;
+export type VideoStepAttemptCallback = (attempt: VideoStepAttemptTrace) => void;
+export type VideoStepArtifactCallback = (artifact: VideoStepArtifact) => void;
+export interface VideoRunCallbacks {
+    onLog?: VideoLogCallback;
+    onPhaseChange?: VideoPhaseCallback;
+    onVariantPhase?: VideoVariantPhaseCallback;
+    onStep?: VideoStepCallback;
+    onStepResult?: VideoStepResultCallback;
+    onStepAttempt?: VideoStepAttemptCallback;
+    onStepArtifact?: VideoStepArtifactCallback;
+    onPreflightResult?: VideoPreflightCallback;
+    /** Emit a live Playwright screenshot (base64 JPEG) for real-time preview in the UI. */
+    onScreenshot?: (base64: string) => void;
+}
+type VideoDryRunFailureSubphase = 'prepared_variant' | 'deterministic_exec' | 'verification_snapshot' | 'verification_llm' | 'repair_lane' | 'llm_fixer' | 'checkpoint_restore' | 'budget';
+export interface VideoDryRunFailure {
+    stepIndex: number;
+    subphase: VideoDryRunFailureSubphase;
+    reason: string;
+    resolvedTargetSummary?: string;
+}
+interface VideoStepCheckpoint {
+    url: string;
+    scrollX: number;
+    scrollY: number;
+}
+interface ViewportRectSnapshot {
+    top: number;
+    bottom: number;
+    left: number;
+    right: number;
+    width: number;
+    height: number;
+}
+export declare function restoreVideoStepCheckpoint(browser: Browser, checkpoint: VideoStepCheckpoint): Promise<void>;
+export declare function isRectMeaningfullyVisibleInViewport(params: {
+    rect: ViewportRectSnapshot;
+    viewportWidth: number;
+    viewportHeight: number;
+    topInset?: number;
+    bottomInset?: number;
+}): boolean;
+export declare function sanitizeVariantPrefixSteps(steps: VideoStep[], requestedLang?: string, requestedTheme?: 'light' | 'dark'): VideoStep[];
+export declare function evaluateVideoStepSafety(step: VideoStep, config: VideoAgentConfig, page: import('playwright').Page, currentUrl: string, interactiveElements: InteractiveElement[]): Promise<{
+    allowed: boolean;
+    reason?: string;
+}>;
+export declare function extractNamedScrollTarget(description?: string): string | null;
+export declare function inspectVideoTarget(page: import('playwright').Page, selector?: string, coordinates?: {
+    x: number;
+    y: number;
+}): Promise<InteractiveElement | null>;
+interface VerifyStepResult {
+    ok: boolean;
+    reason?: string;
+    suggestion?: string;
+    giveUp?: boolean;
+    usage: Partial<StepUsage>;
+}
+export declare function verifyVideoStepDeterministically(browser: Browser, step: VideoStep): Promise<VerifyStepResult | null>;
+export declare function runVariantPreflight(config: VideoAgentConfig, prefixPlan: VideoPlan | null, callbacks?: VideoRunCallbacks): Promise<VideoPreflightResult>;
+export declare function composeHybridPlan(basePlan: VideoPlan, variantPrefixPlan: VideoPlan | null): VideoPlan;
+export declare function createBasePlan(config: VideoAgentConfig): Promise<{
+    plan: VideoPlan;
+    usage: StepUsage;
+}>;
+export declare function createVariantPrefixPlan(config: VideoAgentConfig): Promise<{
+    plan: VideoPlan;
+    usage: StepUsage;
+}>;
+export declare function verifyAndPatchPlan(plan: VideoPlan, config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<{
+    success: true;
+    plan: VideoPlan;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+} | {
+    success: false;
+    error: string;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+    failedStepIndex?: number;
+    failedSubphase?: VideoDryRunFailureSubphase;
+}>;
+/**
+ * Backward-compatible helper: full planning + verify in one call.
+ */
+export declare function planAndVerify(config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<{
+    success: true;
+    plan: VideoPlan;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+} | {
+    success: false;
+    error: string;
+    usage: StepUsage[];
+    memoryUpdates: VideoSelectorMemoryUpdate[];
+}>;
+/**
+ * Run phase 3 (recording only) using a pre-verified plan.
+ * Call after `planAndVerify()` to record across multiple lang/theme combinations.
+ */
+export declare function recordPlan(plan: VideoPlan, config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<{
+    videoPath: string | null;
+    durationMs: number;
+    stepsExecuted: number;
+    setupDurationSec: number;
+    usage: StepUsage[];
+}>;
+/**
+ * Run the full 3-phase video capture pipeline:
+ * 1. Planning: script → structured plan (LLM)
+ * 2. Dry-run: verify all steps work
+ * 3. Recording: execute with Bezier mouse + cursor overlay
+ */
+export declare function runVideoAgent(config: VideoAgentConfig, callbacks?: VideoRunCallbacks): Promise<VideoAgentResult>;
+export {};