autokap 1.0.7 → 1.0.10

package/dist/v2/types.d.ts

@@ -0,0 +1,444 @@
+/**
+ * Capture Agent — IR Types
+ *
+ * All types for the compiled execution model:
+ * preset (natural language) -> ExecutionProgram (typed IR) -> deterministic runtime
+ */
+import type { AKTree, BrowserStorageState, BrowserSessionStorageState, VideoCursorTheme, VideoPageSignals } from '../types.js';
+export type OpcodeKind = 'NAVIGATE' | 'DISMISS_OVERLAYS' | 'ASSERT_ROUTE' | 'ASSERT_SURFACE' | 'CLICK' | 'TYPE' | 'PRESS_KEY' | 'WAIT_FOR' | 'SET_LOCALE' | 'SET_THEME' | 'SCROLL' | 'CAPTURE_SCREENSHOT' | 'BEGIN_CLIP' | 'END_CLIP';
+export type PostconditionType = 'route_matches' | 'element_visible' | 'element_absent' | 'text_contains' | 'overlay_dismissed' | 'screenshot_stable' | 'any_change' | 'always';
+export interface PostconditionSpec {
+    type: PostconditionType;
+    /** URL pattern (glob) for route_matches */
+    pattern?: string;
+    /** CSS/AKTree selector for element_visible, element_absent, text_contains */
+    selector?: string;
+    /** Expected text substring for text_contains */
+    text?: string;
+    /** Pixel diff threshold (0-1) for screenshot_stable. Default: 0.01 */
+    threshold?: number;
+    /** Max wait time (ms) for the postcondition to become true. Default: 5000 */
+    waitMs?: number;
+}
+/**
+ * Describes an element semantically, without requiring a CSS selector.
+ * The runtime resolves the target using Playwright semantic locators
+ * (getByRole, getByText, getByLabel) then falls back to AKTree fuzzy matching.
+ *
+ * At least one of text/role/label should be provided.
+ */
+export interface SemanticTarget {
+    /** Visible text content of the element (exact or partial match) */
+    text?: string;
+    /** ARIA role: "button", "link", "textbox", "checkbox", "tab", "menuitem", etc. */
+    role?: string;
+    /** Accessible name: aria-label, associated label text, or title attribute */
+    label?: string;
+    /** Nearby text or heading for disambiguation (e.g. "in the pricing section") */
+    near?: string;
+    /** Placeholder text (for inputs) */
+    placeholder?: string;
+    /** Whether the text match should be exact. Default: false (substring match) */
+    exact?: boolean;
+}
+export interface RecoveryPolicy {
+    /** Number of deterministic retries before escalating. Default: 2 */
+    retries: number;
+    /** Try selector memory alternatives. Default: true */
+    useSelectorMemory: boolean;
+    /** Try alternative interaction methods (keyboard, JS, coords). Default: true */
+    useAltInteraction: boolean;
+    /** Reload the page and retry from last checkpoint. Default: false */
+    allowReload: boolean;
+    /** Allow LLM healer as last resort. Default: true */
+    allowHealer: boolean;
+}
+export declare const DEFAULT_RECOVERY_POLICY: RecoveryPolicy;
+interface OpcodeBase {
+    kind: OpcodeKind;
+    /** Human-readable description of what this opcode does */
+    description: string;
+    postcondition: PostconditionSpec;
+    recovery: RecoveryPolicy;
+    /** Max time (ms) for this opcode to complete, including recovery. Default: 15000 */
+    timeoutMs: number;
+    /** Max recovery attempts before this opcode is considered failed. Default: 3 */
+    maxFailures: number;
+}
+export interface NavigateOpcode extends OpcodeBase {
+    kind: 'NAVIGATE';
+    url: string;
+}
+export interface DismissOverlaysOpcode extends OpcodeBase {
+    kind: 'DISMISS_OVERLAYS';
+}
+export interface AssertRouteOpcode extends OpcodeBase {
+    kind: 'ASSERT_ROUTE';
+    /** URL pattern (glob or regex) to assert */
+    urlPattern: string;
+}
+export interface AssertSurfaceOpcode extends OpcodeBase {
+    kind: 'ASSERT_SURFACE';
+    /** Selector(s) that must be visible */
+    selectors: string[];
+    /** If true, ALL selectors must match. If false, ANY one suffices. Default: true */
+    matchAll: boolean;
+}
+export interface ClickOpcode extends OpcodeBase {
+    kind: 'CLICK';
+    /** CSS selector — typically a data-ak attribute set by the user's AI assistant */
+    selector: string;
+    /** Semantic target fallback — used by recovery chain when selector fails */
+    target?: SemanticTarget;
+    /** AKTree fingerprint for fuzzy matching fallback */
+    fingerprint?: string;
+    /** Alternative selectors to try in order */
+    selectorAlternates?: string[];
+}
+export interface TypeOpcode extends OpcodeBase {
+    kind: 'TYPE';
+    /** CSS selector — typically a data-ak attribute set by the user's AI assistant */
+    selector: string;
+    /** Semantic target fallback */
+    target?: SemanticTarget;
+    text: string;
+    /** Clear existing value before typing. Default: true */
+    clearFirst: boolean;
+    fingerprint?: string;
+    selectorAlternates?: string[];
+}
+export interface PressKeyOpcode extends OpcodeBase {
+    kind: 'PRESS_KEY';
+    key: string;
+}
+export interface WaitForOpcode extends OpcodeBase {
+    kind: 'WAIT_FOR';
+    /** CSS selector to wait for — required unless target is provided */
+    selector?: string;
+    /** Semantic target fallback — used when selector is not available */
+    target?: SemanticTarget;
+    /** 'visible' = element visible in viewport, 'attached' = exists in DOM. Default: 'visible' */
+    state: 'visible' | 'attached';
+}
+export interface SetLocaleOpcode extends OpcodeBase {
+    kind: 'SET_LOCALE';
+    /** Target BCP-47 locale (e.g. "fr", "en-US") */
+    locale: string;
+    /** Strategy to set locale */
+    method: 'browser_context' | 'ui_interaction' | 'storage';
+    /** Selector to interact with (for ui_interaction) */
+    selector?: string;
+    /** Storage key/value pairs to set (for storage method) */
+    storageHints?: {
+        storage: 'localStorage' | 'sessionStorage';
+        key: string;
+        value: string;
+    }[];
+}
+export interface SetThemeOpcode extends OpcodeBase {
+    kind: 'SET_THEME';
+    theme: 'light' | 'dark';
+    method: 'color_scheme' | 'ui_interaction' | 'storage';
+    selector?: string;
+    storageHints?: {
+        storage: 'localStorage' | 'sessionStorage';
+        key: string;
+        value: string;
+    }[];
+}
+export interface ScrollOpcode extends OpcodeBase {
+    kind: 'SCROLL';
+    direction: 'up' | 'down' | 'left' | 'right';
+    /** Pixels to scroll. Default: viewport height */
+    amount?: number;
+    /** Optional target selector to scroll into view */
+    targetSelector?: string;
+    /** Semantic target to scroll into view (used when targetSelector is absent) */
+    target?: SemanticTarget;
+}
+export interface CaptureScreenshotOpcode extends OpcodeBase {
+    kind: 'CAPTURE_SCREENSHOT';
+    /** Stable preset capture identifier (page id / element name) for Studio and dev links */
+    captureId?: string;
+    /** Human-readable preset capture name */
+    captureName?: string;
+    /** Optional element selector for element-level capture */
+    elementSelector?: string;
+}
+export interface BeginClipOpcode extends OpcodeBase {
+    kind: 'BEGIN_CLIP';
+    /** Stable preset clip identifier for Studio and dev links */
+    clipId?: string;
+    /** Human-readable preset clip name */
+    clipName?: string;
+}
+export interface EndClipOpcode extends OpcodeBase {
+    kind: 'END_CLIP';
+    /** Stable preset clip identifier for Studio and dev links */
+    clipId?: string;
+    /** Human-readable preset clip name */
+    clipName?: string;
+}
+export type ExecutionOpcode = NavigateOpcode | DismissOverlaysOpcode | AssertRouteOpcode | AssertSurfaceOpcode | ClickOpcode | TypeOpcode | PressKeyOpcode | WaitForOpcode | SetLocaleOpcode | SetThemeOpcode | ScrollOpcode | CaptureScreenshotOpcode | BeginClipOpcode | EndClipOpcode;
+export interface VariantSpec {
+    id: string;
+    viewport: {
+        width: number;
+        height: number;
+    };
+    deviceScaleFactor?: number;
+    locale?: string;
+    theme?: 'light' | 'dark';
+    /** Stable target identifier from the preset config */
+    targetId?: string;
+    /** Human-readable target label from the preset config */
+    targetLabel?: string;
+    /** Device frame label (e.g. "iPhone 15 Pro") for mockup rendering */
+    deviceFrame?: string;
+}
+export interface PreconditionSpec {
+    /** Auth requirement */
+    auth: 'authenticated' | 'anonymous' | 'any';
+    /** Credentials reference ID (resolved at runtime from CLI config) */
+    credentialsId?: string;
+    /** Pre-loaded storage state */
+    storageState?: BrowserStorageState;
+    sessionStorage?: BrowserSessionStorageState;
+    /** Cookies to set before execution */
+    cookies?: Array<{
+        name: string;
+        value: string;
+        domain: string;
+        path?: string;
+    }>;
+}
+export type MediaMode = 'screenshot' | 'clip';
+export interface ArtifactSpec {
+    mediaMode: MediaMode;
+    /** Output format preferences */
+    format?: {
+        /** For clips: 'gif' | 'mp4' | 'both'. Default: 'gif' */
+        clipFormat?: 'gif' | 'mp4' | 'both';
+        /** For screenshots: 'png' | 'jpeg'. Default: 'png' */
+        screenshotFormat?: 'png' | 'jpeg';
+    };
+    /** Cursor style for clip recordings. Default: 'minimal'. */
+    cursorTheme?: VideoCursorTheme;
+    /** Whether to apply device frame mockup. Default: false */
+    applyMockup?: boolean;
+    /** Whether to add status bar. Default: false */
+    applyStatusBar?: boolean;
+}
+export interface ExecutionProgram {
+    presetId: string;
+    programVersion: number;
+    mediaMode: MediaMode;
+    baseUrl: string;
+    variants: VariantSpec[];
+    preconditions: PreconditionSpec;
+    steps: ExecutionOpcode[];
+    artifactPlan: ArtifactSpec;
+    /** Hash of the source preset content, used to detect recompilation needs */
+    compileFingerprint: string;
+    /** ISO timestamp of compilation */
+    compiledAt: string;
+    /** Model used for compilation */
+    compiledWith?: string;
+}
+export interface CircuitBreakerConfig {
+    /** Max recovery attempts per opcode. Default: 3 */
+    maxPerOpcode: number;
+    /** Max total failures per page navigation. Default: 5 */
+    maxPerPage: number;
+    /** Max total failures per run. Default: 10 */
+    maxPerRun: number;
+}
+export declare const DEFAULT_CIRCUIT_BREAKER: CircuitBreakerConfig;
+export interface HealerPatch {
+    /** Index of the opcode in the program that was patched */
+    opcodeIndex: number;
+    /** The original opcode before patching */
+    originalOpcode: ExecutionOpcode;
+    /** The replacement opcode(s). Usually 1, max 3. */
+    replacementOpcodes: ExecutionOpcode[];
+    /** Why the healer made this change */
+    reason: string;
+    /** Timestamp */
+    patchedAt: string;
+}
+export type OpcodeResultStatus = 'ok' | 'recovered' | 'skipped' | 'failed';
+export interface OpcodeResult {
+    opcodeIndex: number;
+    kind: OpcodeKind;
+    status: OpcodeResultStatus;
+    /** Time taken for this opcode (ms) */
+    durationMs: number;
+    /** Number of recovery attempts used */
+    recoveryAttempts: number;
+    /** Recovery strategy that succeeded, if any */
+    recoveryStrategy?: 'retry' | 'selector_memory' | 'alt_interaction' | 'reload' | 'healer';
+    /** Error message if failed */
+    error?: string;
+}
+export interface VariantResult {
+    variantId: string;
+    success: boolean;
+    opcodeResults: OpcodeResult[];
+    /** Total duration for this variant (ms) */
+    durationMs: number;
+    /** Artifact buffers produced */
+    artifacts: ArtifactResult[];
+    error?: string;
+}
+export interface ArtifactResult {
+    mediaMode: MediaMode;
+    buffer: Buffer;
+    mimeType: string;
+    durationMs?: number;
+    trimStartMs?: number;
+    /** For screenshots: viewport dimensions of the capture */
+    dimensions?: {
+        width: number;
+        height: number;
+    };
+    /** Generated alt text for accessibility */
+    altText?: string;
+    /** Final URL at the time the artifact was produced */
+    captureUrl?: string;
+    /** Step index that produced the artifact */
+    stepIndex?: number;
+    /** Human-readable label for the artifact */
+    stepDescription?: string;
+    /** Variant metadata used by persistence and gallery views */
+    variantId?: string;
+    /** Stable preset capture identifier for screenshots */
+    captureId?: string;
+    /** Human-readable preset capture name for screenshots */
+    captureName?: string;
+    /** Stable preset clip identifier for clips */
+    clipId?: string;
+    /** Human-readable preset clip name for clips */
+    clipName?: string;
+    captureType?: 'fullpage' | 'element';
+    elementSelector?: string;
+}
+export type LLMStepType = 'capture_verification' | 'alt_text_generation' | 'healer_invocation';
+export interface LLMStepUsage {
+    stepType: LLMStepType;
+    generationId: string | null;
+    model: string;
+    promptTokens: number;
+    completionTokens: number;
+}
+export interface RunTelemetry {
+    /** Total LLM calls (compilation excluded) */
+    llmCallCount: number;
+    /** Total LLM cost in EUR */
+    llmCostEur: number;
+    /** Detailed per-call LLM usage for cost logging */
+    llmStepUsages: LLMStepUsage[];
+    /** Total opcodes executed across all variants */
+    totalOpcodes: number;
+    /** Opcodes that needed recovery */
+    recoveredOpcodes: number;
+    /** Opcodes that failed even after recovery */
+    failedOpcodes: number;
+    /** Number of healer invocations */
+    healerInvocations: number;
+    /** Circuit breaker trips */
+    circuitBreakerTrips: number;
+}
+export interface RunResult {
+    programId: string;
+    success: boolean;
+    variantResults: VariantResult[];
+    telemetry: RunTelemetry;
+    /** Healer patches accumulated during the run. Only propagate to server if success=true */
+    healerPatches: HealerPatch[];
+    /** Total run duration (ms) */
+    totalDurationMs: number;
+    error?: string;
+}
+export interface WaitCondition {
+    selector: string;
+    state: 'visible' | 'attached';
+    timeoutMs: number;
+}
+export interface ClickOptions {
+    /** Force click even if element is covered */
+    force?: boolean;
+    /** Use keyboard (Tab + Enter) instead of mouse */
+    useKeyboard?: boolean;
+    /** Use JS dispatch instead of Playwright click */
+    useJsDispatch?: boolean;
+    /** Click by coordinates from bounding box */
+    coordinates?: {
+        x: number;
+        y: number;
+    };
+}
+export interface RecordingOptions {
+    mediaMode: 'clip';
+}
+export interface RecordingResult {
+    buffer: Buffer;
+    durationMs: number;
+    mimeType: string;
+    trimStartMs?: number;
+}
+export interface RuntimeAdapter {
+    navigate(url: string): Promise<void>;
+    getCurrentUrl(): Promise<string>;
+    getAKTree(): Promise<AKTree>;
+    getPageSignals(): Promise<VideoPageSignals>;
+    click(selector: string, options?: ClickOptions): Promise<void>;
+    type(selector: string, text: string, clearFirst?: boolean): Promise<void>;
+    pressKey(key: string): Promise<void>;
+    scroll(direction: 'up' | 'down' | 'left' | 'right', amount?: number): Promise<void>;
+    scrollIntoView(selector: string): Promise<void>;
+    waitFor(condition: WaitCondition): Promise<boolean>;
+    dismissOverlays(): Promise<{
+        dismissed: boolean;
+        method: string | null;
+    }>;
+    takeScreenshot(): Promise<Buffer>;
+    takeElementScreenshot?(selector: string): Promise<Buffer>;
+    takeCleanScreenshot(): Promise<Buffer>;
+    beginRecording(options: RecordingOptions): Promise<void>;
+    endRecording(): Promise<RecordingResult>;
+    setLocale(locale: string): Promise<void>;
+    setColorScheme(scheme: 'light' | 'dark'): Promise<void>;
+    reloadPage?(): Promise<void>;
+    writeStorageHint?(params: {
+        storage: 'localStorage' | 'sessionStorage';
+        key: string;
+        value: string;
+        kind: 'locale' | 'theme';
+    }): Promise<boolean>;
+    close(): Promise<void>;
+    /** Click an element by semantic target. Falls back to selector if target not found. */
+    clickByTarget?(opts: {
+        selector?: string;
+        target?: SemanticTarget;
+        selectorAlternates?: string[];
+    }): Promise<void>;
+    /** Type into an element by semantic target. */
+    typeByTarget?(opts: {
+        selector?: string;
+        target?: SemanticTarget;
+        selectorAlternates?: string[];
+    }, text: string, clearFirst?: boolean): Promise<void>;
+    /** Wait for an element by semantic target. */
+    waitForTarget?(opts: {
+        selector?: string;
+        target?: SemanticTarget;
+        selectorAlternates?: string[];
+    }, timeoutMs?: number): Promise<boolean>;
+    /** Scroll an element into view by semantic target. */
+    scrollIntoViewByTarget?(opts: {
+        selector?: string;
+        target?: SemanticTarget;
+        selectorAlternates?: string[];
+    }): Promise<void>;
+}
+export {};

package/dist/v2/types.js

@@ -0,0 +1,19 @@
+/**
+ * Capture Agent — IR Types
+ *
+ * All types for the compiled execution model:
+ * preset (natural language) -> ExecutionProgram (typed IR) -> deterministic runtime
+ */
+export const DEFAULT_RECOVERY_POLICY = {
+    retries: 2,
+    useSelectorMemory: true,
+    useAltInteraction: true,
+    allowReload: false,
+    allowHealer: true,
+};
+export const DEFAULT_CIRCUIT_BREAKER = {
+    maxPerOpcode: 3,
+    maxPerPage: 5,
+    maxPerRun: 10,
+};
+//# sourceMappingURL=types.js.map

package/dist/v2/web-playwright-local.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Capture Agent — WebPlaywrightLocal RuntimeAdapter
+ *
+ * Thin adapter delegating to the existing Browser class from src/browser.ts.
+ * This is the first (and for now only) RuntimeAdapter implementation.
+ */
+import type { Browser } from '../browser.js';
+import type { AKTree, VideoPageSignals } from '../types.js';
+import type { RuntimeAdapter, ClickOptions, WaitCondition, RecordingOptions, RecordingResult } from './types.js';
+import { type ResolveOptions } from './semantic-resolver.js';
+export declare class WebPlaywrightLocal implements RuntimeAdapter {
+    private browser;
+    private recordingDir?;
+    private readonly sessionStartedAt;
+    private recording;
+    private clipCursor;
+    constructor(browser: Browser, recordingDir?: string | undefined);
+    navigate(url: string): Promise<void>;
+    getCurrentUrl(): Promise<string>;
+    getAKTree(): Promise<AKTree>;
+    getPageSignals(): Promise<VideoPageSignals>;
+    click(selector: string, options?: ClickOptions): Promise<void>;
+    /**
+     * Click an element using semantic target resolution.
+     * Tries CSS selector first, falls back to Playwright semantic locators.
+     */
+    clickByTarget(opts: ResolveOptions): Promise<void>;
+    /**
+     * Type into an element using semantic target resolution.
+     */
+    typeByTarget(opts: ResolveOptions, text: string, clearFirst?: boolean): Promise<void>;
+    /**
+     * Wait for an element using semantic target resolution.
+     */
+    waitForTarget(opts: ResolveOptions, timeoutMs?: number): Promise<boolean>;
+    /**
+     * Scroll an element into view using semantic target resolution.
+     */
+    scrollIntoViewByTarget(opts: ResolveOptions): Promise<void>;
+    type(selector: string, text: string, clearFirst?: boolean): Promise<void>;
+    pressKey(key: string): Promise<void>;
+    scroll(direction: 'up' | 'down' | 'left' | 'right', amount?: number): Promise<void>;
+    scrollIntoView(selector: string): Promise<void>;
+    waitFor(condition: WaitCondition): Promise<boolean>;
+    dismissOverlays(): Promise<{
+        dismissed: boolean;
+        method: string | null;
+    }>;
+    takeScreenshot(): Promise<Buffer>;
+    takeElementScreenshot(selector: string): Promise<Buffer>;
+    takeCleanScreenshot(): Promise<Buffer>;
+    beginRecording(options: RecordingOptions): Promise<void>;
+    endRecording(): Promise<RecordingResult>;
+    setLocale(locale: string): Promise<void>;
+    setColorScheme(scheme: 'light' | 'dark'): Promise<void>;
+    reloadPage(): Promise<void>;
+    writeStorageHint(params: {
+        storage: 'localStorage' | 'sessionStorage';
+        key: string;
+        value: string;
+        kind: 'locale' | 'theme';
+    }): Promise<boolean>;
+    close(): Promise<void>;
+    private typeIntoLocator;
+    private seedClipCursor;
+    private moveClipCursorToViewportCenter;
+    private moveClipCursorToLocator;
+    private moveClipCursorToPoint;
+}