autokap 1.0.7 → 1.0.8

package/dist/v2/explorer.js

@@ -0,0 +1,56 @@
+/**
+ * V2 Capture Agent — Site Explorer
+ *
+ * Deterministic exploration of the target site to capture AKTree snapshots.
+ * These snapshots ground the compiler's opcode generation in real DOM structure,
+ * preventing hallucinated selectors.
+ *
+ * The explorer uses the Browser class directly — no LLM calls.
+ */
+import { serializeAKTree } from '../ak-tree.js';
+/**
+ * Explores a single URL: navigates, dismisses overlays, and captures AKTree.
+ * Uses an existing Browser instance — does not create or close it.
+ */
+export async function explorePage(browser, options) {
+    const { url, dismissOverlays: shouldDismiss = true, loadTimeoutMs = 15000 } = options;
+    // Navigate to the target URL
+    await browser.navigateTo(url);
+    // Dismiss overlays if requested
+    let overlaysDismissed = false;
+    if (shouldDismiss) {
+        try {
+            const result = await browser.dismissOverlays();
+            overlaysDismissed = result.dismissed;
+        }
+        catch {
+            // Non-fatal: proceed without dismissal
+        }
+    }
+    // Capture AKTree
+    const akTree = await browser.getAKTree();
+    const akTreeSerialized = serializeAKTree(akTree);
+    const currentUrl = (await browser.currentPage).url();
+    return {
+        akTreeSerialized,
+        akTree,
+        url: currentUrl,
+        overlaysDismissed,
+    };
+}
+/**
+ * Explores multiple URLs on the same site.
+ * Useful for presets that involve navigation across pages.
+ */
+export async function explorePages(browser, urls, options) {
+    const results = [];
+    for (const url of urls) {
+        const result = await explorePage(browser, {
+            url,
+            dismissOverlays: options?.dismissOverlays ?? true,
+        });
+        results.push(result);
+    }
+    return results;
+}
+//# sourceMappingURL=explorer.js.map

package/dist/v2/index.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Capture Agent — Public API
+ *
+ * Architecture: deterministic opcode execution with selectors set by the user's AI assistant.
+ * AutoKap AI is limited to: overlay dismissal + capture verification.
+ */
+export type { ExecutionProgram, ExecutionOpcode, OpcodeKind, SemanticTarget, PostconditionSpec, PostconditionType, RecoveryPolicy, VariantSpec, PreconditionSpec, ArtifactSpec, MediaMode, CircuitBreakerConfig, HealerPatch, OpcodeResult, OpcodeResultStatus, VariantResult, ArtifactResult, RunResult, RunTelemetry, LLMStepUsage, LLMStepType, RuntimeAdapter, ClickOptions, WaitCondition, RecordingOptions, RecordingResult, } from './types.js';
+export { DEFAULT_RECOVERY_POLICY, DEFAULT_CIRCUIT_BREAKER } from './types.js';
+export { ExecutionProgramSchema, ExecutionOpcodeSchema, parseProgram, parseOpcode, safeParseProgramResult, } from './schema.js';
+export { WebPlaywrightLocal } from './web-playwright-local.js';
+export { evaluatePostcondition } from './postcondition.js';
+export { ActionVerifier } from './action-verifier.js';
+export { CircuitBreaker } from './circuit-breaker.js';
+export { executeProgram, NoOpRecoveryChain } from './opcode-runner.js';
+export type { RecoveryChain, RecoveryAttemptResult, RunOptions, ProgressEvent } from './opcode-runner.js';
+export { resolveSelector } from './selector-resolver.js';
+export type { ResolvedSelector, SelectorResolverOptions } from './selector-resolver.js';
+export { resolveTarget } from './semantic-resolver.js';
+export type { ResolvedTarget, ResolveOptions } from './semantic-resolver.js';
+export { RecoveryChainImpl } from './recovery-chain.js';
+export { LLMHealer } from './llm-healer.js';
+export type { HealerLLMProvider, HealerContext, HealerResult } from './llm-healer.js';
+export { applyPatch, applyAllPatches, cloneProgram } from './program-patcher.js';
+export { createCaptureStrategy, ScreenshotStrategy, ClipStrategy } from './capture-strategy.js';
+export type { CaptureStrategy } from './capture-strategy.js';
+export { dismissAllOverlays } from './overlay-engine.js';
+export type { OverlayDismissResult } from './overlay-engine.js';
+export { callLLM } from './llm-provider.js';
+export type { LLMProviderConfig, LLMCallResult } from './llm-provider.js';
+export { verifyCaptureQuality } from './capture-verification.js';
+export type { VerificationResult, VerificationContext } from './capture-verification.js';
+export { generateAltText } from './alt-text.js';
+export type { AltTextResult, AltTextContext } from './alt-text.js';
+export { smartWaitForStability } from './smart-wait.js';
+export type { SmartWaitResult } from './smart-wait.js';
+export { runBenchmark, formatBenchmarkSummary } from './benchmark.js';
+export type { BenchmarkPreset, BenchmarkRunResult, BenchmarkSummary } from './benchmark.js';

package/dist/v2/index.js

@@ -0,0 +1,31 @@
+/**
+ * Capture Agent — Public API
+ *
+ * Architecture: deterministic opcode execution with selectors set by the user's AI assistant.
+ * AutoKap AI is limited to: overlay dismissal + capture verification.
+ */
+export { DEFAULT_RECOVERY_POLICY, DEFAULT_CIRCUIT_BREAKER } from './types.js';
+export { ExecutionProgramSchema, ExecutionOpcodeSchema, parseProgram, parseOpcode, safeParseProgramResult, } from './schema.js';
+// ── Runtime ─────────────────────────────────────────────────────────
+export { WebPlaywrightLocal } from './web-playwright-local.js';
+export { evaluatePostcondition } from './postcondition.js';
+export { ActionVerifier } from './action-verifier.js';
+export { CircuitBreaker } from './circuit-breaker.js';
+export { executeProgram, NoOpRecoveryChain } from './opcode-runner.js';
+// ── Recovery & Healing ──────────────────────────────────────────────
+export { resolveSelector } from './selector-resolver.js';
+export { resolveTarget } from './semantic-resolver.js';
+export { RecoveryChainImpl } from './recovery-chain.js';
+export { LLMHealer } from './llm-healer.js';
+export { applyPatch, applyAllPatches, cloneProgram } from './program-patcher.js';
+// ── Capture & Overlays ──────────────────────────────────────────────
+export { createCaptureStrategy, ScreenshotStrategy, ClipStrategy } from './capture-strategy.js';
+export { dismissAllOverlays } from './overlay-engine.js';
+// ── AI Services (verification, alt text, smart wait) ────────────────
+export { callLLM } from './llm-provider.js';
+export { verifyCaptureQuality } from './capture-verification.js';
+export { generateAltText } from './alt-text.js';
+export { smartWaitForStability } from './smart-wait.js';
+// ── Benchmark ───────────────────────────────────────────────────────
+export { runBenchmark, formatBenchmarkSummary } from './benchmark.js';
+//# sourceMappingURL=index.js.map

package/dist/v2/llm-healer.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Capture Agent — LLM Healer
+ *
+ * Last-resort recovery: when all deterministic strategies fail,
+ * the healer asks an LLM to analyze the current page state and
+ * produce a replacement opcode.
+ *
+ * Constraints:
+ * - Cannot change the preset's intention
+ * - Cannot change variant order
+ * - Cannot invent new capture targets
+ * - Cannot skip the failed opcode's postcondition
+ * - Max 1 LLM call per healing attempt
+ * - Max 3 healing attempts per run
+ */
+import type { ExecutionOpcode, HealerPatch } from './types.js';
+export interface HealerContext {
+    /** The failed opcode */
+    failedOpcode: ExecutionOpcode;
+    /** Index in the program */
+    opcodeIndex: number;
+    /** Serialized AKTree of the current page */
+    akTreeSerialized: string;
+    /** Current page URL */
+    currentUrl: string;
+    /** Screenshot buffer (for multimodal models) */
+    screenshot?: Buffer;
+    /** The 3 opcodes before and after the failed one (for context) */
+    surroundingOpcodes: ExecutionOpcode[];
+    /** Error message from the failed attempt */
+    errorMessage: string;
+}
+export interface HealerResult {
+    healed: boolean;
+    patch?: HealerPatch;
+    reason: string;
+    /** LLM cost for this healing attempt */
+    costEur?: number;
+}
+export interface HealerLLMProvider {
+    /**
+     * Call the LLM with a healing prompt. Returns raw JSON string.
+     * The provider handles model selection, API keys, etc.
+     */
+    call(prompt: string, screenshot?: Buffer): Promise<{
+        response: string;
+        costEur: number;
+    }>;
+}
+/**
+ * The LLM Healer — a constrained agent that repairs failed opcodes.
+ */
+export declare class LLMHealer {
+    private llmProvider;
+    private invocationCount;
+    private maxInvocations;
+    constructor(llmProvider: HealerLLMProvider, options?: {
+        maxInvocations?: number;
+    });
+    get remainingInvocations(): number;
+    heal(context: HealerContext): Promise<HealerResult>;
+}

package/dist/v2/llm-healer.js

@@ -0,0 +1,166 @@
+/**
+ * Capture Agent — LLM Healer
+ *
+ * Last-resort recovery: when all deterministic strategies fail,
+ * the healer asks an LLM to analyze the current page state and
+ * produce a replacement opcode.
+ *
+ * Constraints:
+ * - Cannot change the preset's intention
+ * - Cannot change variant order
+ * - Cannot invent new capture targets
+ * - Cannot skip the failed opcode's postcondition
+ * - Max 1 LLM call per healing attempt
+ * - Max 3 healing attempts per run
+ */
+import { ExecutionOpcodeSchema } from './schema.js';
+/**
+ * The LLM Healer — a constrained agent that repairs failed opcodes.
+ */
+export class LLMHealer {
+    llmProvider;
+    invocationCount = 0;
+    maxInvocations;
+    constructor(llmProvider, options) {
+        this.llmProvider = llmProvider;
+        this.maxInvocations = options?.maxInvocations ?? 3;
+    }
+    get remainingInvocations() {
+        return Math.max(0, this.maxInvocations - this.invocationCount);
+    }
+    async heal(context) {
+        if (this.invocationCount >= this.maxInvocations) {
+            return { healed: false, reason: `healer budget exhausted (${this.maxInvocations}/${this.maxInvocations} used)` };
+        }
+        this.invocationCount++;
+        try {
+            const prompt = buildHealerPrompt(context);
+            const { response, costEur } = await this.llmProvider.call(prompt, context.screenshot);
+            // Parse the LLM response
+            const parsed = parseHealerResponse(response, context);
+            if (!parsed) {
+                return { healed: false, reason: 'healer produced invalid response', costEur };
+            }
+            if (parsed.cannotHeal) {
+                return { healed: false, reason: parsed.reason ?? 'healer cannot resolve this failure', costEur };
+            }
+            // Validate replacement opcodes against schema
+            const validOpcodes = [];
+            for (const opcode of parsed.replacementOpcodes) {
+                const result = ExecutionOpcodeSchema.safeParse(opcode);
+                if (!result.success) {
+                    return { healed: false, reason: `healer produced invalid opcode: ${result.error}`, costEur };
+                }
+                validOpcodes.push(result.data);
+            }
+            if (validOpcodes.length === 0 || validOpcodes.length > 3) {
+                return { healed: false, reason: `healer produced ${validOpcodes.length} opcodes (expected 1-3)`, costEur };
+            }
+            const patch = {
+                opcodeIndex: context.opcodeIndex,
+                originalOpcode: context.failedOpcode,
+                replacementOpcodes: validOpcodes,
+                reason: parsed.reason ?? 'healer repair',
+                patchedAt: new Date().toISOString(),
+            };
+            return { healed: true, patch, reason: parsed.reason ?? 'healer repair applied', costEur };
+        }
+        catch (err) {
+            return {
+                healed: false,
+                reason: `healer error: ${err instanceof Error ? err.message : String(err)}`,
+            };
+        }
+    }
+}
+// ── Prompt construction ─────────────────────────────────────────────
+function buildHealerPrompt(context) {
+    const { failedOpcode, errorMessage, akTreeSerialized, currentUrl, surroundingOpcodes } = context;
+    const surroundingDesc = surroundingOpcodes
+        .map((op, i) => `  ${i + 1}. ${op.kind}: ${op.description}`)
+        .join('\n');
+    return `You are a web automation repair agent. An opcode in a deterministic browser automation program has failed. Your job is to analyze the current page state and produce a replacement opcode that achieves the same goal.
+
+## Failed Opcode
+Kind: ${failedOpcode.kind}
+Description: ${failedOpcode.description}
+${failedOpcode.kind === 'CLICK' ? `Selector: ${failedOpcode.selector}` : ''}
+${failedOpcode.kind === 'TYPE' ? `Selector: ${failedOpcode.selector}, Text: ${failedOpcode.text}` : ''}
+${failedOpcode.kind === 'WAIT_FOR' ? `Selector: ${failedOpcode.selector}` : ''}
+
+## Error
+${errorMessage}
+
+## Current Page
+URL: ${currentUrl}
+
+## AKTree (current page structure)
+${akTreeSerialized.slice(0, 8000)}
+
+## Surrounding Opcodes (context)
+${surroundingDesc}
+
+## Rules
+1. You MUST produce the same kind of opcode (${failedOpcode.kind}) or a sequence of 1-3 opcodes that achieve the same postcondition.
+2. You CANNOT change the intention — only fix the selector or interaction method.
+3. You CANNOT invent new navigation targets or capture actions.
+4. Every opcode must have a valid postcondition.
+5. Use selectors you can actually see in the AKTree above.
+
+## Response Format
+Respond with a JSON object:
+{
+  "cannotHeal": false,
+  "reason": "explanation of what you changed",
+  "replacementOpcodes": [
+    { "kind": "${failedOpcode.kind}", "description": "...", "selector": "...", "postcondition": {...}, "recovery": {...}, "timeoutMs": 15000, "maxFailures": 3, ... }
+  ]
+}
+
+Or if you cannot fix it:
+{
+  "cannotHeal": true,
+  "reason": "explanation of why this cannot be fixed"
+}
+
+Respond ONLY with the JSON object, no markdown, no explanation outside JSON.`;
+}
+function parseHealerResponse(response, context) {
+    try {
+        // Extract JSON from response (handle markdown code blocks)
+        const jsonStr = response
+            .replace(/```json\s*/g, '')
+            .replace(/```\s*/g, '')
+            .trim();
+        const parsed = JSON.parse(jsonStr);
+        if (typeof parsed !== 'object' || parsed === null)
+            return null;
+        if (parsed.cannotHeal) {
+            return {
+                cannotHeal: true,
+                reason: parsed.reason ?? null,
+                replacementOpcodes: [],
+            };
+        }
+        if (!Array.isArray(parsed.replacementOpcodes))
+            return null;
+        // Inject the original postcondition and recovery into replacement opcodes
+        // (healer cannot skip postconditions)
+        const enriched = parsed.replacementOpcodes.map((op) => ({
+            ...op,
+            postcondition: op.postcondition ?? context.failedOpcode.postcondition,
+            recovery: op.recovery ?? context.failedOpcode.recovery,
+            timeoutMs: op.timeoutMs ?? context.failedOpcode.timeoutMs,
+            maxFailures: op.maxFailures ?? context.failedOpcode.maxFailures,
+        }));
+        return {
+            cannotHeal: false,
+            reason: parsed.reason ?? null,
+            replacementOpcodes: enriched,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=llm-healer.js.map

package/dist/v2/llm-provider.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Capture Agent — LLM Provider
+ *
+ * Thin wrapper around OpenAI SDK for OpenRouter calls.
+ * Used by: capture verification, alt text generation, healer.
+ */
+export interface LLMCallResult {
+    text: string;
+    model: string;
+    promptTokens: number;
+    completionTokens: number;
+    costEur: number;
+    /** OpenRouter generation ID for async cost resolution */
+    generationId: string | null;
+}
+export interface LLMProviderConfig {
+    apiKey: string;
+    /** Model to use. Default: google/gemini-2.5-flash */
+    model?: string;
+    /** Max tokens for response. Default: 200 */
+    maxTokens?: number;
+    /** Timeout in ms. Default: 15000 */
+    timeoutMs?: number;
+}
+/**
+ * Makes a single LLM call with optional image input.
+ * Returns the text response and usage metadata.
+ */
+export declare function callLLM(config: LLMProviderConfig, systemPrompt: string, userContent: string, image?: Buffer): Promise<LLMCallResult>;

package/dist/v2/llm-provider.js

@@ -0,0 +1,80 @@
+/**
+ * Capture Agent — LLM Provider
+ *
+ * Thin wrapper around OpenAI SDK for OpenRouter calls.
+ * Used by: capture verification, alt text generation, healer.
+ */
+import OpenAI from 'openai';
+import { createOpenRouterCompletion } from '../openrouter-client.js';
+const OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
+const DEFAULT_MODEL = 'google/gemini-2.5-flash';
+/**
+ * Makes a single LLM call with optional image input.
+ * Returns the text response and usage metadata.
+ */
+export async function callLLM(config, systemPrompt, userContent, image) {
+    const client = new OpenAI({
+        baseURL: OPENROUTER_BASE_URL,
+        apiKey: config.apiKey,
+    });
+    const model = config.model ?? DEFAULT_MODEL;
+    const maxTokens = config.maxTokens ?? 200;
+    const messages = [
+        { role: 'system', content: systemPrompt },
+    ];
+    if (image) {
+        const base64 = image.toString('base64');
+        messages.push({
+            role: 'user',
+            content: [
+                { type: 'text', text: userContent },
+                {
+                    type: 'image_url',
+                    image_url: { url: `data:image/png;base64,${base64}`, detail: 'low' },
+                },
+            ],
+        });
+    }
+    else {
+        messages.push({ role: 'user', content: userContent });
+    }
+    const params = {
+        model,
+        messages,
+        max_tokens: maxTokens,
+        temperature: 0.1,
+    };
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), config.timeoutMs ?? 15000);
+    try {
+        const response = await createOpenRouterCompletion(client, params, { signal: controller.signal });
+        clearTimeout(timer);
+        const text = response.choices?.[0]?.message?.content?.trim() ?? '';
+        const usage = response.usage;
+        return {
+            text,
+            model: response.model ?? model,
+            promptTokens: usage?.prompt_tokens ?? 0,
+            completionTokens: usage?.completion_tokens ?? 0,
+            costEur: estimateCost(model, usage?.prompt_tokens ?? 0, usage?.completion_tokens ?? 0),
+            generationId: response.id ?? null,
+        };
+    }
+    catch (err) {
+        clearTimeout(timer);
+        throw err;
+    }
+}
+/**
+ * Rough cost estimation based on known model pricing.
+ * Actual cost is resolved asynchronously via OpenRouter generation API.
+ */
+function estimateCost(model, promptTokens, completionTokens) {
+    // Gemini 2.5 Flash pricing (approximate)
+    if (model.includes('gemini')) {
+        return (promptTokens * 0.15 + completionTokens * 0.6) / 1_000_000;
+    }
+    // Fallback: conservative estimate
+    return (promptTokens * 0.5 + completionTokens * 1.5) / 1_000_000;
+}
+//# sourceMappingURL=llm-provider.js.map

package/dist/v2/opcode-runner.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Capture Agent — Opcode Runner
+ *
+ * Deterministic execution engine for ExecutionProgram.
+ * Executes opcodes sequentially, verifies postconditions,
+ * delegates to recovery chain on failure, and respects circuit breaker.
+ */
+import type { ExecutionProgram, ExecutionOpcode, RuntimeAdapter, OpcodeResultStatus, RunResult, HealerPatch, VariantSpec } from './types.js';
+import type { LLMProviderConfig } from './llm-provider.js';
+export interface RecoveryChain {
+    attempt(failedOpcode: ExecutionOpcode, opcodeIndex: number, adapter: RuntimeAdapter, options?: RecoveryAttemptOptions): Promise<RecoveryAttemptResult>;
+}
+export interface RecoveryAttemptResult {
+    recovered: boolean;
+    strategy?: 'retry' | 'selector_memory' | 'alt_interaction' | 'reload' | 'healer';
+    patch?: HealerPatch;
+    reason: string;
+}
+export interface RecoveryAttemptOptions {
+    remainingTimeMs?: number;
+    maxDeterministicRetries?: number;
+}
+/** No-op recovery chain for Phase 1 (deterministic-only execution) */
+export declare class NoOpRecoveryChain implements RecoveryChain {
+    attempt(): Promise<RecoveryAttemptResult>;
+}
+export interface RunOptions {
+    /** Recovery chain implementation. Defaults to no-op. */
+    recoveryChain?: RecoveryChain;
+    /** Abort signal for cancellation */
+    abortSignal?: AbortSignal;
+    /** Callback for progress updates */
+    onProgress?: (event: ProgressEvent) => void;
+    /** LLM config for capture verification + alt text. If absent, verification is skipped. */
+    llmConfig?: LLMProviderConfig;
+    /** Preset name for alt text context */
+    presetName?: string;
+}
+export interface ProgressEvent {
+    type: 'variant_start' | 'variant_end' | 'opcode_start' | 'opcode_end' | 'recovery' | 'breaker_trip';
+    variantId: string;
+    opcodeIndex?: number;
+    opcodeKind?: string;
+    status?: OpcodeResultStatus;
+    message: string;
+}
+export declare function executeProgram(program: ExecutionProgram, createAdapter: (variant: VariantSpec) => Promise<RuntimeAdapter>, options?: RunOptions): Promise<RunResult>;