autokap 1.0.8 → 1.1.0

package/dist/llm-healer.js

@@ -3,19 +3,10 @@
  *
  * Last-resort recovery: when all deterministic strategies fail,
  * the healer asks an LLM to analyze the current page state and
- * produce a replacement opcode.
- *
- * Constraints:
- * - Cannot change the preset's intention
- * - Cannot change variant order
- * - Cannot invent new capture targets
- * - Cannot skip the failed opcode's postcondition
- * - Max 1 LLM call per healing attempt
- * - Max 3 healing attempts per run
+ * produce a bounded selector patch.
  */
-import { ExecutionOpcodeSchema } from './execution-schema.js';
 /**
- * The LLM Healer — a constrained agent that repairs failed opcodes.
+ * The LLM Healer — a constrained agent that repairs failed selectors.
  */
 export class LLMHealer {
     llmProvider;
@@ -36,34 +27,27 @@ export class LLMHealer {
         try {
             const prompt = buildHealerPrompt(context);
             const { response, llmResult } = await this.llmProvider.call(prompt, context.screenshot);
-            // Parse the LLM response
-            const parsed = parseHealerResponse(response, context);
+            const parsed = parseHealerResponse(response);
             if (!parsed) {
                 return { healed: false, reason: 'healer produced invalid response', llmResult };
             }
             if (parsed.cannotHeal) {
                 return { healed: false, reason: parsed.reason ?? 'healer cannot resolve this failure', llmResult };
             }
-            // Validate replacement opcodes against schema
-            const validOpcodes = [];
-            for (const opcode of parsed.replacementOpcodes) {
-                const result = ExecutionOpcodeSchema.safeParse(opcode);
-                if (!result.success) {
-                    return { healed: false, reason: `healer produced invalid opcode: ${result.error}`, llmResult };
-                }
-                validOpcodes.push(result.data);
-            }
-            if (validOpcodes.length === 0 || validOpcodes.length > 3) {
-                return { healed: false, reason: `healer produced ${validOpcodes.length} opcodes (expected 1-3)`, llmResult };
+            const replacementOpcode = buildReplacementOpcode(context.failedOpcode, parsed);
+            if (!replacementOpcode) {
+                return { healed: false, reason: 'healer could not produce a bounded selector patch', llmResult };
             }
             const patch = {
                 opcodeIndex: context.opcodeIndex,
                 originalOpcode: context.failedOpcode,
-                replacementOpcodes: validOpcodes,
-                reason: parsed.reason ?? 'healer repair',
+                replacementOpcodes: [replacementOpcode],
+                patchType: 'selector_patch',
+                interactionMode: parsed.interactionMode,
+                reason: parsed.reason ?? 'healer selector repair',
                 patchedAt: new Date().toISOString(),
             };
-            return { healed: true, patch, reason: parsed.reason ?? 'healer repair applied', llmResult };
+            return { healed: true, patch, reason: patch.reason, llmResult };
         }
         catch (err) {
             return {
@@ -73,48 +57,50 @@ export class LLMHealer {
         }
     }
 }
-// ── Prompt construction ─────────────────────────────────────────────
 function buildHealerPrompt(context) {
     const { failedOpcode, errorMessage, akTreeSerialized, currentUrl, surroundingOpcodes } = context;
+    const selector = hasPatchableSelector(failedOpcode) ? failedOpcode.selector : null;
     const surroundingDesc = surroundingOpcodes
-        .map((op, i) => `  ${i + 1}. ${op.kind}: ${op.description}`)
+        .map((op, i) => `  ${i + 1}. ${op.kind}: ${sanitizePromptText(op.description)}`)
         .join('\n');
-    return `You are a web automation repair agent. An opcode in a deterministic browser automation program has failed. Your job is to analyze the current page state and produce a replacement opcode that achieves the same goal.
+    return `You are a web automation repair agent. An opcode in a deterministic browser automation program has failed. Your job is to repair the selector only.
 
 ## Failed Opcode
 Kind: ${failedOpcode.kind}
-Description: ${failedOpcode.description}
-${failedOpcode.kind === 'CLICK' ? `Selector: ${failedOpcode.selector}` : ''}
-${failedOpcode.kind === 'TYPE' ? `Selector: ${failedOpcode.selector}, Text: ${failedOpcode.text}` : ''}
-${failedOpcode.kind === 'WAIT_FOR' ? `Selector: ${failedOpcode.selector}` : ''}
+Description: ${sanitizePromptText(failedOpcode.description)}
+Selector: ${selector ?? '<unsupported>'}
 
 ## Error
-${errorMessage}
+<<<ERROR>>>
+${sanitizePromptText(errorMessage)}
+<<<END_ERROR>>>
 
 ## Current Page
-URL: ${currentUrl}
+URL: ${sanitizeUrlForPrompt(currentUrl)}
 
 ## AKTree (current page structure)
-${akTreeSerialized.slice(0, 8000)}
+<<<AKTREE>>>
+${sanitizePromptText(akTreeSerialized).slice(0, 8000)}
+<<<END_AKTREE>>>
 
 ## Surrounding Opcodes (context)
 ${surroundingDesc}
 
 ## Rules
-1. You MUST produce the same kind of opcode (${failedOpcode.kind}) or a sequence of 1-3 opcodes that achieve the same postcondition.
-2. You CANNOT change the intention — only fix the selector or interaction method.
-3. You CANNOT invent new navigation targets or capture actions.
-4. Every opcode must have a valid postcondition.
-5. Use selectors you can actually see in the AKTree above.
+1. You MUST NOT emit opcodes or navigation targets.
+2. You may only return a selector patch for the existing opcode.
+3. You may optionally return a bounded interaction mode: default, keyboard, js_dispatch, or coordinates.
+4. Use selectors you can actually see in the AKTree above.
+5. If the opcode kind is unsupported or there is no reliable selector, return cannotHeal=true.
 
 ## Response Format
 Respond with a JSON object:
 {
   "cannotHeal": false,
   "reason": "explanation of what you changed",
-  "replacementOpcodes": [
-    { "kind": "${failedOpcode.kind}", "description": "...", "selector": "...", "postcondition": {...}, "recovery": {...}, "timeoutMs": 15000, "maxFailures": 3, ... }
-  ]
+  "selector": "[data-ak='submit']",
+  "selectorAlternates": ["button[type='submit']"],
+  "interactionMode": "default"
 }
 
 Or if you cannot fix it:
@@ -125,42 +111,103 @@ Or if you cannot fix it:
 
 Respond ONLY with the JSON object, no markdown, no explanation outside JSON.`;
 }
-function parseHealerResponse(response, context) {
+function parseHealerResponse(response) {
     try {
-        // Extract JSON from response (handle markdown code blocks)
         const jsonStr = response
             .replace(/```json\s*/g, '')
             .replace(/```\s*/g, '')
             .trim();
         const parsed = JSON.parse(jsonStr);
-        if (typeof parsed !== 'object' || parsed === null)
+        if (!parsed || typeof parsed !== 'object')
             return null;
         if (parsed.cannotHeal) {
             return {
                 cannotHeal: true,
-                reason: parsed.reason ?? null,
-                replacementOpcodes: [],
+                reason: typeof parsed.reason === 'string' ? parsed.reason : null,
+                selector: null,
+                selectorAlternates: [],
+                interactionMode: 'default',
             };
         }
-        if (!Array.isArray(parsed.replacementOpcodes))
+        const selector = typeof parsed.selector === 'string' && parsed.selector.trim()
+            ? parsed.selector.trim()
+            : null;
+        if (!selector)
             return null;
-        // Inject the original postcondition and recovery into replacement opcodes
-        // (healer cannot skip postconditions)
-        const enriched = parsed.replacementOpcodes.map((op) => ({
-            ...op,
-            postcondition: op.postcondition ?? context.failedOpcode.postcondition,
-            recovery: op.recovery ?? context.failedOpcode.recovery,
-            timeoutMs: op.timeoutMs ?? context.failedOpcode.timeoutMs,
-            maxFailures: op.maxFailures ?? context.failedOpcode.maxFailures,
-        }));
         return {
             cannotHeal: false,
-            reason: parsed.reason ?? null,
-            replacementOpcodes: enriched,
+            reason: typeof parsed.reason === 'string' ? parsed.reason : null,
+            selector,
+            selectorAlternates: Array.isArray(parsed.selectorAlternates)
+                ? parsed.selectorAlternates
+                    .filter((entry) => typeof entry === 'string' && entry.trim().length > 0)
+                    .map((entry) => entry.trim())
+                    .slice(0, 5)
+                : [],
+            interactionMode: parsed.interactionMode === 'keyboard'
+                || parsed.interactionMode === 'js_dispatch'
+                || parsed.interactionMode === 'coordinates'
+                ? parsed.interactionMode
+                : 'default',
         };
     }
     catch {
         return null;
     }
 }
+function hasPatchableSelector(opcode) {
+    return ((opcode.kind === 'CLICK'
+        || opcode.kind === 'TYPE'
+        || opcode.kind === 'WAIT_FOR'
+        || opcode.kind === 'HOVER'
+        || opcode.kind === 'SELECT_OPTION'
+        || opcode.kind === 'CHECK'
+        || opcode.kind === 'DOUBLE_CLICK')
+        && typeof opcode.selector === 'string'
+        && opcode.selector.trim().length > 0);
+}
+function buildReplacementOpcode(failedOpcode, parsed) {
+    if (!hasPatchableSelector(failedOpcode) || !parsed.selector) {
+        return null;
+    }
+    switch (failedOpcode.kind) {
+        case 'CLICK':
+        case 'TYPE':
+        case 'WAIT_FOR':
+        case 'HOVER':
+        case 'SELECT_OPTION':
+        case 'CHECK':
+        case 'DOUBLE_CLICK':
+            return {
+                ...failedOpcode,
+                selector: parsed.selector,
+                ...('selectorAlternates' in failedOpcode
+                    ? { selectorAlternates: parsed.selectorAlternates }
+                    : {}),
+            };
+        default:
+            return null;
+    }
+}
+function sanitizePromptText(value) {
+    return value
+        .replace(/\{\{(email|password|loginUrl)\}\}/g, '<credential-placeholder>')
+        .replace(/data:[^"' )]+/gi, '<data-url>')
+        .replace(/https?:\/\/[^\s"'<>]+/gi, (match) => sanitizeUrlForPrompt(match))
+        .replace(/[^\x09\x0a\x0d\x20-\x7e]/g, ' ')
+        .slice(0, 12_000);
+}
+function sanitizeUrlForPrompt(value) {
+    try {
+        const parsed = new URL(value);
+        parsed.username = '';
+        parsed.password = '';
+        parsed.search = parsed.search ? '?<redacted>' : '';
+        parsed.hash = parsed.hash ? '#<redacted>' : '';
+        return parsed.toString();
+    }
+    catch {
+        return value.slice(0, 512);
+    }
+}
 //# sourceMappingURL=llm-healer.js.map

package/dist/llm-provider.js

@@ -6,6 +6,7 @@
  */
 import OpenAI from 'openai';
 import { createOpenRouterCompletion } from './openrouter-client.js';
+import { assertZdrCompatibleModel, zdrParam } from './provider-config.js';
 const OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
 const DEFAULT_MODEL = 'google/gemini-2.5-flash';
 /**
@@ -19,6 +20,7 @@ export async function callLLM(config, systemPrompt, userContent, image) {
     });
     const model = config.model ?? DEFAULT_MODEL;
     const maxTokens = config.maxTokens ?? 200;
+    assertZdrCompatibleModel(model);
     const messages = [
         { role: 'system', content: systemPrompt },
     ];
@@ -43,6 +45,7 @@ export async function callLLM(config, systemPrompt, userContent, image) {
         messages,
         max_tokens: maxTokens,
         temperature: 0.1,
+        provider: Object.keys(zdrParam()).length > 0 ? zdrParam() : undefined,
     };
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), config.timeoutMs ?? 15000);

package/dist/opcode-actions.js

@@ -171,6 +171,19 @@ export async function executeOpcodeCoreAction(opcode, adapter, context = {}) {
                     return { success: false, error: 'adapter does not support DOUBLE_CLICK' };
                 await adapter.doubleClick(opcode.selector);
                 break;
+            case 'DRAG':
+                if (!adapter.drag)
+                    return { success: false, error: 'adapter does not support DRAG' };
+                await adapter.drag({
+                    selector: opcode.selector,
+                    target: opcode.target,
+                    selectorAlternates: opcode.selectorAlternates,
+                    toSelector: opcode.toSelector,
+                    toTarget: opcode.toTarget,
+                    toSelectorAlternates: opcode.toSelectorAlternates,
+                    offset: opcode.offset,
+                });
+                break;
             case 'CLONE_ELEMENT':
                 if (!adapter.cloneElement) {
                     return { success: false, error: 'adapter does not support CLONE_ELEMENT' };

package/dist/opcode-runner.js

@@ -53,6 +53,13 @@ export class NoOpRecoveryChain {
         return { recovered: false, reason: 'no recovery chain configured' };
     }
 }
+const MIN_CLIP_FINALIZATION_TIMEOUT_MS = 30000;
+function resolveOpcodeTimeoutMs(opcode) {
+    if (opcode.kind === 'END_CLIP') {
+        return Math.max(opcode.timeoutMs, MIN_CLIP_FINALIZATION_TIMEOUT_MS);
+    }
+    return opcode.timeoutMs;
+}
 // ── Main execution function ─────────────────────────────────────────
 export async function executeProgram(program, createAdapter, options = {}) {
     const recoveryChain = options.recoveryChain ?? new NoOpRecoveryChain();
@@ -238,7 +245,8 @@ function softSkipResult(opcode, index, startTime, reason, telemetry) {
 }
 async function executeOpcode(opcode, index, adapter, verifier, breaker, recoveryChain, telemetry, healerPatches, artifacts, options, variantId, executionState, artifactPlan, mockDataGroups, currentVariant, credentials) {
     const startTime = Date.now();
-    const deadlineMs = startTime + opcode.timeoutMs;
+    const effectiveTimeoutMs = resolveOpcodeTimeoutMs(opcode);
+    const deadlineMs = startTime + effectiveTimeoutMs;
     const isInteraction = ['CLICK', 'TYPE', 'PRESS_KEY', 'SCROLL'].includes(opcode.kind);
     const isSoft = isSoftOpcodeKind(opcode.kind);
     // Track page context for circuit breaker
@@ -249,7 +257,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
     catch { /* ignore */ }
     // Execute with timeout
     try {
-        logger.debug(`[opcode ${index}] ${opcode.kind} start — budget ${opcode.timeoutMs}ms${formatOpcodeDebug(opcode)}`);
+        logger.debug(`[opcode ${index}] ${opcode.kind} start — budget ${effectiveTimeoutMs}ms${formatOpcodeDebug(opcode)}`);
         if (isInteraction) {
             const beforeStart = Date.now();
             await verifier.captureBeforeState(adapter);
@@ -257,11 +265,11 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
         }
         const actionBudgetMs = getRemainingTimeMs(deadlineMs);
         if (actionBudgetMs <= 0) {
-            const reason = `timeout after ${opcode.timeoutMs}ms`;
+            const reason = `timeout after ${effectiveTimeoutMs}ms`;
             logger.debug(`[opcode ${index}] no budget left after captureBeforeState (deadline=${deadlineMs}, now=${Date.now()})`);
             if (isSoft)
                 return softSkipResult(opcode, index, startTime, reason, telemetry);
-            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, reason);
+            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, effectiveTimeoutMs, reason);
         }
         logger.debug(`[opcode ${index}] action exec start — actionBudget ${actionBudgetMs}ms`);
         const actionStart = Date.now();
@@ -271,16 +279,16 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
             const reason = result.error ?? 'action failed';
             if (isSoft)
                 return softSkipResult(opcode, index, startTime, reason, telemetry);
-            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, reason);
+            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, effectiveTimeoutMs, reason);
         }
         // Verify postcondition
         const postconditionBudgetMs = getRemainingTimeMs(deadlineMs);
         if (postconditionBudgetMs <= 0) {
-            const reason = `timeout after ${opcode.timeoutMs}ms`;
+            const reason = `timeout after ${effectiveTimeoutMs}ms`;
             logger.debug(`[opcode ${index}] no budget left for postcondition check`);
             if (isSoft)
                 return softSkipResult(opcode, index, startTime, reason, telemetry);
-            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, reason);
+            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, effectiveTimeoutMs, reason);
         }
         const postStart = Date.now();
         const postcondition = await evaluatePostcondition(adapter, withClampedPostconditionTimeout(opcode.postcondition, postconditionBudgetMs));
@@ -289,13 +297,13 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
             const reason = `postcondition failed: ${postcondition.reason}`;
             if (isSoft)
                 return softSkipResult(opcode, index, startTime, reason, telemetry);
-            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, reason);
+            return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, effectiveTimeoutMs, reason);
         }
         // Verify action had effect (for interaction opcodes)
         if (isInteraction) {
             const verification = await verifier.verifyAfterAction(adapter);
             if (!verification.hadEffect && opcode.postcondition.type !== 'always' && opcode.postcondition.type !== 'any_change') {
-                return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, `action had no effect: ${verification.summary}`);
+                return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, effectiveTimeoutMs, `action had no effect: ${verification.summary}`);
             }
         }
         // Record successful mock data application
@@ -317,11 +325,11 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
         const errorMsg = err instanceof Error ? err.message : String(err);
         if (isSoft)
             return softSkipResult(opcode, index, startTime, errorMsg, telemetry);
-        return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, errorMsg);
+        return handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, effectiveTimeoutMs, errorMsg);
     }
 }
 // ── Failure handling with recovery ──────────────────────────────────
-async function handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, errorMsg) {
+async function handleFailure(opcode, index, adapter, verifier, isInteraction, breaker, recoveryChain, telemetry, healerPatches, options, variantId, currentVariant, startTime, deadlineMs, effectiveTimeoutMs, errorMsg) {
     const breakerState = breaker.recordFailure(index, opcode.maxFailures);
     if (breakerState.tripped) {
         telemetry.circuitBreakerTrips++;
@@ -342,7 +350,7 @@ async function handleFailure(opcode, index, adapter, verifier, isInteraction, br
             status: 'failed',
             durationMs: Date.now() - startTime,
             recoveryAttempts: 0,
-            error: `${errorMsg} (timeout after ${opcode.timeoutMs}ms)`,
+            error: `${errorMsg} (timeout after ${effectiveTimeoutMs}ms)`,
         };
     }
     // Attempt recovery
@@ -424,6 +432,7 @@ async function executeOpcodeAction(opcode, opcodeIndex, adapter, artifacts, tele
             case 'SELECT_OPTION':
             case 'CHECK':
             case 'DOUBLE_CLICK':
+            case 'DRAG':
             case 'CLONE_ELEMENT':
             case 'INJECT_MOCK_DATA':
             case 'REMOVE_ELEMENT':