@tangle-network/browser-agent-driver 0.22.0 → 0.23.0

package/dist/runner/runner.js

@@ -20,10 +20,11 @@ import { detectSupervisorSignal, formatSupervisorSignal } from '../supervisor/po
 import { requestSupervisorDirective } from '../supervisor/critic.js';
 import { shouldAcceptFirstPartyBoundaryCompletion } from '../domain-policy.js';
 import { deriveWasteMetrics } from '../run-metrics.js';
-import { RunState } from '../run-state.js';
+import { RunState, DEFAULT_TOKEN_BUDGET } from '../run-state.js';
 import { ContextBudget } from '../context-budget.js';
 import { runOverridePipeline } from '../override-pipeline.js';
 import { withRetry, findElementForRef, safeHostname, pushGoalVerificationEvidence } from './utils.js';
+import { runExtractWithIndex, formatExtractWithIndexResult } from '../drivers/extract-with-index.js';
 import { buildSearchResultsGuidance, buildVisibleLinkRecommendation, getVisibleLinkRecommendation, getRankedVisibleLinkCandidates, rankSearchCandidates } from './search-guidance.js';
 import { buildGoalVerificationClaim, collectSearchWorkflowEvidence, shouldAcceptSearchWorkflowCompletion, shouldAcceptScriptBackedCompletion, detectCompletionContentTypeMismatch } from './goal-verification.js';
 import { verifyExpectedEffect } from './effect-verification.js';
@@ -130,7 +131,9 @@ const DEFAULT_MAX_TURNS = 20;
 const DEFAULT_RETRIES = 3;
 const DEFAULT_RETRY_DELAY_MS = 1000;
 const DEFAULT_MICRO_PLAN_ACTIONS = 2;
-const SAFE_MICRO_ACTIONS = new Set(['click', 'type', 'press', 'hover', 'select', 'scroll', 'wait']);
+// Gen 18: clickAt/typeAt added so vision-mode can emit multi-action turns
+// Gen 23: clickLabel/typeLabel for SoM-based actions
+const SAFE_MICRO_ACTIONS = new Set(['click', 'type', 'press', 'hover', 'select', 'scroll', 'wait', 'clickAt', 'typeAt', 'clickLabel', 'typeLabel']);
 const DEFAULT_SUPERVISOR = {
     enabled: true,
     useVision: true,
@@ -183,6 +186,60 @@ export function hasPlaceholderPattern(text) {
     }
     return false;
 }
+/**
+ * Gen 9 — runtime two-pass extraction. When the planner emits a single
+ * runScript step (per Gen 7.2 rule #7) and that script returns null /
+ * empty / whitespace / `{x: null}` / a placeholder pattern, the auto-
+ * complete-from-runScript path should NOT fire. Instead the runner should
+ * mark the plan as deviated and fall through to the per-action loop where
+ * Brain.decide can re-observe the loaded page and emit a smarter action
+ * (different selector, click+wait, scroll, etc.).
+ *
+ * This addresses the failure mode the Gen 8 head-to-head gauntlet
+ * surfaced: bad's planner-only path lost to browser-use's per-action loop
+ * on tasks where the first runScript pick was wrong (npm, mdn signature,
+ * w3c, github, wikipedia variance). Two-pass gives bad's per-action loop
+ * the same recovery surface browser-use uses, with the planner's speed
+ * advantage on the cases where runScript succeeds first try.
+ *
+ * "Meaningful" means: not empty/whitespace, not the literal string `null`
+ * or `undefined`, and not matching `hasPlaceholderPattern` (which already
+ * detects JSON null fields, "<from prior step>" markers, etc.).
+ */
+export function isMeaningfulRunScriptOutput(output) {
+    if (typeof output !== 'string')
+        return false;
+    const trimmed = output.trim();
+    if (trimmed.length === 0)
+        return false;
+    if (trimmed === 'null' || trimmed === 'undefined' || trimmed === '""' || trimmed === "''")
+        return false;
+    // Empty JSON shells: `{}`, `[]`, `{"x": null}`, `[null, null]`
+    if (trimmed === '{}' || trimmed === '[]')
+        return false;
+    if (hasPlaceholderPattern(trimmed))
+        return false;
+    // If the output parses as JSON and EVERY top-level value is null/empty,
+    // treat it as not meaningful. This catches `{"x": null, "y": ""}` even
+    // though the placeholder regex would already catch the null one.
+    try {
+        const parsed = JSON.parse(trimmed);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+            const values = Object.values(parsed);
+            if (values.length > 0) {
+                const allEmpty = values.every((v) => v === null || v === undefined || v === '' || v === 0);
+                if (allEmpty)
+                    return false;
+            }
+        }
+        if (Array.isArray(parsed) && parsed.length === 0)
+            return false;
+    }
+    catch {
+        // Not JSON, that's fine — fall through to "meaningful" if we got here.
+    }
+    return true;
+}
 export class BrowserAgent {
     driver;
     brain;
@@ -225,13 +282,24 @@ export class BrowserAgent {
         this.runRegistry = options.runRegistry;
     }
     async run(scenario) {
-        const maxTurns = scenario.maxTurns || DEFAULT_MAX_TURNS;
+        // Gen 14: vision mode gets more turns — each turn takes ~15s (screenshot
+        // encode + image tokens) vs ~5s for DOM-first. Without the boost, vision
+        // runs out of turns before completing multi-step tasks.
+        const isVisionMode = this.config.observationMode === 'vision' || this.config.observationMode === 'hybrid';
+        const baseMaxTurns = scenario.maxTurns || DEFAULT_MAX_TURNS;
+        // Gen 26: 30 turn minimum for vision. 15/51 failures were turn budget
+        // exhaustion at 20. The cost cap (200k tokens) is the real bound.
+        const maxTurns = isVisionMode ? Math.max(baseMaxTurns, 30) : baseMaxTurns;
         const retries = this.config.retries ?? DEFAULT_RETRIES;
         const retryDelayMs = this.config.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
         const turns = [];
         const startTime = Date.now();
         const phaseTimings = {};
-        const runState = new RunState(maxTurns);
+        // Gen 27: vision+planner mode gets 3× token budget (300k). Gen 26 showed
+        // 4 cost_cap failures and 18 turn-exhausted tasks (now getting 30 turns
+        // but hitting 200k cap). The timeout (600s) is the real safety net.
+        const visionBudgetMultiplier = isVisionMode ? 3 : 1;
+        const runState = new RunState(maxTurns, Math.round(DEFAULT_TOKEN_BUDGET * visionBudgetMultiplier));
         const runId = scenario.sessionId
             ? `${scenario.sessionId}_${Date.now()}`
             : RunRegistry.generateRunId();
@@ -278,10 +346,12 @@ export class BrowserAgent {
             });
             return agentResult;
         };
-        // Wrap onTurn to include mid-run manifest updates (every 3 turns)
+        // Wrap onTurn to include mid-run manifest updates (every 3 turns) and to
+        // accumulate per-turn token usage for the Gen 10 cost cap.
         const originalOnTurn = this.onTurn;
         this.onTurn = (turn) => {
             originalOnTurn?.(turn);
+            runState.recordTokens(turn.tokensUsed);
             if (this.runRegistry && turns.length % 3 === 0) {
                 try {
                     this.runRegistry.updateRun(runId, {
@@ -541,6 +611,18 @@ export class BrowserAgent {
                     totalMs: Date.now() - startTime,
                 });
             }
+            // Gen 10: hard cost cap. Stops the per-action loop from burning unbounded
+            // tokens on cases where recovery isn't converging (the Gen 9 death-spiral
+            // failure mode where reddit hit $0.32 / 173K tokens). The cap is enforced
+            // BEFORE the next LLM call so the case aborts cleanly with a reason.
+            if (runState.isTokenBudgetExhausted) {
+                return buildResult({
+                    success: false,
+                    reason: `cost_cap_exceeded: ${runState.totalTokensUsed} tokens used, budget ${runState.tokenBudget}`,
+                    turns,
+                    totalMs: Date.now() - startTime,
+                });
+            }
             const turnStart = Date.now();
             this.bus.emitNow({ type: 'turn-started', runId, turn: i });
             try {
@@ -680,7 +762,11 @@ export class BrowserAgent {
                                 terminalBlocker = detectTerminalBlocker(state);
                             }
                         }
-                        catch { /* CAPTCHA solve failed, fall through to abort */ }
+                        catch (captchaErr) {
+                            if (this.config.debug) {
+                                console.log('[Runner] CAPTCHA solve error:', captchaErr instanceof Error ? captchaErr.message : String(captchaErr));
+                            }
+                        }
                     }
                 }
                 if (terminalBlocker) {
@@ -964,6 +1050,30 @@ export class BrowserAgent {
                 if (lastTurn?.action.action === 'runScript' && !lastTurn.error) {
                     ctxBudget.add('extraction-guard', '\nData extracted. If it answers the goal, complete now.\n', 80);
                 }
+                // Gen 27: form stall detection — escalating urgency. Compare by
+                // origin+pathname (ignoring query params) because sites like Google
+                // Flights update URL params with each form interaction without actually
+                // navigating to results.
+                {
+                    const urlBase = (u) => { try {
+                        const p = new URL(u);
+                        return (p.origin + p.pathname).replace(/\/+$/, '');
+                    }
+                    catch {
+                        return u;
+                    } };
+                    const currentBase = urlBase(state.url);
+                    const sameBaseCount = turns.filter(t => t.state?.url && urlBase(t.state.url) === currentBase).length;
+                    if (sameBaseCount >= 15) {
+                        // Hard stall: 15+ turns on same page. Demand navigation away.
+                        // Use DuckDuckGo — Google Search triggers anti-bot CAPTCHAs.
+                        ctxBudget.add('form-stall', `\nCRITICAL FORM STALL: You have spent ${sameBaseCount} turns on this page without completing your goal. The form is NOT going to cooperate. Your NEXT action MUST be: navigate to https://duckduckgo.com/?q={your goal as a natural language search query}. Do NOT try the form again. Do NOT use google.com/search (it blocks automated access). Use DuckDuckGo.\n`, 95);
+                    }
+                    else if (sameBaseCount >= 10) {
+                        // Soft stall: suggest fallback but don't force it.
+                        ctxBudget.add('form-stall', `\nFORM STALL WARNING: You have been on this page for ${sameBaseCount} turns. Consider navigating to https://duckduckgo.com/?q={your goal as a search query} to find the answer via search results instead. Do NOT use google.com/search (it blocks automated browsers).\n`, 85);
+                    }
+                }
                 const extraContext = ctxBudget.build();
                 const forceVision = shouldEscalateVision({
                     config: this.config,
@@ -1247,6 +1357,9 @@ export class BrowserAgent {
                                 : JSON.stringify(scriptResult, null, 2);
                             runState.firstSufficientEvidenceTurn ??= i;
                             pushGoalVerificationEvidence(runState.goalVerificationEvidence, `SCRIPT RESULT:\n${stringified ?? '(undefined)'}`);
+                            if (typeof stringified === 'string' && stringified.length > 10) {
+                                runState.recordEvidence(`EXTRACTED (turn ${i}): ${stringified.slice(0, 500)}`);
+                            }
                             this.brain.injectFeedback(`SCRIPT RESULT:\n${stringified ?? '(undefined)'}`);
                         }
                         catch (scriptErr) {
@@ -1262,6 +1375,39 @@ export class BrowserAgent {
                     this.onTurn?.(turn);
                     continue;
                 }
+                // -- 5d. Handle extractWithIndex action (Gen 10) --
+                // Returns a numbered list of every visible element matching `query`,
+                // each with its tag, textContent, key attributes, and a stable
+                // selector. The agent picks elements by index in the next turn.
+                // This is the Gen 10 capability change: pick-by-content instead of
+                // pick-by-selector. Works on data the planner couldn't see at plan
+                // time (XHR-loaded content, dl/dt/dd, deeply-nested wrappers).
+                if (action.action === 'extractWithIndex') {
+                    const page = this.driver.getPage?.();
+                    if (page) {
+                        try {
+                            const matches = await runExtractWithIndex(page, action.query, action.contains);
+                            const formatted = formatExtractWithIndexResult(matches, action.query, action.contains);
+                            runState.firstSufficientEvidenceTurn ??= i;
+                            pushGoalVerificationEvidence(runState.goalVerificationEvidence, `EXTRACT RESULT (${matches.length} matches):\n${formatted}`);
+                            if (formatted.length > 10) {
+                                runState.recordEvidence(`EXTRACTED (turn ${i}): ${formatted.slice(0, 500)}`);
+                            }
+                            this.brain.injectFeedback(`EXTRACT RESULT (${matches.length} matches for query "${action.query}"${action.contains ? ` containing "${action.contains}"` : ''}):\n${formatted}`);
+                        }
+                        catch (extractErr) {
+                            const msg = extractErr instanceof Error ? extractErr.message : String(extractErr);
+                            this.brain.injectFeedback(`EXTRACT ERROR: ${msg}`);
+                        }
+                    }
+                    else {
+                        this.brain.injectFeedback('EXTRACT ERROR: Cannot access page — driver does not expose a Playwright page.');
+                    }
+                    turn.durationMs = Date.now() - turnStart;
+                    turns.push(turn);
+                    this.onTurn?.(turn);
+                    continue;
+                }
                 // -- 6. Check for terminal actions --
                 if (action.action === 'complete') {
                     // Step 1: Goal verification — did the agent actually achieve the goal?
@@ -1270,6 +1416,7 @@ export class BrowserAgent {
                     const persistentSearchEvidence = collectSearchWorkflowEvidence(scenario.goal, action.result || '', turns);
                     const verificationEvidence = [
                         ...runState.goalVerificationEvidence,
+                        ...runState.extractedEvidence,
                         ...persistentSearchEvidence,
                     ];
                     if (shouldVerifyGoal) {
@@ -1277,22 +1424,44 @@ export class BrowserAgent {
                         // evidence and had no recent errors. The detailed result text
                         // (>50 chars) combined with script-extracted evidence means the
                         // verifier almost always agrees — save the round-trip.
+                        //
+                        // Gen 12: content-aware gate. gpt-5.4 writes verbose narratives
+                        // that admit failure ("could not complete", "not visible", "did
+                        // not take effect") yet marks success. The old heuristic (length
+                        // + evidence + no errors) rubber-stamped these. Now we scan the
+                        // result text for self-contradicting phrases and force LLM
+                        // verification when found. This fixes the 6/8 judge disagreement
+                        // cases from Gen 11 evolve R2.
                         const agentResult = action.result || '';
                         const recentErrors = turns.slice(-2).filter(t => t.error).length;
                         const hasScriptEvidence = verificationEvidence.some(e => e.startsWith('SCRIPT RESULT:'));
+                        // Content-aware gate: detect when the agent's own text admits
+                        // failure despite claiming success. These phrases were found in
+                        // 6 of 8 false-pass cases on WebVoyager with gpt-5.4.
+                        const selfContradicting = /\b(?:could not (?:complete|find|fulfill|verify|confirm|locate|access|extract|retrieve)|not (?:visible|available|found|present|accessible|displayed|shown|confirmed|verified)|did not (?:take effect|work|succeed|load|return)|unable to (?:find|complete|verify|access|extract|retrieve)|no (?:visible (?:answer|result|data|content)|results? (?:found|returned|available))|(?:failed|failure) to (?:find|complete|set|select|navigate)|unfortunately|I (?:was|am) unable|(?:task|request|goal) (?:is|was) (?:not |in)complete)\b/i.test(agentResult);
                         const fastPathEligible = agentResult.length > 50 &&
                             recentErrors === 0 &&
-                            hasScriptEvidence;
+                            hasScriptEvidence &&
+                            !selfContradicting;
                         if (fastPathEligible) {
                             goalResult = {
                                 achieved: true,
                                 confidence: 0.9,
-                                evidence: ['Fast-path: agent provided detailed result with script-backed evidence and no recent errors.'],
+                                evidence: ['Fast-path: agent provided detailed result with script-backed evidence, no recent errors, and no self-contradicting language.'],
                                 missing: [],
                             };
                             if (this.config.debug) {
-                                console.log('[Runner] Goal verification fast-path: skipped LLM call (strong evidence + no errors)');
+                                console.log('[Runner] Goal verification fast-path: skipped LLM call (strong evidence + no errors + no self-contradiction)');
+                            }
+                        }
+                        else if (selfContradicting) {
+                            // Force LLM verification — the agent claims success but its
+                            // own text suggests failure. The LLM verifier reads the actual
+                            // content and makes the right call.
+                            if (this.config.debug) {
+                                console.log('[Runner] Gen 12: fast-path BLOCKED — agent result contains self-contradicting language, forcing LLM verification');
                             }
+                            goalResult = await this.brain.verifyGoalCompletion(state, scenario.goal, buildGoalVerificationClaim(agentResult, verificationEvidence));
                         }
                         else {
                             goalResult = await this.brain.verifyGoalCompletion(state, scenario.goal, buildGoalVerificationClaim(agentResult, verificationEvidence));
@@ -1378,10 +1547,18 @@ export class BrowserAgent {
                             runState.verificationRejectionCount++;
                             turn.verificationFailure = goalResult.missing.join('; ') || 'Goal verification failed';
                             runState.firstSufficientEvidenceTurn ??= i;
-                            // Goal not met — reject completion and feed back what's missing
-                            const escalation = runState.verificationRejectionCount >= 2
-                                ? ' Use runScript to extract exact data and include in completion.'
-                                : '';
+                            // Gen 19: progressive strategy-shift escalation on rejection.
+                            // Each rejection level suggests a MORE different approach.
+                            let escalation;
+                            if (runState.verificationRejectionCount >= 3) {
+                                escalation = ' STRATEGY SHIFT REQUIRED: Your previous approaches have failed 3 times. Try a COMPLETELY different method: use navigate to go to a different search engine or URL, try extractWithIndex instead of runScript, or scroll to look for the data in a different part of the page. Do NOT repeat what you just tried.';
+                            }
+                            else if (runState.verificationRejectionCount >= 2) {
+                                escalation = ' Use runScript or extractWithIndex to extract the exact data from the page and include ALL required values in your completion result.';
+                            }
+                            else {
+                                escalation = ' Re-read the GOAL carefully — your result is missing specific data the goal asked for. Find and include it before completing.';
+                            }
                             this.brain.injectFeedback(`REJECTED (${goalResult.confidence.toFixed(2)}). Missing: ${goalResult.missing.join('; ')}.${escalation}`);
                             turn.durationMs = Date.now() - turnStart;
                             turns.push(turn);
@@ -1555,6 +1732,14 @@ export class BrowserAgent {
                 else {
                     runState.clearConsecutiveErrors();
                     executeTimeoutRecoveries = 0; // Reset on successful action
+                    // Gen 27: surface form reset warnings from batch fill verification
+                    if ('warning' in execResult && typeof execResult.warning === 'string') {
+                        const warning = execResult.warning;
+                        this.brain.injectFeedback(warning);
+                        if (this.config.debug) {
+                            console.log(`[Runner] Fill warning: ${warning}`);
+                        }
+                    }
                     // Capture element bounding box for replay overlays
                     if (execResult.bounds) {
                         turn.actionBounds = execResult.bounds;
@@ -1814,6 +1999,12 @@ export class BrowserAgent {
         // runScript runs, so on extraction tasks it fabricates placeholders.
         // This deterministic substitution fixes that without an extra LLM call.
         let lastRunScriptOutput = null;
+        // Gen 10: track the last extractWithIndex match list. Unlike runScript,
+        // we do NOT auto-substitute this into a placeholder complete — the LLM
+        // must read the formatted match list and pick by index. When the plan
+        // ends with extractWithIndex (or runs out of valid steps), we fall
+        // through to the per-action loop with the match list as feedback.
+        let lastExtractOutput = null;
         for (let stepIdx = 0; stepIdx < plan.steps.length; stepIdx++) {
             if (scenario.signal?.aborted) {
                 return {
@@ -1998,6 +2189,22 @@ export class BrowserAgent {
             // This is the supply side of the placeholder-substitution fix above.
             if (step.action.action === 'runScript' && typeof execResult.data === 'string' && execResult.data.length > 0) {
                 lastRunScriptOutput = execResult.data;
+                if (execResult.data.length > 10) {
+                    runState.recordEvidence(`EXTRACTED (turn ${currentTurnIndex}): ${execResult.data.slice(0, 500)}`);
+                }
+            }
+            // Gen 10: capture extractWithIndex match list for fall-through to the
+            // per-action loop. The LLM must read the list and pick by index — we
+            // do not auto-complete with the raw match list.
+            if (step.action.action === 'extractWithIndex' && typeof execResult.data === 'string' && execResult.data.length > 0) {
+                lastExtractOutput = execResult.data;
+                // Also push as goal verification evidence so the verifier sees what
+                // the agent extracted.
+                runState.firstSufficientEvidenceTurn ??= currentTurnIndex;
+                pushGoalVerificationEvidence(runState.goalVerificationEvidence, `EXTRACT RESULT:\n${execResult.data}`);
+                if (execResult.data.length > 10) {
+                    runState.recordEvidence(`EXTRACTED (turn ${currentTurnIndex}): ${execResult.data.slice(0, 500)}`);
+                }
             }
             // Verify the post-condition. We re-observe to get the post-action
             // state, then run the same verifyExpectedEffect helper the per-action
@@ -2027,6 +2234,7 @@ export class BrowserAgent {
                 || step.action.action === 'scroll'
                 || step.action.action === 'hover'
                 || step.action.action === 'runScript'
+                || step.action.action === 'extractWithIndex'
                 || step.action.action === 'evaluate'
                 || step.action.action === 'verifyPreview'
                 || step.action.action === 'fill'
@@ -2141,7 +2349,7 @@ export class BrowserAgent {
         const lastStep = plan.steps[plan.steps.length - 1];
         if (lastStep
             && lastStep.action.action === 'runScript'
-            && lastRunScriptOutput) {
+            && isMeaningfulRunScriptOutput(lastRunScriptOutput)) {
             const synthTurnNumber = currentTurnIndex + 1;
             const synthTurn = {
                 turn: synthTurnNumber,
@@ -2173,6 +2381,44 @@ export class BrowserAgent {
                 turnsConsumed: plan.steps.length + 1,
             };
         }
+        // Gen 10: if the plan ended with extractWithIndex, fall through to the
+        // per-action loop with the match list as feedback. The LLM must read
+        // the matches and pick by index — we do NOT auto-complete with the raw
+        // match list. This is the planner-emits-extract path for extraction
+        // tasks like npm/mdn/python-docs where the planner used the new
+        // extractWithIndex action.
+        if (lastExtractOutput) {
+            return {
+                kind: 'deviated',
+                lastState,
+                failedStepIndex: plan.steps.length,
+                reason: `plan completed extractWithIndex but the LLM must read the matches and pick by index. Match list:\n${lastExtractOutput.slice(0, 4000)}\n\nPick the index whose text matches the goal, then emit complete with result: <picked text>`,
+                turnsConsumed: plan.steps.length,
+            };
+        }
+        // Gen 9 (cherry-picked into Gen 10): if the last step WAS a runScript
+        // but the output was NOT meaningful (null, empty, placeholder), DO NOT
+        // auto-complete with garbage. Fall through to the per-action loop with
+        // a deviation reason that names the empty output. In Gen 10 the per-
+        // action loop has TWO new tools that make this recovery actually work:
+        //   1. extractWithIndex (the wide-query content-match action) — see
+        //      data-extraction rule #25
+        //   2. cost cap (100k tokens) — bounds any death-spiral if the LLM
+        //      can't recover, preventing the Gen 9.1 reddit failure mode
+        if (lastStep
+            && lastStep.action.action === 'runScript'
+            && !isMeaningfulRunScriptOutput(lastRunScriptOutput)) {
+            if (this.config.debug) {
+                console.log(`[Runner] Gen 9: runScript returned no meaningful output (${JSON.stringify(lastRunScriptOutput).slice(0, 100)}); falling through to per-action loop for two-pass extraction`);
+            }
+            return {
+                kind: 'deviated',
+                lastState,
+                failedStepIndex: plan.steps.length - 1,
+                reason: `runScript returned no meaningful output (got: ${JSON.stringify(lastRunScriptOutput).slice(0, 200)}). The first-pass extraction failed — re-observe the page and try extractWithIndex with a wide query (e.g. 'p, span, dd, code') and a contains filter naming the expected text fragment. Pick-by-content beats pick-by-selector when the planner couldn't see the data at plan time.`,
+                turnsConsumed: plan.steps.length,
+            };
+        }
         // All steps verified BUT the plan ended without an explicit complete/abort.
         // This means the planner emitted a finite sequence of "work" steps and
         // didn't terminate. The right behavior is NOT to fabricate a complete —