@tangle-network/browser-agent-driver 0.17.0 → 0.19.0

package/dist/runner/runner.js

@@ -313,7 +313,169 @@ export class BrowserAgent {
             maxInterventions: this.config.supervisor?.maxInterventions ?? DEFAULT_SUPERVISOR.maxInterventions,
             hardStallWindow: this.config.supervisor?.hardStallWindow ?? DEFAULT_SUPERVISOR.hardStallWindow,
         };
-        for (let i = 1; i <= maxTurns; i++) {
+        // Gen 7 / 7.1: planner-first path. When `plannerEnabled: true` (and not
+        // disabled via BAD_PLANNER=0), make a single LLM call to generate a
+        // plan, then execute it deterministically.
+        //
+        // Gen 7.1 (replan-on-deviation): when a plan deviates, instead of
+        // immediately falling through to the per-action loop, call Brain.plan()
+        // AGAIN with the current page state and a deviation context. Cap at
+        // `maxReplans` total replan attempts (= initial plan + maxReplans
+        // additional plan calls). The system prompt is byte-stable so prompt
+        // cache still hits — only the user message carries the deviation
+        // history. On exhaustion, fall through to the per-action loop with a
+        // [REPLAN] hint, exactly like Gen 7 did.
+        //
+        // Plan execution writes to the same `turns` array, so post-run analysis
+        // sees a unified timeline regardless of which path completed the run.
+        let planFallbackContext = '';
+        let plannerStartTurn = 0;
+        const plannerEnabled = this.config.plannerEnabled === true && process.env.BAD_PLANNER !== '0';
+        const maxReplans = 3;
+        if (plannerEnabled && scenario.startUrl) {
+            // Need an initial observe so the planner has something to look at.
+            // The runner's main loop also observes on every iteration; this one
+            // primes the planner. The result is also stashed as cachedPostState
+            // so the per-action fallback's first observe is short-circuited.
+            const initialState = await this.driver.observe().catch(() => undefined);
+            if (initialState) {
+                this.cachedPostState = initialState;
+                let planLoopState = initialState;
+                let cumulativeTurnsConsumed = 0;
+                let attempt = 0;
+                let lastDeviationReason = '';
+                let lastFailedStepIndex = 0;
+                let lastTotalSteps = 0;
+                let replanLoopDone = false;
+                let replanLoopCompleted = false;
+                let lastFinalResult;
+                let lastCompletedState = initialState;
+                while (!replanLoopDone && attempt <= maxReplans) {
+                    // Re-observe before every replan (attempt > 0); the initial plan
+                    // already has the freshly-observed state above.
+                    if (attempt > 0) {
+                        const reobserved = await this.driver.observe().catch(() => planLoopState);
+                        planLoopState = reobserved;
+                        this.cachedPostState = reobserved;
+                        this.bus.emitNow({
+                            type: 'plan-replan-started',
+                            runId,
+                            turn: turns.length,
+                            replanIndex: attempt,
+                            maxReplans,
+                            reason: lastDeviationReason,
+                        });
+                        if (this.config.debug) {
+                            console.log(`[Runner] Replan attempt ${attempt}/${maxReplans}: ${lastDeviationReason}`);
+                        }
+                    }
+                    this.bus.emitNow({
+                        type: 'plan-started',
+                        runId,
+                        turn: turns.length,
+                        goal: scenario.goal,
+                    });
+                    const extraContext = attempt === 0
+                        ? undefined
+                        : `[REPLAN ${attempt}/${maxReplans}] The previous plan attempt failed at step ${lastFailedStepIndex + 1}/${lastTotalSteps}: ${lastDeviationReason}\nGenerate a FRESH plan from the current page state to complete the goal. Do NOT repeat the failed step verbatim — diagnose why it failed and route around it. Steps already executed in earlier attempts are persisted in the page state below; pick up from there.`;
+                    const planResult = await this.brain.plan(scenario.goal, planLoopState, {
+                        extraContext,
+                    }).catch((err) => ({
+                        plan: null,
+                        raw: '',
+                        durationMs: 0,
+                        parseError: err instanceof Error ? err.message : String(err),
+                    }));
+                    if (!planResult.plan || planResult.plan.steps.length === 0) {
+                        // Planner unavailable / parse failure / zero steps. Fall through.
+                        if (this.config.debug) {
+                            console.log(`[Runner] Planner unavailable on attempt ${attempt}: ${planResult.parseError ?? 'no plan returned'}`);
+                        }
+                        replanLoopDone = true;
+                        break;
+                    }
+                    this.bus.emitNow({
+                        type: 'plan-completed',
+                        runId,
+                        turn: turns.length,
+                        stepCount: planResult.plan.steps.length,
+                        plan: planResult.plan,
+                        durationMs: planResult.durationMs,
+                        ...(planResult.inputTokens !== undefined ? { inputTokens: planResult.inputTokens } : {}),
+                        ...(planResult.outputTokens !== undefined ? { outputTokens: planResult.outputTokens } : {}),
+                        ...(planResult.cacheReadInputTokens !== undefined ? { cacheReadInputTokens: planResult.cacheReadInputTokens } : {}),
+                    });
+                    if (this.config.debug) {
+                        console.log(`[Runner] Plan attempt ${attempt}: ${planResult.plan.steps.length} steps in ${planResult.durationMs}ms`);
+                    }
+                    const planResultRun = await this.executePlan(planResult.plan, scenario, runId, turns, runState, cumulativeTurnsConsumed, {
+                        tokensUsed: planResult.tokensUsed,
+                        inputTokens: planResult.inputTokens,
+                        outputTokens: planResult.outputTokens,
+                        cacheReadInputTokens: planResult.cacheReadInputTokens,
+                        cacheCreationInputTokens: planResult.cacheCreationInputTokens,
+                    });
+                    cumulativeTurnsConsumed += planResultRun.turnsConsumed;
+                    if (planResultRun.kind === 'completed') {
+                        replanLoopCompleted = true;
+                        lastFinalResult = planResultRun.finalResult;
+                        lastCompletedState = planResultRun.lastState;
+                        replanLoopDone = true;
+                        break;
+                    }
+                    // Deviated. Capture context, then either replan or fall through.
+                    lastDeviationReason = planResultRun.reason;
+                    lastFailedStepIndex = planResultRun.failedStepIndex;
+                    lastTotalSteps = planResult.plan.steps.length;
+                    planLoopState = planResultRun.lastState;
+                    attempt++;
+                    // Loop continues; if attempt > maxReplans the while-cond exits
+                    // and we fall through to the per-action loop below.
+                }
+                if (replanLoopCompleted) {
+                    // Plan finished without deviation. Synthesize a complete turn if
+                    // the plan didn't include one explicitly.
+                    const lastTurn = turns[turns.length - 1];
+                    if (!lastTurn || lastTurn.action.action !== 'complete') {
+                        const completeTurn = {
+                            turn: turns.length + 1,
+                            state: lastCompletedState,
+                            action: { action: 'complete', result: lastFinalResult ?? 'Plan executed successfully' },
+                            reasoning: 'Plan execution completed',
+                            durationMs: 0,
+                        };
+                        turns.push(completeTurn);
+                        this.onTurn?.(completeTurn);
+                    }
+                    return buildResult({
+                        success: true,
+                        result: lastFinalResult ?? 'Plan executed successfully',
+                        turns,
+                        totalMs: Date.now() - startTime,
+                    });
+                }
+                // All replan attempts (or initial plan) deviated. Fall through to
+                // the per-action loop with a [REPLAN] hint that names the final
+                // deviation. The per-action loop with Gen 6.1 batch detection will
+                // finish the work.
+                if (lastDeviationReason) {
+                    plannerStartTurn = cumulativeTurnsConsumed;
+                    planFallbackContext = `\n[REPLAN] After ${attempt} planner attempt${attempt === 1 ? '' : 's'} (1 initial + ${attempt - 1} replan${attempt === 2 ? '' : 's'}), the planner could not produce a working plan. Final deviation: ${lastDeviationReason}\nThe runner has fallen back to per-action mode. Continue toward the original goal from the current page state.\n`;
+                    this.bus.emitNow({
+                        type: 'plan-fallback-entered',
+                        runId,
+                        turn: turns.length,
+                        stepsCompleted: lastFailedStepIndex,
+                        totalSteps: lastTotalSteps,
+                        fallbackContext: planFallbackContext,
+                    });
+                    if (this.config.debug) {
+                        console.log(`[Runner] Falling back to per-action loop after ${attempt} planner attempts`);
+                    }
+                }
+            }
+        }
+        for (let i = 1 + plannerStartTurn; i <= maxTurns; i++) {
             if (scenario.signal?.aborted) {
                 return buildResult({
                     success: false,
@@ -764,7 +926,14 @@ export class BrowserAgent {
                 const aiTangleOutputContext = aiTangleOutputCompletion
                     ? `\nVERIFIED OUTPUT STATE DETECTED:\n${aiTangleOutputCompletion.feedback}\nReturn a terminal \`complete\` action now with concrete evidence.\n`
                     : '';
-                const finalExtraContext = [extraContext, aiTanglePartnerContext, aiTangleOutputContext].filter(Boolean).join('');
+                // Gen 7: include the plan fallback hint on the FIRST per-action turn
+                // after a plan deviation. The hint tells the LLM what failed and from
+                // what point to recover. We only inject it once (consume it after
+                // first use) so it doesn't pollute every subsequent turn.
+                const planFallbackHint = planFallbackContext;
+                if (planFallbackContext)
+                    planFallbackContext = '';
+                const finalExtraContext = [extraContext, aiTanglePartnerContext, aiTangleOutputContext, planFallbackHint].filter(Boolean).join('');
                 const decisionState = forceVision
                     ? await this.attachDecisionScreenshot(state)
                     : state;
@@ -1547,6 +1716,344 @@ export class BrowserAgent {
      * - "text should appear" -> check snapshot
      * - Generic text match -> check if text appears in snapshot
      */
+    /**
+     * Gen 7: execute a Plan deterministically without re-entering the LLM
+     * between steps. Each step:
+     *   1. Drives the action via driver.execute (existing path, gets bus events)
+     *   2. Verifies the post-condition via verifyExpectedEffect
+     *   3. On success → advance to the next step
+     *   4. On failure → bail and return the deviation context for the
+     *      caller to inject into the per-action fallback loop
+     *
+     * Returns a structured summary so the caller can decide whether to
+     * complete the run, fall back, or replan. Plan execution emits these
+     * events on the bus:
+     *   - plan-step-executed (per step, success or failure)
+     *   - plan-deviated (on first failure)
+     *
+     * Plan steps are wrapped in Turn artifacts and pushed onto the same
+     * `turns` array the per-action loop uses, so post-run analysis sees a
+     * unified timeline. The Turn's `reasoning` field carries the plan
+     * step's rationale, and `verified` carries the verification result.
+     */
+    async executePlan(plan, scenario, runId, turns, runState, startingTurnIndex, 
+    /**
+     * Token usage from the Brain.plan() LLM call that produced this plan.
+     * The plan call's tokens are NOT attached to any per-step turn — there's
+     * one plan call per N steps. To make the run-level cost tally honest,
+     * we attribute the plan call to the FIRST step's Turn artifact so the
+     * downstream sum (in baseline-summary.json / report.json) reflects the
+     * real LLM spend. This was the metric bug that caused Gen 7.1 runs to
+     * report $0 cost while Gen 7 baseline runs reported $0.50.
+     */
+    planCallTokens) {
+        let currentTurnIndex = startingTurnIndex;
+        let lastState = turns[turns.length - 1]?.state
+            ?? { url: '', title: '', snapshot: '' };
+        for (let stepIdx = 0; stepIdx < plan.steps.length; stepIdx++) {
+            if (scenario.signal?.aborted) {
+                return {
+                    kind: 'deviated',
+                    lastState,
+                    failedStepIndex: stepIdx,
+                    reason: scenario.signal.reason || 'Cancelled',
+                    turnsConsumed: stepIdx,
+                };
+            }
+            const step = plan.steps[stepIdx];
+            const stepStartedAt = Date.now();
+            const turnNumber = currentTurnIndex + 1;
+            currentTurnIndex++;
+            // Refresh the snapshot before EVERY step. The plan was built from a
+            // single observe() call at turn 1; later steps may target a different
+            // page entirely (after navigate / click "Next"). The first observe
+            // here is also what verify-against will eventually consume.
+            const preStepState = await this.driver.observe().catch(() => lastState);
+            lastState = preStepState;
+            // Wrap each plan step in a Turn artifact so post-run analysis (the
+            // viewer, the events.jsonl persistence, the metrics) sees a unified
+            // timeline regardless of whether the runner used the planner or the
+            // per-action loop.
+            //
+            // Token attribution: the FIRST step of each plan carries the
+            // Brain.plan() LLM call's token usage. Without this, runs that stay
+            // in plan-mode (Gen 7.1) report $0 cost while their Brain.plan()
+            // calls actually spent real tokens.
+            const isFirstStep = stepIdx === 0;
+            const turn = {
+                turn: turnNumber,
+                state: preStepState,
+                action: step.action,
+                reasoning: step.rationale ?? `Plan step ${stepIdx + 1}/${plan.steps.length}`,
+                expectedEffect: step.expectedEffect,
+                plan: plan.steps.map((s) => s.rationale ?? s.action.action),
+                currentStep: stepIdx,
+                durationMs: 0,
+                ...(isFirstStep && planCallTokens?.tokensUsed !== undefined ? { tokensUsed: planCallTokens.tokensUsed } : {}),
+                ...(isFirstStep && planCallTokens?.inputTokens !== undefined ? { inputTokens: planCallTokens.inputTokens } : {}),
+                ...(isFirstStep && planCallTokens?.outputTokens !== undefined ? { outputTokens: planCallTokens.outputTokens } : {}),
+                ...(isFirstStep && planCallTokens?.cacheReadInputTokens !== undefined ? { cacheReadInputTokens: planCallTokens.cacheReadInputTokens } : {}),
+                ...(isFirstStep && planCallTokens?.cacheCreationInputTokens !== undefined ? { cacheCreationInputTokens: planCallTokens.cacheCreationInputTokens } : {}),
+            };
+            // Terminal actions: complete and abort don't go through driver.execute
+            // — the runner handles them as the end of the plan.
+            if (step.action.action === 'complete') {
+                turn.durationMs = Date.now() - stepStartedAt;
+                turns.push(turn);
+                this.onTurn?.(turn);
+                this.bus.emitNow({
+                    type: 'plan-step-executed',
+                    runId,
+                    turn: turnNumber,
+                    stepIndex: stepIdx + 1,
+                    totalSteps: plan.steps.length,
+                    action: step.action,
+                    executeSuccess: true,
+                    verified: true,
+                    durationMs: turn.durationMs,
+                });
+                return {
+                    kind: 'completed',
+                    lastState,
+                    finalResult: step.action.result,
+                    turnsConsumed: stepIdx + 1,
+                };
+            }
+            if (step.action.action === 'abort') {
+                turn.durationMs = Date.now() - stepStartedAt;
+                turns.push(turn);
+                this.onTurn?.(turn);
+                this.bus.emitNow({
+                    type: 'plan-deviated',
+                    runId,
+                    turn: turnNumber,
+                    stepIndex: stepIdx + 1,
+                    totalSteps: plan.steps.length,
+                    reason: `plan aborted: ${step.action.reason}`,
+                });
+                return {
+                    kind: 'deviated',
+                    lastState,
+                    failedStepIndex: stepIdx,
+                    reason: `plan aborted: ${step.action.reason}`,
+                    turnsConsumed: stepIdx + 1,
+                };
+            }
+            // Execute the action via the existing driver path. This emits
+            // execute-started / execute-completed events on the bus exactly
+            // like the per-action loop does.
+            //
+            // CRITICAL: each plan step gets a 10s wall-clock cap (vs the driver's
+            // default 30s). Plan steps assume every selector was just observed in
+            // the snapshot at planning time — a missing element should fail
+            // FAST and trigger fallback to per-action mode, NOT block the run for
+            // 30s. Batch verbs already enforce a 5s per-field cap internally,
+            // but single-step type/click/press/select use the full 30s default.
+            this.bus.emitNow({ type: 'execute-started', runId, turn: turnNumber, action: step.action });
+            const execStartedAt = Date.now();
+            const planStepTimeoutMs = 10_000;
+            let execResult;
+            try {
+                execResult = await Promise.race([
+                    this.driver.execute(step.action),
+                    new Promise((resolve) => setTimeout(() => resolve({ success: false, error: `plan step wall-clock timeout after ${planStepTimeoutMs}ms` }), planStepTimeoutMs)),
+                ]);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                execResult = { success: false, error: message };
+            }
+            const execDurationMs = Date.now() - execStartedAt;
+            this.bus.emitNow({
+                type: 'execute-completed',
+                runId,
+                turn: turnNumber,
+                action: step.action,
+                success: execResult.success,
+                ...(execResult.error ? { error: execResult.error } : {}),
+                ...(execResult.bounds ? { bounds: execResult.bounds } : {}),
+                durationMs: execDurationMs,
+            });
+            if (!execResult.success) {
+                turn.error = execResult.error;
+                turn.durationMs = Date.now() - stepStartedAt;
+                turn.verified = false;
+                turn.verificationFailure = `execute failed: ${execResult.error}`;
+                turns.push(turn);
+                this.onTurn?.(turn);
+                runState.recordError();
+                this.bus.emitNow({
+                    type: 'plan-step-executed',
+                    runId,
+                    turn: turnNumber,
+                    stepIndex: stepIdx + 1,
+                    totalSteps: plan.steps.length,
+                    action: step.action,
+                    executeSuccess: false,
+                    verified: false,
+                    durationMs: Date.now() - stepStartedAt,
+                    ...(execResult.error ? { verifyReason: execResult.error } : {}),
+                });
+                this.bus.emitNow({
+                    type: 'plan-deviated',
+                    runId,
+                    turn: turnNumber,
+                    stepIndex: stepIdx + 1,
+                    totalSteps: plan.steps.length,
+                    reason: `execute failed: ${execResult.error}`,
+                });
+                return {
+                    kind: 'deviated',
+                    lastState,
+                    failedStepIndex: stepIdx,
+                    reason: `execute failed at step ${stepIdx + 1}: ${execResult.error}`,
+                    turnsConsumed: stepIdx + 1,
+                };
+            }
+            runState.clearConsecutiveErrors();
+            if (execResult.bounds)
+                turn.actionBounds = execResult.bounds;
+            // Verify the post-condition. We re-observe to get the post-action
+            // state, then run the same verifyExpectedEffect helper the per-action
+            // loop uses. The fresh observe is also stashed in cachedPostState so
+            // the next step's pre-step observe is short-circuited (Gen 4 lazy
+            // observe optimization).
+            this.bus.emitNow({
+                type: 'verify-started',
+                runId,
+                turn: turnNumber,
+                expectedEffect: step.expectedEffect,
+            });
+            const verifyStartedAt = Date.now();
+            // Auto-pass list — these actions either don't observably mutate
+            // the page state OR they're self-verifying (the underlying Playwright
+            // call throws on real failure, so a successful return means the
+            // mutation actually happened). Strict expectedEffect verification
+            // would generate false negatives on the per-action loop fallback for
+            // these. Plan execution trusts the execute result.
+            //
+            // - wait / scroll / hover: don't mutate observable snapshot state
+            // - runScript / evaluate / verifyPreview: meta actions
+            // - fill / clickSequence: self-verifying (Playwright throws on miss),
+            //   AND input values don't always reflect in the ARIA snapshot, so
+            //   the permissive "did state change?" check would also miss them
+            const isAutoPass = step.action.action === 'wait'
+                || step.action.action === 'scroll'
+                || step.action.action === 'hover'
+                || step.action.action === 'runScript'
+                || step.action.action === 'evaluate'
+                || step.action.action === 'verifyPreview'
+                || step.action.action === 'fill'
+                || step.action.action === 'clickSequence';
+            // Settle wait for mutating actions, mirroring verifyEffect's logic
+            const needsSettleWait = step.action.action === 'click'
+                || step.action.action === 'navigate'
+                || step.action.action === 'press'
+                || step.action.action === 'select'
+                || step.action.action === 'fill'
+                || step.action.action === 'clickSequence';
+            const observePromise = this.driver.observe().catch(() => preStepState);
+            if (needsSettleWait) {
+                await Promise.all([
+                    observePromise,
+                    new Promise((r) => setTimeout(r, 50)),
+                ]);
+            }
+            const postStepState = await observePromise;
+            this.cachedPostState = postStepState;
+            lastState = postStepState;
+            // Plan verification is more permissive than per-action verification:
+            // a step passes if (a) it's a non-mutating action, OR (b) the strict
+            // verifier passes, OR (c) the snapshot/url changed in any meaningful
+            // way (the action did SOMETHING). Strict failure-on-no-change is
+            // appropriate for the per-action loop where the agent can recover,
+            // but plan execution needs to push forward unless there's positive
+            // evidence of failure.
+            let verifyResult;
+            if (isAutoPass) {
+                verifyResult = { verified: true };
+            }
+            else {
+                const strictResult = verifyExpectedEffect({
+                    expectedEffect: step.expectedEffect,
+                    preActionState: preStepState,
+                    postActionState: postStepState,
+                });
+                if (strictResult.verified) {
+                    verifyResult = strictResult;
+                }
+                else {
+                    // Permissive fallback: did the page change at all?
+                    const stateChanged = preStepState.url !== postStepState.url
+                        || preStepState.title !== postStepState.title
+                        || preStepState.snapshot !== postStepState.snapshot;
+                    if (stateChanged) {
+                        verifyResult = { verified: true };
+                    }
+                    else {
+                        verifyResult = strictResult;
+                    }
+                }
+            }
+            turn.verified = verifyResult.verified;
+            if (!verifyResult.verified) {
+                turn.verificationFailure = verifyResult.reason;
+            }
+            turn.durationMs = Date.now() - stepStartedAt;
+            turns.push(turn);
+            this.onTurn?.(turn);
+            this.bus.emitNow({
+                type: 'verify-completed',
+                runId,
+                turn: turnNumber,
+                verified: verifyResult.verified,
+                ...(verifyResult.reason ? { reason: verifyResult.reason } : {}),
+                durationMs: Date.now() - verifyStartedAt,
+            });
+            this.bus.emitNow({
+                type: 'plan-step-executed',
+                runId,
+                turn: turnNumber,
+                stepIndex: stepIdx + 1,
+                totalSteps: plan.steps.length,
+                action: step.action,
+                executeSuccess: true,
+                verified: verifyResult.verified,
+                durationMs: Date.now() - stepStartedAt,
+                ...(verifyResult.reason ? { verifyReason: verifyResult.reason } : {}),
+            });
+            if (!verifyResult.verified) {
+                this.bus.emitNow({
+                    type: 'plan-deviated',
+                    runId,
+                    turn: turnNumber,
+                    stepIndex: stepIdx + 1,
+                    totalSteps: plan.steps.length,
+                    reason: `verification failed at step ${stepIdx + 1}: ${verifyResult.reason ?? 'expected effect not observed'}`,
+                });
+                return {
+                    kind: 'deviated',
+                    lastState,
+                    failedStepIndex: stepIdx,
+                    reason: `verification failed at step ${stepIdx + 1}: ${verifyResult.reason ?? 'expected effect not observed'}`,
+                    turnsConsumed: stepIdx + 1,
+                };
+            }
+        }
+        // All steps verified BUT the plan ended without an explicit complete/abort.
+        // This means the planner emitted a finite sequence of "work" steps and
+        // didn't terminate. The right behavior is NOT to fabricate a complete —
+        // we treat plan exhaustion as a deviation that triggers fallback to the
+        // per-action loop. The per-action loop will continue from the current
+        // state and emit `complete` when the goal is genuinely met.
+        return {
+            kind: 'deviated',
+            lastState,
+            failedStepIndex: plan.steps.length,
+            reason: 'plan exhausted without an explicit complete or abort step — falling through to per-action loop to finish the task',
+            turnsConsumed: plan.steps.length,
+        };
+    }
     async verifyEffect(expectedEffect, preActionState, actionType) {
         // Only pause for actions that mutate the page in flight (navigation,
         // clicks that may trigger XHR/route transitions, form submits). For