@tangle-network/browser-agent-driver 0.32.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artifacts/filesystem-sink.d.ts +1 -1
- package/dist/artifacts/filesystem-sink.js +1 -1
- package/dist/brain/index.d.ts +7 -12
- package/dist/brain/index.d.ts.map +1 -1
- package/dist/brain/index.js +42 -67
- package/dist/brain/index.js.map +1 -1
- package/dist/browser-launch.js +1 -3
- package/dist/browser-launch.js.map +1 -1
- package/dist/captcha.d.ts.map +1 -1
- package/dist/captcha.js +1 -3
- package/dist/captcha.js.map +1 -1
- package/dist/cli-attach.d.ts +2 -2
- package/dist/cli-attach.js +2 -2
- package/dist/cli-design-audit.js +6 -6
- package/dist/cli-design-audit.js.map +1 -1
- package/dist/cli.js +26 -28
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +5 -3
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/design/audit/build-result.d.ts +3 -0
- package/dist/design/audit/build-result.d.ts.map +1 -1
- package/dist/design/audit/build-result.js +10 -2
- package/dist/design/audit/build-result.js.map +1 -1
- package/dist/design/audit/evaluate.d.ts +5 -2
- package/dist/design/audit/evaluate.d.ts.map +1 -1
- package/dist/design/audit/evaluate.js +9 -3
- package/dist/design/audit/evaluate.js.map +1 -1
- package/dist/design/audit/patches/generate.d.ts +12 -0
- package/dist/design/audit/patches/generate.d.ts.map +1 -1
- package/dist/design/audit/patches/generate.js +16 -9
- package/dist/design/audit/patches/generate.js.map +1 -1
- package/dist/design/audit/pipeline.d.ts +3 -3
- package/dist/design/audit/pipeline.d.ts.map +1 -1
- package/dist/design/audit/pipeline.js +4 -3
- package/dist/design/audit/pipeline.js.map +1 -1
- package/dist/drivers/cursor-overlay.d.ts +1 -1
- package/dist/drivers/cursor-overlay.d.ts.map +1 -1
- package/dist/drivers/cursor-overlay.js +2 -2
- package/dist/drivers/extract-with-index.d.ts +4 -7
- package/dist/drivers/extract-with-index.d.ts.map +1 -1
- package/dist/drivers/extract-with-index.js +4 -7
- package/dist/drivers/extract-with-index.js.map +1 -1
- package/dist/drivers/overlay-label.d.ts +3 -4
- package/dist/drivers/overlay-label.d.ts.map +1 -1
- package/dist/drivers/overlay-label.js.map +1 -1
- package/dist/drivers/playwright.d.ts +2 -2
- package/dist/drivers/playwright.d.ts.map +1 -1
- package/dist/drivers/playwright.js +10 -24
- package/dist/drivers/playwright.js.map +1 -1
- package/dist/drivers/som-overlay.d.ts +2 -3
- package/dist/drivers/som-overlay.d.ts.map +1 -1
- package/dist/drivers/som-overlay.js +2 -3
- package/dist/drivers/som-overlay.js.map +1 -1
- package/dist/drivers/types.d.ts +3 -9
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/multi-actor.d.ts.map +1 -1
- package/dist/multi-actor.js +2 -3
- package/dist/multi-actor.js.map +1 -1
- package/dist/run-state.d.ts +3 -32
- package/dist/run-state.d.ts.map +1 -1
- package/dist/run-state.js +5 -39
- package/dist/run-state.js.map +1 -1
- package/dist/runner/completion-language.d.ts +2 -0
- package/dist/runner/completion-language.d.ts.map +1 -0
- package/dist/runner/completion-language.js +5 -0
- package/dist/runner/completion-language.js.map +1 -0
- package/dist/runner/direct-start.d.ts +19 -0
- package/dist/runner/direct-start.d.ts.map +1 -0
- package/dist/runner/direct-start.js +295 -0
- package/dist/runner/direct-start.js.map +1 -0
- package/dist/runner/events.d.ts +3 -6
- package/dist/runner/events.d.ts.map +1 -1
- package/dist/runner/events.js.map +1 -1
- package/dist/runner/fan-out.d.ts +6 -9
- package/dist/runner/fan-out.d.ts.map +1 -1
- package/dist/runner/fan-out.js +7 -15
- package/dist/runner/fan-out.js.map +1 -1
- package/dist/runner/goal-decomposer.d.ts +0 -1
- package/dist/runner/goal-decomposer.d.ts.map +1 -1
- package/dist/runner/goal-decomposer.js +0 -1
- package/dist/runner/goal-decomposer.js.map +1 -1
- package/dist/runner/google-flights-preflight.d.ts +35 -0
- package/dist/runner/google-flights-preflight.d.ts.map +1 -0
- package/dist/runner/google-flights-preflight.js +411 -0
- package/dist/runner/google-flights-preflight.js.map +1 -0
- package/dist/runner/interrupt-controller.d.ts +1 -1
- package/dist/runner/interrupt-controller.js +1 -1
- package/dist/runner/overlay-narration.d.ts +1 -1
- package/dist/runner/overlay-narration.js +1 -1
- package/dist/runner/parallel-runner.d.ts +3 -6
- package/dist/runner/parallel-runner.d.ts.map +1 -1
- package/dist/runner/parallel-runner.js +2 -3
- package/dist/runner/parallel-runner.js.map +1 -1
- package/dist/runner/pattern-extractor.d.ts +2 -2
- package/dist/runner/pattern-extractor.js +2 -2
- package/dist/runner/runner.d.ts +15 -39
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +208 -255
- package/dist/runner/runner.js.map +1 -1
- package/dist/runner/stream-webhook.d.ts +1 -1
- package/dist/runner/stream-webhook.js +1 -1
- package/dist/skills/domain-loader.d.ts +1 -2
- package/dist/skills/domain-loader.d.ts.map +1 -1
- package/dist/skills/domain-loader.js +1 -2
- package/dist/skills/domain-loader.js.map +1 -1
- package/dist/telemetry/schema.d.ts +2 -2
- package/dist/telemetry/schema.js +2 -2
- package/dist/test-runner.d.ts +4 -5
- package/dist/test-runner.d.ts.map +1 -1
- package/dist/test-runner.js +2 -3
- package/dist/test-runner.js.map +1 -1
- package/dist/types.d.ts +21 -21
- package/dist/types.d.ts.map +1 -1
- package/package.json +5 -2
package/dist/runner/runner.js
CHANGED
|
@@ -37,9 +37,12 @@ import { DecisionCache } from './decision-cache.js';
|
|
|
37
37
|
import { VerdictTracker, extractCurrentMarker, buildProgressLabel, } from './overlay-narration.js';
|
|
38
38
|
import { applyDemoOverride } from './demo-overrides.js';
|
|
39
39
|
import { matchDeterministicPattern } from './deterministic-patterns.js';
|
|
40
|
+
import { deriveDirectStartUrl, shouldAcceptRolledBookingCompletion } from './direct-start.js';
|
|
41
|
+
import { prepareGoogleFlightsSearch, shouldAcceptRolledGoogleFlightsCompletion } from './google-flights-preflight.js';
|
|
42
|
+
import { containsSelfContradictingCompletion } from './completion-language.js';
|
|
40
43
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
44
|
+
* Detect when the agent is filling a multi-field form one input at a time and
|
|
45
|
+
* inject a hint that demands a `fill` batch on the next turn.
|
|
43
46
|
*
|
|
44
47
|
* Trigger conditions (all must hold):
|
|
45
48
|
* 1. The agent's most recent action was a single-step `type` on the
|
|
@@ -49,12 +52,9 @@ import { matchDeterministicPattern } from './deterministic-patterns.js';
|
|
|
49
52
|
* 3. We haven't already injected this hint in the last turn (to avoid
|
|
50
53
|
* hint loops if the agent ignores it)
|
|
51
54
|
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
* on a typical 2-field-per-step form. Firing on the FIRST type action
|
|
56
|
-
* when the form clearly has more fields catches every multi-field form
|
|
57
|
-
* the moment the agent starts on it.
|
|
55
|
+
* The detector fires after one type action when two or more unused fields
|
|
56
|
+
* remain, which catches common two-field-per-step forms before the agent
|
|
57
|
+
* burns extra turns.
|
|
58
58
|
*
|
|
59
59
|
* The hint is high-priority (100) so it survives ctxBudget truncation, and
|
|
60
60
|
* it explicitly lists the unused @refs from the current snapshot so the LLM
|
|
@@ -133,8 +133,7 @@ const DEFAULT_MAX_TURNS = 20;
|
|
|
133
133
|
const DEFAULT_RETRIES = 3;
|
|
134
134
|
const DEFAULT_RETRY_DELAY_MS = 1000;
|
|
135
135
|
const DEFAULT_MICRO_PLAN_ACTIONS = 2;
|
|
136
|
-
//
|
|
137
|
-
// Gen 23: clickLabel/typeLabel for SoM-based actions
|
|
136
|
+
// Safe action verbs for micro-plans emitted by the model.
|
|
138
137
|
const SAFE_MICRO_ACTIONS = new Set(['click', 'type', 'press', 'hover', 'select', 'scroll', 'wait', 'clickAt', 'typeAt', 'clickLabel', 'typeLabel']);
|
|
139
138
|
const DEFAULT_SUPERVISOR = {
|
|
140
139
|
enabled: true,
|
|
@@ -154,7 +153,7 @@ const DEFI_BRAIN_CONTEXT = '\nWALLET/DeFi MODE ACTIVE — crypto app patterns:\n
|
|
|
154
153
|
'- NETWORK SELECTOR: Do NOT change the network/chain. If a network dropdown opens accidentally, close it immediately.\n' +
|
|
155
154
|
'- COOKIE BANNERS: Dismiss immediately via Escape or Reject button — don\'t spend multiple turns on consent dialogs.\n';
|
|
156
155
|
/**
|
|
157
|
-
*
|
|
156
|
+
* Detect placeholder patterns in a planner-generated complete.result.
|
|
158
157
|
*
|
|
159
158
|
* The planner has to commit to its `complete.result` text BEFORE any prior
|
|
160
159
|
* runScript step actually runs, so on extraction tasks it fabricates
|
|
@@ -188,26 +187,7 @@ export function hasPlaceholderPattern(text) {
|
|
|
188
187
|
}
|
|
189
188
|
return false;
|
|
190
189
|
}
|
|
191
|
-
/**
|
|
192
|
-
* Gen 9 — runtime two-pass extraction. When the planner emits a single
|
|
193
|
-
* runScript step (per Gen 7.2 rule #7) and that script returns null /
|
|
194
|
-
* empty / whitespace / `{x: null}` / a placeholder pattern, the auto-
|
|
195
|
-
* complete-from-runScript path should NOT fire. Instead the runner should
|
|
196
|
-
* mark the plan as deviated and fall through to the per-action loop where
|
|
197
|
-
* Brain.decide can re-observe the loaded page and emit a smarter action
|
|
198
|
-
* (different selector, click+wait, scroll, etc.).
|
|
199
|
-
*
|
|
200
|
-
* This addresses the failure mode the Gen 8 head-to-head gauntlet
|
|
201
|
-
* surfaced: bad's planner-only path lost to browser-use's per-action loop
|
|
202
|
-
* on tasks where the first runScript pick was wrong (npm, mdn signature,
|
|
203
|
-
* w3c, github, wikipedia variance). Two-pass gives bad's per-action loop
|
|
204
|
-
* the same recovery surface browser-use uses, with the planner's speed
|
|
205
|
-
* advantage on the cases where runScript succeeds first try.
|
|
206
|
-
*
|
|
207
|
-
* "Meaningful" means: not empty/whitespace, not the literal string `null`
|
|
208
|
-
* or `undefined`, and not matching `hasPlaceholderPattern` (which already
|
|
209
|
-
* detects JSON null fields, "<from prior step>" markers, etc.).
|
|
210
|
-
*/
|
|
190
|
+
/** Returns true only when runScript output contains usable extracted data. */
|
|
211
191
|
export function isMeaningfulRunScriptOutput(output) {
|
|
212
192
|
if (typeof output !== 'string')
|
|
213
193
|
return false;
|
|
@@ -242,6 +222,25 @@ export function isMeaningfulRunScriptOutput(output) {
|
|
|
242
222
|
}
|
|
243
223
|
return true;
|
|
244
224
|
}
|
|
225
|
+
export function shouldUsePlannerForScenario(scenario, mode = 'always') {
|
|
226
|
+
if (mode !== 'auto')
|
|
227
|
+
return true;
|
|
228
|
+
const tags = new Set((scenario.tags ?? []).map((tag) => tag.toLowerCase()));
|
|
229
|
+
if (tags.has('extraction'))
|
|
230
|
+
return false;
|
|
231
|
+
const goal = scenario.goal.toLowerCase();
|
|
232
|
+
if (/\breturn\s+only\s+(?:a\s+)?json\b/.test(goal) ||
|
|
233
|
+
/\bvalid\s+json\s+object\b/.test(goal) ||
|
|
234
|
+
/\bexactly\s+these?\s+keys?\b/.test(goal)) {
|
|
235
|
+
return false;
|
|
236
|
+
}
|
|
237
|
+
if (/\b(?:find|extract|look up|lookup|read|identify)\b/.test(goal) &&
|
|
238
|
+
/\b(?:return|answer|provide)\b/.test(goal) &&
|
|
239
|
+
/\b(?:json|number|year|date|price|downloads?|count|signature|metric|value)\b/.test(goal)) {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
return true;
|
|
243
|
+
}
|
|
245
244
|
export class BrowserAgent {
|
|
246
245
|
driver;
|
|
247
246
|
brain;
|
|
@@ -263,7 +262,7 @@ export class BrowserAgent {
|
|
|
263
262
|
// persists, never crosses runs.
|
|
264
263
|
decisionCache;
|
|
265
264
|
extensions;
|
|
266
|
-
/**
|
|
265
|
+
/** Cached so compound-goal sub-tabs inherit the macro catalog. */
|
|
267
266
|
macroPromptBlock;
|
|
268
267
|
constructor(options) {
|
|
269
268
|
this.driver = options.driver;
|
|
@@ -292,8 +291,7 @@ export class BrowserAgent {
|
|
|
292
291
|
this.runRegistry = options.runRegistry;
|
|
293
292
|
}
|
|
294
293
|
async run(scenario) {
|
|
295
|
-
//
|
|
296
|
-
// Pre-flight: check if the goal should be decomposed into parallel sub-goals.
|
|
294
|
+
// Pre-flight compound goals into parallel sub-goals when enabled.
|
|
297
295
|
if (this.config.parallelTabs?.enabled && scenario.goal && scenario.startUrl) {
|
|
298
296
|
const context = this.driver.getPage?.()?.context();
|
|
299
297
|
if (context) {
|
|
@@ -307,8 +305,7 @@ export class BrowserAgent {
|
|
|
307
305
|
const { runParallel } = await import('./parallel-runner.js');
|
|
308
306
|
// Inherit the top-level macro catalog + driver's macro registry so
|
|
309
307
|
// sub-tab agents see the same capability surface. Without this the
|
|
310
|
-
// sub-agents emit macro actions
|
|
311
|
-
// into a driver that rejects them — a silent failure mode.
|
|
308
|
+
// sub-agents emit macro actions into a driver that rejects them.
|
|
312
309
|
const topDriverOptions = this.driver.getDriverOptions?.();
|
|
313
310
|
const result = await runParallel({
|
|
314
311
|
context,
|
|
@@ -330,22 +327,12 @@ export class BrowserAgent {
|
|
|
330
327
|
}
|
|
331
328
|
}
|
|
332
329
|
}
|
|
333
|
-
//
|
|
334
|
-
//
|
|
335
|
-
// runs out of turns before completing multi-step tasks.
|
|
330
|
+
// Vision and hybrid modes get more turns because screenshot turns are
|
|
331
|
+
// slower and multi-step tasks need more wall-clock budget.
|
|
336
332
|
const isVisionMode = this.config.observationMode === 'vision' || this.config.observationMode === 'hybrid';
|
|
337
333
|
const baseMaxTurns = scenario.maxTurns || DEFAULT_MAX_TURNS;
|
|
338
|
-
//
|
|
339
|
-
//
|
|
340
|
-
//
|
|
341
|
-
// 2026-04-28 update: `maxTurns` is now `let`, not `const`, because the
|
|
342
|
-
// adaptive-extension logic at the top of each iteration may bump it
|
|
343
|
-
// by EXTENSION_TURNS_GRANTED when the agent shows recent progress at
|
|
344
|
-
// the cap. WebVoyager-590 baseline showed 21/54 fails were
|
|
345
|
-
// "agent_gave_up_at_max_turns" mid-flow; the extension converts those
|
|
346
|
-
// near-misses into successes without rewarding stuck loops (extension
|
|
347
|
-
// requires URL-or-DOM progress in the prior 3 turns; capped absolute
|
|
348
|
-
// at EXTENSION_HARD_CAP).
|
|
334
|
+
// maxTurns is mutable because active runs can receive a one-time extension
|
|
335
|
+
// when they reach the configured cap while still making page progress.
|
|
349
336
|
let maxTurns = isVisionMode ? Math.max(baseMaxTurns, 30) : baseMaxTurns;
|
|
350
337
|
let extensionGranted = false;
|
|
351
338
|
const EXTENSION_TURNS_GRANTED = 5;
|
|
@@ -356,11 +343,13 @@ export class BrowserAgent {
|
|
|
356
343
|
const turns = [];
|
|
357
344
|
const startTime = Date.now();
|
|
358
345
|
const phaseTimings = {};
|
|
359
|
-
//
|
|
360
|
-
//
|
|
361
|
-
// but hitting 200k cap). The timeout (600s) is the real safety net.
|
|
346
|
+
// Vision modes get a larger token budget; the wall-clock timeout remains
|
|
347
|
+
// the outer safety bound.
|
|
362
348
|
const visionBudgetMultiplier = isVisionMode ? 3 : 1;
|
|
363
349
|
const runState = new RunState(maxTurns, Math.round(DEFAULT_TOKEN_BUDGET * visionBudgetMultiplier));
|
|
350
|
+
const directStart = deriveDirectStartUrl(scenario);
|
|
351
|
+
let googleFlightsPreflight;
|
|
352
|
+
const initialStartUrl = directStart?.url || scenario.startUrl;
|
|
364
353
|
const runId = scenario.sessionId
|
|
365
354
|
? `${scenario.sessionId}_${Date.now()}`
|
|
366
355
|
: RunRegistry.generateRunId();
|
|
@@ -408,7 +397,7 @@ export class BrowserAgent {
|
|
|
408
397
|
return agentResult;
|
|
409
398
|
};
|
|
410
399
|
// Wrap onTurn to include mid-run manifest updates (every 3 turns) and to
|
|
411
|
-
// accumulate per-turn token usage for the
|
|
400
|
+
// accumulate per-turn token usage for the cost cap.
|
|
412
401
|
const originalOnTurn = this.onTurn;
|
|
413
402
|
this.onTurn = (turn) => {
|
|
414
403
|
originalOnTurn?.(turn);
|
|
@@ -445,20 +434,47 @@ export class BrowserAgent {
|
|
|
445
434
|
// Start navigation and load memory in parallel. Navigation is async (network
|
|
446
435
|
// I/O) while memory init is sync (readFileSync), so memory completes while
|
|
447
436
|
// the network request is in flight — saving the serial cost of disk reads.
|
|
448
|
-
if (
|
|
437
|
+
if (initialStartUrl) {
|
|
449
438
|
const navigateStartedAt = Date.now();
|
|
450
|
-
const navPromise = withRetry(() => this.driver.execute({ action: 'navigate', url:
|
|
439
|
+
const navPromise = withRetry(() => this.driver.execute({ action: 'navigate', url: initialStartUrl }), retries, retryDelayMs, undefined, scenario.signal);
|
|
451
440
|
// Load domain-scoped memory while navigation is in progress
|
|
452
441
|
if (this.projectStore) {
|
|
453
|
-
|
|
454
|
-
this.
|
|
442
|
+
const memoryStartUrl = scenario.startUrl || initialStartUrl;
|
|
443
|
+
this.knowledge = new AppKnowledge(this.projectStore.getKnowledgePath(memoryStartUrl), memoryStartUrl);
|
|
444
|
+
this.selectorCache = new SelectorCache(this.projectStore.getSelectorCachePath(memoryStartUrl));
|
|
455
445
|
}
|
|
456
446
|
await navPromise;
|
|
457
447
|
phaseTimings.initialNavigateMs = Date.now() - navigateStartedAt;
|
|
458
448
|
this.onPhaseTiming?.('navigate', phaseTimings.initialNavigateMs);
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
449
|
+
googleFlightsPreflight = await prepareGoogleFlightsSearch(this.driver.getPage?.(), scenario, { timeoutMs: 45_000 });
|
|
450
|
+
if (googleFlightsPreflight?.blockingReason) {
|
|
451
|
+
const state = await this.driver.observe().catch(() => ({
|
|
452
|
+
url: this.driver.getUrl?.() || googleFlightsPreflight?.finalUrl || initialStartUrl,
|
|
453
|
+
title: 'Google Flights',
|
|
454
|
+
snapshot: googleFlightsPreflight?.blockingReason || 'Google Flights preflight blocked the requested search.',
|
|
455
|
+
}));
|
|
456
|
+
const reason = googleFlightsPreflight.blockingReason;
|
|
457
|
+
turns.push({
|
|
458
|
+
turn: 1,
|
|
459
|
+
state,
|
|
460
|
+
action: { action: 'abort', reason },
|
|
461
|
+
reasoning: 'Google Flights preflight detected a first-party blocking state before LLM control.',
|
|
462
|
+
expectedEffect: 'Run exits without substituting adjacent dates for an unavailable requested date.',
|
|
463
|
+
durationMs: Date.now() - navigateStartedAt,
|
|
464
|
+
});
|
|
465
|
+
return buildResult({
|
|
466
|
+
success: false,
|
|
467
|
+
reason,
|
|
468
|
+
turns,
|
|
469
|
+
totalMs: Date.now() - startTime,
|
|
470
|
+
goalVerification: {
|
|
471
|
+
achieved: false,
|
|
472
|
+
confidence: 0.95,
|
|
473
|
+
evidence: [reason, `URL: ${state.url}`],
|
|
474
|
+
missing: ['The exact requested Google Flights date could not be searched in the live site state.'],
|
|
475
|
+
},
|
|
476
|
+
});
|
|
477
|
+
}
|
|
462
478
|
}
|
|
463
479
|
// Don't wait on warmup before entering the loop — it races against the
|
|
464
480
|
// first observe and decode. Make sure any unhandled rejection is silenced.
|
|
@@ -474,7 +490,7 @@ export class BrowserAgent {
|
|
|
474
490
|
});
|
|
475
491
|
const supervisorConfig = {
|
|
476
492
|
enabled: this.config.supervisor?.enabled ?? DEFAULT_SUPERVISOR.enabled,
|
|
477
|
-
//
|
|
493
|
+
// Per-role supervisor model overrides the supervisor default, then falls back to main.
|
|
478
494
|
model: this.config.models?.supervisor?.model || this.config.supervisor?.model || this.config.model || 'gpt-5.4',
|
|
479
495
|
provider: (this.config.models?.supervisor?.provider || this.config.supervisor?.provider || this.config.provider || 'openai'),
|
|
480
496
|
useVision: this.config.supervisor?.useVision ?? DEFAULT_SUPERVISOR.useVision,
|
|
@@ -483,24 +499,17 @@ export class BrowserAgent {
|
|
|
483
499
|
maxInterventions: this.config.supervisor?.maxInterventions ?? DEFAULT_SUPERVISOR.maxInterventions,
|
|
484
500
|
hardStallWindow: this.config.supervisor?.hardStallWindow ?? DEFAULT_SUPERVISOR.hardStallWindow,
|
|
485
501
|
};
|
|
486
|
-
//
|
|
487
|
-
//
|
|
488
|
-
//
|
|
489
|
-
//
|
|
490
|
-
// Gen 7.1 (replan-on-deviation): when a plan deviates, instead of
|
|
491
|
-
// immediately falling through to the per-action loop, call Brain.plan()
|
|
492
|
-
// AGAIN with the current page state and a deviation context. Cap at
|
|
493
|
-
// `maxReplans` total replan attempts (= initial plan + maxReplans
|
|
494
|
-
// additional plan calls). The system prompt is byte-stable so prompt
|
|
495
|
-
// cache still hits — only the user message carries the deviation
|
|
496
|
-
// history. On exhaustion, fall through to the per-action loop with a
|
|
497
|
-
// [REPLAN] hint, exactly like Gen 7 did.
|
|
502
|
+
// Planner-first path: make one LLM call to generate a plan, then execute
|
|
503
|
+
// it deterministically. On deviation, replan from the current page state
|
|
504
|
+
// with deviation context; after the retry budget is exhausted, fall back
|
|
505
|
+
// to the per-action loop with a [REPLAN] hint.
|
|
498
506
|
//
|
|
499
507
|
// Plan execution writes to the same `turns` array, so post-run analysis
|
|
500
508
|
// sees a unified timeline regardless of which path completed the run.
|
|
501
509
|
let planFallbackContext = '';
|
|
502
510
|
let plannerStartTurn = 0;
|
|
503
|
-
const plannerEnabled = this.config.plannerEnabled === true && process.env.BAD_PLANNER !== '0'
|
|
511
|
+
const plannerEnabled = this.config.plannerEnabled === true && process.env.BAD_PLANNER !== '0'
|
|
512
|
+
&& shouldUsePlannerForScenario(scenario, this.config.plannerMode ?? 'always');
|
|
504
513
|
const maxReplans = 3;
|
|
505
514
|
if (plannerEnabled && scenario.startUrl) {
|
|
506
515
|
// Need an initial observe so the planner has something to look at.
|
|
@@ -508,11 +517,7 @@ export class BrowserAgent {
|
|
|
508
517
|
// primes the planner. The result is also stashed as cachedPostState
|
|
509
518
|
// so the per-action fallback's first observe is short-circuited.
|
|
510
519
|
//
|
|
511
|
-
//
|
|
512
|
-
// the page to settle BEFORE the planner observes. SPA pages like
|
|
513
|
-
// npmjs.com load their data via JS after DOMContentLoaded — without
|
|
514
|
-
// a settle wait the planner snapshots a half-loaded page and emits
|
|
515
|
-
// runScript queries against selectors that don't exist yet.
|
|
520
|
+
// Give dynamic pages time to populate before the planner observes.
|
|
516
521
|
const settleMs = this.config.initialObserveSettleMs ?? 0;
|
|
517
522
|
if (settleMs > 0) {
|
|
518
523
|
const page = this.driver.getPage?.();
|
|
@@ -526,7 +531,7 @@ export class BrowserAgent {
|
|
|
526
531
|
await new Promise((resolve) => setTimeout(resolve, settleMs));
|
|
527
532
|
}
|
|
528
533
|
if (this.config.debug) {
|
|
529
|
-
console.log(`[Runner]
|
|
534
|
+
console.log(`[Runner] Initial settle: waited ${settleMs}ms (or networkidle) before planner observe`);
|
|
530
535
|
}
|
|
531
536
|
}
|
|
532
537
|
const initialState = await this.driver.observe().catch(() => undefined);
|
|
@@ -646,10 +651,8 @@ export class BrowserAgent {
|
|
|
646
651
|
totalMs: Date.now() - startTime,
|
|
647
652
|
});
|
|
648
653
|
}
|
|
649
|
-
// All replan attempts
|
|
650
|
-
//
|
|
651
|
-
// deviation. The per-action loop with Gen 6.1 batch detection will
|
|
652
|
-
// finish the work.
|
|
654
|
+
// All replan attempts deviated. Fall through to the per-action loop
|
|
655
|
+
// with a [REPLAN] hint that names the final deviation.
|
|
653
656
|
if (lastDeviationReason) {
|
|
654
657
|
plannerStartTurn = cumulativeTurnsConsumed;
|
|
655
658
|
planFallbackContext = `\n[REPLAN] After ${attempt} planner attempt${attempt === 1 ? '' : 's'} (1 initial + ${attempt - 1} replan${attempt === 2 ? '' : 's'}), the planner could not produce a working plan. Final deviation: ${lastDeviationReason}\nThe runner has fallen back to per-action mode. Continue toward the original goal from the current page state.\n`;
|
|
@@ -667,18 +670,11 @@ export class BrowserAgent {
|
|
|
667
670
|
}
|
|
668
671
|
}
|
|
669
672
|
}
|
|
670
|
-
//
|
|
671
|
-
// markers so a ledger the agent keeps re-emitting doesn't spam badges.
|
|
673
|
+
// Per-session overlay narration tracker; suppresses duplicate verdict badges.
|
|
672
674
|
const verdictTracker = new VerdictTracker();
|
|
673
675
|
for (let i = 1 + plannerStartTurn; i <= maxTurns; i++) {
|
|
674
|
-
//
|
|
675
|
-
//
|
|
676
|
-
// EXTENSION_PROGRESS_LOOKBACK turns, grant a one-time extension up
|
|
677
|
-
// to EXTENSION_HARD_CAP. Triggers only on the original maxTurns
|
|
678
|
-
// boundary (extensionGranted guards against cascading); the
|
|
679
|
-
// recovery-fired event is emitted so the trace is honest about the
|
|
680
|
-
// turns being borrowed. No-op for vision-mode (already gets +5
|
|
681
|
-
// baseline boost) and for explicitly-large maxTurns settings.
|
|
676
|
+
// Grant a one-time max-turns extension when the run reaches its cap
|
|
677
|
+
// while still making recent progress.
|
|
682
678
|
if (i === maxTurns
|
|
683
679
|
&& !extensionGranted
|
|
684
680
|
&& !isVisionMode
|
|
@@ -698,9 +694,7 @@ export class BrowserAgent {
|
|
|
698
694
|
extensionGranted = true;
|
|
699
695
|
}
|
|
700
696
|
}
|
|
701
|
-
//
|
|
702
|
-
// Blocks until `r` is pressed (resume) or `q` is pressed (abort).
|
|
703
|
-
// A rejected beforeTurn is treated as an abort.
|
|
697
|
+
// Honor user-driven pause or abort from the interrupt controller.
|
|
704
698
|
if (this.beforeTurn) {
|
|
705
699
|
try {
|
|
706
700
|
await this.beforeTurn(i);
|
|
@@ -722,10 +716,8 @@ export class BrowserAgent {
|
|
|
722
716
|
totalMs: Date.now() - startTime,
|
|
723
717
|
});
|
|
724
718
|
}
|
|
725
|
-
//
|
|
726
|
-
//
|
|
727
|
-
// failure mode where reddit hit $0.32 / 173K tokens). The cap is enforced
|
|
728
|
-
// BEFORE the next LLM call so the case aborts cleanly with a reason.
|
|
719
|
+
// Enforce the token budget before the next LLM call so stalled recovery
|
|
720
|
+
// loops abort cleanly with a reason.
|
|
729
721
|
if (runState.isTokenBudgetExhausted) {
|
|
730
722
|
return buildResult({
|
|
731
723
|
success: false,
|
|
@@ -738,10 +730,7 @@ export class BrowserAgent {
|
|
|
738
730
|
this.bus.emitNow({ type: 'turn-started', runId, turn: i });
|
|
739
731
|
try {
|
|
740
732
|
// -- 1. Check for recovery before observing --
|
|
741
|
-
// Only run analyzeRecovery when
|
|
742
|
-
// to run unconditionally; lazy-skipping it when there are no recent
|
|
743
|
-
// errors avoids the per-turn cost on the happy path. (Gen 5 lazy
|
|
744
|
-
// decision graph computation, change #20 in the pursuit spec.)
|
|
733
|
+
// Only run analyzeRecovery when recent turns show errors.
|
|
745
734
|
const hasErrorTrail = turns.length >= 2
|
|
746
735
|
&& (runState.consecutiveErrors > 0
|
|
747
736
|
|| turns.slice(-5).some((t) => t.error || t.verified === false));
|
|
@@ -843,16 +832,8 @@ export class BrowserAgent {
|
|
|
843
832
|
...(screenshotDataUrl ? { screenshot: screenshotDataUrl } : {}),
|
|
844
833
|
durationMs: observeDurationMs,
|
|
845
834
|
});
|
|
846
|
-
//
|
|
847
|
-
//
|
|
848
|
-
// "progress" when the URL changed from the prior turn, OR the
|
|
849
|
-
// snapshot byte size moved more than 5% (which filters out
|
|
850
|
-
// decorative animations + dynamic-id reshuffles but catches
|
|
851
|
-
// real DOM changes from clicks/typing/navigation). The 5% floor
|
|
852
|
-
// is intentionally loose — false positives just keep the run
|
|
853
|
-
// alive longer; false negatives cut off agents mid-flow on
|
|
854
|
-
// hot-spot sites (booking, google-flights), which is the
|
|
855
|
-
// failure mode we're trying to fix.
|
|
835
|
+
// Track page progress for the one-time max-turns extension. URL
|
|
836
|
+
// changes or >5% snapshot-size movement count as progress.
|
|
856
837
|
const priorTurn = turns[turns.length - 1];
|
|
857
838
|
if (priorTurn) {
|
|
858
839
|
const urlChanged = priorTurn.state?.url !== state.url;
|
|
@@ -871,9 +852,9 @@ export class BrowserAgent {
|
|
|
871
852
|
// Auto-navigate: if we're on about:blank with a startUrl, navigate without
|
|
872
853
|
// consuming an LLM turn. The agent always does wait->navigate on blank pages.
|
|
873
854
|
if (state.url === 'about:blank' &&
|
|
874
|
-
|
|
855
|
+
initialStartUrl &&
|
|
875
856
|
turns.length === 0) {
|
|
876
|
-
await this.driver.execute({ action: 'navigate', url:
|
|
857
|
+
await this.driver.execute({ action: 'navigate', url: initialStartUrl }).catch(() => { });
|
|
877
858
|
// Re-observe after navigation
|
|
878
859
|
const reState = await withRetry(() => this.driver.observe(), retries, retryDelayMs, undefined, scenario.signal);
|
|
879
860
|
Object.assign(state, reState);
|
|
@@ -958,16 +939,8 @@ export class BrowserAgent {
|
|
|
958
939
|
ctxBudget.add('blocker-recovery', '\nDialog dismissed but URL unchanged — prior action may have been voided. Re-submit if needed.\n', 90);
|
|
959
940
|
}
|
|
960
941
|
}
|
|
961
|
-
//
|
|
962
|
-
//
|
|
963
|
-
// If the agent has done 3+ consecutive single-step `type` actions on
|
|
964
|
-
// the same URL (i.e., it's filling a multi-field form one input at a
|
|
965
|
-
// time), inject a high-priority hint into extraContext that DEMANDS
|
|
966
|
-
// the next action be a `fill` covering the remaining fields.
|
|
967
|
-
//
|
|
968
|
-
// This is the runner-side enforcement layer for Gen 6 batch verbs.
|
|
969
|
-
// Prompt rules alone (Gen 6) didn't reliably steer the agent toward
|
|
970
|
-
// batch fill — runtime feedback does.
|
|
942
|
+
// Enforce batch fill when the agent starts filling a multi-field form
|
|
943
|
+
// one input at a time.
|
|
971
944
|
const batchFillHint = detectBatchFillOpportunity(turns, state);
|
|
972
945
|
if (batchFillHint && process.env.BAD_BATCH_HINT !== '0') {
|
|
973
946
|
ctxBudget.add('mandatory-batch-fill', batchFillHint, 100);
|
|
@@ -1027,9 +1000,7 @@ export class BrowserAgent {
|
|
|
1027
1000
|
if (searchScoutFeedback) {
|
|
1028
1001
|
ctxBudget.add('search-scout', `\n${searchScoutFeedback}\n`, 50);
|
|
1029
1002
|
}
|
|
1030
|
-
//
|
|
1031
|
-
// AND we're past the minimum-turns gate. Used to run unconditionally
|
|
1032
|
-
// every turn even when supervisor was disabled. Gen 5 evolve round 1.
|
|
1003
|
+
// Compute supervisor signals only after the supervisor is eligible.
|
|
1033
1004
|
const supervisorEligible = supervisorConfig.enabled &&
|
|
1034
1005
|
i >= supervisorConfig.minTurnsBeforeInvoke &&
|
|
1035
1006
|
runState.supervisorInterventions < supervisorConfig.maxInterventions &&
|
|
@@ -1160,6 +1131,19 @@ export class BrowserAgent {
|
|
|
1160
1131
|
if (this.config.walletMode && i === 1) {
|
|
1161
1132
|
ctxBudget.add('wallet-defi-context', DEFI_BRAIN_CONTEXT, 35);
|
|
1162
1133
|
}
|
|
1134
|
+
if (directStart && i === 1) {
|
|
1135
|
+
ctxBudget.add('direct-start', `\nDIRECT START: The initial URL was intentionally prepared by the ${directStart.profile} site profile (${directStart.reason}). If the URL uses future-equivalent Booking dates, keep those live bookable dates unless the page itself rejects them; do not navigate back to stale past dates.\n`, 82);
|
|
1136
|
+
}
|
|
1137
|
+
if (googleFlightsPreflight && i === 1) {
|
|
1138
|
+
const { spec } = googleFlightsPreflight;
|
|
1139
|
+
const rollText = spec.dateRoll
|
|
1140
|
+
? ` The stale benchmark date(s) were intentionally rolled from ${spec.dateRoll.originalDepartureDate}${spec.dateRoll.originalReturnDate ? `..${spec.dateRoll.originalReturnDate}` : ''} to live Google Flights dates ${spec.dateRoll.departureDate}${spec.dateRoll.returnDate ? `..${spec.dateRoll.returnDate}` : ''}; keep those live dates unless Google rejects them.`
|
|
1141
|
+
: '';
|
|
1142
|
+
const comparisonText = /\b(non-?stop|fewest stops?|least number of stops?|shortest|duration|total travel time)\b/i.test(scenario.goal)
|
|
1143
|
+
? ' For non-stop, fewest-stop, shortest-duration, or travel-time comparison tasks, first use the visible result cards already on the prepared results page; they include airline, price, stops, and duration. Do not open Price graph, Date grid, or broad filter panels unless the visible cards do not contain the requested comparison evidence.'
|
|
1144
|
+
: '';
|
|
1145
|
+
ctxBudget.add('google-flights-preflight', `\nGOOGLE FLIGHTS PREFLIGHT: ${googleFlightsPreflight.prepared ? 'The runner already prepared the search page' : 'The runner attempted to prepare the search page'} for ${spec.origin} to ${spec.destination}, ${spec.tripType}, ${spec.departureDate}${spec.returnDate ? ` to ${spec.returnDate}` : ''}. ${googleFlightsPreflight.reason}.${rollText} Your job now is to extract concrete visible Google Flights evidence from the current page; do not restart the form unless the prepared state is visibly wrong.${comparisonText} If Google says the exact requested live date is unavailable, too far in the future, or returns no results, abort with that evidence instead of substituting adjacent dates.\n`, 84);
|
|
1146
|
+
}
|
|
1163
1147
|
if (this.referenceTrajectory) {
|
|
1164
1148
|
ctxBudget.add('reference-trajectory', `\nREFERENCE TRAJECTORY — A similar task was completed before:\n${this.referenceTrajectory}\nUse this as a guide, but adapt to the current page state.\n`, 40);
|
|
1165
1149
|
}
|
|
@@ -1186,12 +1170,8 @@ export class BrowserAgent {
|
|
|
1186
1170
|
if (lastTurn?.action.action === 'runScript' && !lastTurn.error) {
|
|
1187
1171
|
ctxBudget.add('extraction-guard', '\nData extracted. If it answers the goal, complete now.\n', 80);
|
|
1188
1172
|
}
|
|
1189
|
-
//
|
|
1190
|
-
//
|
|
1191
|
-
// anti-bot. Worse than staying on the original site and grinding.
|
|
1192
|
-
// The stall detection idea is sound but the fallback destination is wrong.
|
|
1193
|
-
// TODO: revisit with a same-site strategy (runScript extraction, URL
|
|
1194
|
-
// construction from current state) instead of cross-site navigation.
|
|
1173
|
+
// Cross-site fallbacks are intentionally disabled here; stay on the
|
|
1174
|
+
// original site and recover with same-site extraction/navigation.
|
|
1195
1175
|
{
|
|
1196
1176
|
}
|
|
1197
1177
|
const extraContext = ctxBudget.build();
|
|
@@ -1213,10 +1193,8 @@ export class BrowserAgent {
|
|
|
1213
1193
|
const aiTangleOutputContext = aiTangleOutputCompletion
|
|
1214
1194
|
? `\nVERIFIED OUTPUT STATE DETECTED:\n${aiTangleOutputCompletion.feedback}\nReturn a terminal \`complete\` action now with concrete evidence.\n`
|
|
1215
1195
|
: '';
|
|
1216
|
-
//
|
|
1217
|
-
//
|
|
1218
|
-
// what point to recover. We only inject it once (consume it after
|
|
1219
|
-
// first use) so it doesn't pollute every subsequent turn.
|
|
1196
|
+
// Include the plan fallback hint on the first per-action turn after
|
|
1197
|
+
// a plan deviation, then consume it.
|
|
1220
1198
|
const planFallbackHint = planFallbackContext;
|
|
1221
1199
|
if (planFallbackContext)
|
|
1222
1200
|
planFallbackContext = '';
|
|
@@ -1305,10 +1283,6 @@ export class BrowserAgent {
|
|
|
1305
1283
|
}
|
|
1306
1284
|
}
|
|
1307
1285
|
else {
|
|
1308
|
-
// REVERTED: micro-movements during LLM thinking caused interference
|
|
1309
|
-
// with page state on interactive sites. The mouse.move calls during
|
|
1310
|
-
// decide() could trigger hover states, tooltips, or dismiss elements
|
|
1311
|
-
// the agent was about to click.
|
|
1312
1286
|
decision = await withRetry(() => this.brain.decide(scenario.goal, decisionState, finalExtraContext || undefined, { current: i, max: maxTurns }, { forceVision }), retries, retryDelayMs, (attempt, err) => {
|
|
1313
1287
|
if (this.config.debug) {
|
|
1314
1288
|
console.log(`[Runner] LLM retry ${attempt}: ${err.message}`);
|
|
@@ -1361,9 +1335,8 @@ export class BrowserAgent {
|
|
|
1361
1335
|
durationMs: decideDurationMs,
|
|
1362
1336
|
});
|
|
1363
1337
|
}
|
|
1364
|
-
// -- 4a.
|
|
1365
|
-
//
|
|
1366
|
-
// the recorded video can READ the agent's work, not just watch it:
|
|
1338
|
+
// -- 4a. Narrate to the cursor overlay. Fire-and-forget.
|
|
1339
|
+
// Signals pushed to the page-context overlay:
|
|
1367
1340
|
// 1. Reasoning panel (top-right) — the agent's own text
|
|
1368
1341
|
// 2. Progress bar + chip (top) — turn N with optional ledger marker
|
|
1369
1342
|
// 3. Verdict badges (bottom-left) — POSITIVE/CLEARED/REVIEW events
|
|
@@ -1541,13 +1514,11 @@ export class BrowserAgent {
|
|
|
1541
1514
|
this.onTurn?.(turn);
|
|
1542
1515
|
continue;
|
|
1543
1516
|
}
|
|
1544
|
-
// -- 5d. Handle extractWithIndex action
|
|
1517
|
+
// -- 5d. Handle extractWithIndex action --
|
|
1545
1518
|
// Returns a numbered list of every visible element matching `query`,
|
|
1546
1519
|
// each with its tag, textContent, key attributes, and a stable
|
|
1547
1520
|
// selector. The agent picks elements by index in the next turn.
|
|
1548
|
-
//
|
|
1549
|
-
// pick-by-selector. Works on data the planner couldn't see at plan
|
|
1550
|
-
// time (XHR-loaded content, dl/dt/dd, deeply-nested wrappers).
|
|
1521
|
+
// Pick by visible content instead of brittle selectors.
|
|
1551
1522
|
if (action.action === 'extractWithIndex') {
|
|
1552
1523
|
const page = this.driver.getPage?.();
|
|
1553
1524
|
if (page) {
|
|
@@ -1574,11 +1545,9 @@ export class BrowserAgent {
|
|
|
1574
1545
|
this.onTurn?.(turn);
|
|
1575
1546
|
continue;
|
|
1576
1547
|
}
|
|
1577
|
-
// --
|
|
1578
|
-
//
|
|
1579
|
-
//
|
|
1580
|
-
// verdicts, and inject the merged result as feedback for the next
|
|
1581
|
-
// turn. Parent page state is untouched.
|
|
1548
|
+
// -- Mid-run parallel fan-out --
|
|
1549
|
+
// Explore independent candidates in sibling tabs, then inject the
|
|
1550
|
+
// merged result as feedback for the next turn.
|
|
1582
1551
|
if (action.action === 'fanOut') {
|
|
1583
1552
|
const page = this.driver.getPage?.();
|
|
1584
1553
|
const context = page?.context();
|
|
@@ -1595,10 +1564,8 @@ export class BrowserAgent {
|
|
|
1595
1564
|
context,
|
|
1596
1565
|
config: this.config,
|
|
1597
1566
|
currentUrl: state.url,
|
|
1598
|
-
//
|
|
1599
|
-
//
|
|
1600
|
-
// streaming, verdict chips, collapse animation) on the
|
|
1601
|
-
// parent page for the whole fan-out duration.
|
|
1567
|
+
// Parent driver lets the executor drive the fan-out overlay for
|
|
1568
|
+
// the whole fan-out duration.
|
|
1602
1569
|
parentDriver: this.driver,
|
|
1603
1570
|
...(topDriverOptions
|
|
1604
1571
|
? { driverOptions: (() => { const { showCursor: _sc, ...rest } = topDriverOptions; return rest; })() }
|
|
@@ -1643,20 +1610,14 @@ export class BrowserAgent {
|
|
|
1643
1610
|
// (>50 chars) combined with script-extracted evidence means the
|
|
1644
1611
|
// verifier almost always agrees — save the round-trip.
|
|
1645
1612
|
//
|
|
1646
|
-
//
|
|
1647
|
-
//
|
|
1648
|
-
// not take effect") yet marks success. The old heuristic (length
|
|
1649
|
-
// + evidence + no errors) rubber-stamped these. Now we scan the
|
|
1650
|
-
// result text for self-contradicting phrases and force LLM
|
|
1651
|
-
// verification when found. This fixes the 6/8 judge disagreement
|
|
1652
|
-
// cases from Gen 11 evolve R2.
|
|
1613
|
+
// Content-aware gate: self-contradicting completion text forces
|
|
1614
|
+
// LLM verification instead of using the fast path.
|
|
1653
1615
|
const agentResult = action.result || '';
|
|
1654
1616
|
const recentErrors = turns.slice(-2).filter(t => t.error).length;
|
|
1655
1617
|
const hasScriptEvidence = verificationEvidence.some(e => e.startsWith('SCRIPT RESULT:'));
|
|
1656
|
-
//
|
|
1657
|
-
//
|
|
1658
|
-
|
|
1659
|
-
const selfContradicting = /\b(?:could not (?:complete|find|fulfill|verify|confirm|locate|access|extract|retrieve)|not (?:visible|available|found|present|accessible|displayed|shown|confirmed|verified)|did not (?:take effect|work|succeed|load|return)|unable to (?:find|complete|verify|access|extract|retrieve)|no (?:visible (?:answer|result|data|content)|results? (?:found|returned|available))|(?:failed|failure) to (?:find|complete|set|select|navigate)|unfortunately|I (?:was|am) unable|(?:task|request|goal) (?:is|was) (?:not |in)complete)\b/i.test(agentResult);
|
|
1618
|
+
// Detect when the agent's own text admits failure despite
|
|
1619
|
+
// claiming success.
|
|
1620
|
+
const selfContradicting = containsSelfContradictingCompletion(agentResult);
|
|
1660
1621
|
const fastPathEligible = agentResult.length > 50 &&
|
|
1661
1622
|
recentErrors === 0 &&
|
|
1662
1623
|
hasScriptEvidence &&
|
|
@@ -1677,7 +1638,7 @@ export class BrowserAgent {
|
|
|
1677
1638
|
// own text suggests failure. The LLM verifier reads the actual
|
|
1678
1639
|
// content and makes the right call.
|
|
1679
1640
|
if (this.config.debug) {
|
|
1680
|
-
console.log('[Runner]
|
|
1641
|
+
console.log('[Runner] Fast-path blocked: agent result contains self-contradicting language, forcing LLM verification');
|
|
1681
1642
|
}
|
|
1682
1643
|
goalResult = await this.brain.verifyGoalCompletion(state, scenario.goal, buildGoalVerificationClaim(agentResult, verificationEvidence));
|
|
1683
1644
|
}
|
|
@@ -1687,7 +1648,19 @@ export class BrowserAgent {
|
|
|
1687
1648
|
if (this.config.debug) {
|
|
1688
1649
|
console.log(`[Runner] Goal verification: achieved=${goalResult.achieved}, confidence=${goalResult.confidence}`);
|
|
1689
1650
|
}
|
|
1651
|
+
if (selfContradicting && goalResult.achieved) {
|
|
1652
|
+
goalResult = {
|
|
1653
|
+
achieved: false,
|
|
1654
|
+
confidence: Math.min(goalResult.confidence, 0.4),
|
|
1655
|
+
evidence: goalResult.evidence,
|
|
1656
|
+
missing: [
|
|
1657
|
+
...goalResult.missing,
|
|
1658
|
+
'Rejected because the completion text itself admits that the exact requested task was not completed.',
|
|
1659
|
+
],
|
|
1660
|
+
};
|
|
1661
|
+
}
|
|
1690
1662
|
if (!goalResult.achieved
|
|
1663
|
+
&& !selfContradicting
|
|
1691
1664
|
&& shouldAcceptFirstPartyBoundaryCompletion(scenario.goal, state.url, goalResult, action.result || '')) {
|
|
1692
1665
|
goalResult = {
|
|
1693
1666
|
...goalResult,
|
|
@@ -1701,6 +1674,7 @@ export class BrowserAgent {
|
|
|
1701
1674
|
};
|
|
1702
1675
|
}
|
|
1703
1676
|
if (!goalResult.achieved
|
|
1677
|
+
&& !selfContradicting
|
|
1704
1678
|
&& shouldAcceptSearchWorkflowCompletion(scenario.goal, goalResult, action.result || '', verificationEvidence)) {
|
|
1705
1679
|
goalResult = {
|
|
1706
1680
|
...goalResult,
|
|
@@ -1714,6 +1688,7 @@ export class BrowserAgent {
|
|
|
1714
1688
|
};
|
|
1715
1689
|
}
|
|
1716
1690
|
if (!goalResult.achieved
|
|
1691
|
+
&& !selfContradicting
|
|
1717
1692
|
&& shouldAcceptScriptBackedCompletion(scenario.goal, state, goalResult, action.result || '', verificationEvidence)) {
|
|
1718
1693
|
goalResult = {
|
|
1719
1694
|
...goalResult,
|
|
@@ -1726,6 +1701,34 @@ export class BrowserAgent {
|
|
|
1726
1701
|
missing: [],
|
|
1727
1702
|
};
|
|
1728
1703
|
}
|
|
1704
|
+
if (!goalResult.achieved
|
|
1705
|
+
&& !selfContradicting
|
|
1706
|
+
&& shouldAcceptRolledBookingCompletion(directStart, goalResult, action.result || '', state)) {
|
|
1707
|
+
goalResult = {
|
|
1708
|
+
...goalResult,
|
|
1709
|
+
achieved: true,
|
|
1710
|
+
confidence: Math.max(goalResult.confidence, 0.82),
|
|
1711
|
+
evidence: [
|
|
1712
|
+
...goalResult.evidence,
|
|
1713
|
+
'Accepted under Booking direct-start date-roll policy: the stale benchmark dates were mapped to equivalent live bookable dates and the completion included concrete Booking evidence.',
|
|
1714
|
+
],
|
|
1715
|
+
missing: [],
|
|
1716
|
+
};
|
|
1717
|
+
}
|
|
1718
|
+
if (!goalResult.achieved
|
|
1719
|
+
&& !selfContradicting
|
|
1720
|
+
&& shouldAcceptRolledGoogleFlightsCompletion(googleFlightsPreflight, goalResult, action.result || '', state)) {
|
|
1721
|
+
goalResult = {
|
|
1722
|
+
...goalResult,
|
|
1723
|
+
achieved: true,
|
|
1724
|
+
confidence: Math.max(goalResult.confidence, 0.82),
|
|
1725
|
+
evidence: [
|
|
1726
|
+
...goalResult.evidence,
|
|
1727
|
+
'Accepted under Google Flights preflight date-roll policy: stale benchmark flight dates were mapped to equivalent live Google Flights dates and the completion included concrete flight-result evidence.',
|
|
1728
|
+
],
|
|
1729
|
+
missing: [],
|
|
1730
|
+
};
|
|
1731
|
+
}
|
|
1729
1732
|
const contentTypeMismatch = detectCompletionContentTypeMismatch(scenario.goal, state, action.result || '', verificationEvidence);
|
|
1730
1733
|
if (contentTypeMismatch) {
|
|
1731
1734
|
runState.verificationRejectionCount++;
|
|
@@ -1745,9 +1748,9 @@ export class BrowserAgent {
|
|
|
1745
1748
|
// Tier C: 3+ rejections + confidence ≥0.40 → accept (prevent total turn exhaustion)
|
|
1746
1749
|
const hasSupplementalEvidence = verificationEvidence.length > 0;
|
|
1747
1750
|
const priorRejections = runState.verificationRejectionCount;
|
|
1748
|
-
const shouldAccept = (priorRejections >= 1 && goalResult.confidence >= 0.55 && hasSupplementalEvidence) ||
|
|
1751
|
+
const shouldAccept = !selfContradicting && ((priorRejections >= 1 && goalResult.confidence >= 0.55 && hasSupplementalEvidence) ||
|
|
1749
1752
|
(priorRejections >= 2 && goalResult.confidence >= 0.50) ||
|
|
1750
|
-
(priorRejections >= 3 && goalResult.confidence >= 0.40);
|
|
1753
|
+
(priorRejections >= 3 && goalResult.confidence >= 0.40));
|
|
1751
1754
|
if (shouldAccept) {
|
|
1752
1755
|
goalResult = {
|
|
1753
1756
|
...goalResult,
|
|
@@ -1765,9 +1768,8 @@ export class BrowserAgent {
|
|
|
1765
1768
|
runState.verificationRejectionCount++;
|
|
1766
1769
|
turn.verificationFailure = goalResult.missing.join('; ') || 'Goal verification failed';
|
|
1767
1770
|
runState.firstSufficientEvidenceTurn ??= i;
|
|
1768
|
-
//
|
|
1769
|
-
//
|
|
1770
|
-
// of continuing from the wrong-path state.
|
|
1771
|
+
// On the second rejection, roll back to a previous checkpoint
|
|
1772
|
+
// before trying a different path.
|
|
1771
1773
|
let replayNote = '';
|
|
1772
1774
|
if (runState.verificationRejectionCount === 2 && runState.checkpoints.length >= 2) {
|
|
1773
1775
|
// Go back to the second-to-last checkpoint (before the wrong path)
|
|
@@ -1780,7 +1782,7 @@ export class BrowserAgent {
|
|
|
1780
1782
|
catch { /* rollback failed, continue from current state */ }
|
|
1781
1783
|
}
|
|
1782
1784
|
}
|
|
1783
|
-
//
|
|
1785
|
+
// Escalate recovery guidance after repeated verification rejects.
|
|
1784
1786
|
let escalation;
|
|
1785
1787
|
if (runState.verificationRejectionCount >= 3) {
|
|
1786
1788
|
escalation = ' STRATEGY SHIFT REQUIRED: Your previous approaches have failed 3 times. Try a COMPLETELY different method: use navigate to go to a different search engine or URL, try extractWithIndex instead of runScript, or scroll to look for the data in a different part of the page. Do NOT repeat what you just tried.';
|
|
@@ -1964,7 +1966,7 @@ export class BrowserAgent {
|
|
|
1964
1966
|
else {
|
|
1965
1967
|
runState.clearConsecutiveErrors();
|
|
1966
1968
|
executeTimeoutRecoveries = 0; // Reset on successful action
|
|
1967
|
-
//
|
|
1969
|
+
// Surface form reset warnings from batch fill verification.
|
|
1968
1970
|
if ('warning' in execResult && typeof execResult.warning === 'string') {
|
|
1969
1971
|
const warning = execResult.warning;
|
|
1970
1972
|
this.brain.injectFeedback(warning);
|
|
@@ -1972,8 +1974,7 @@ export class BrowserAgent {
|
|
|
1972
1974
|
console.log(`[Runner] Fill warning: ${warning}`);
|
|
1973
1975
|
}
|
|
1974
1976
|
}
|
|
1975
|
-
//
|
|
1976
|
-
// These are rollback points for wrong-path recovery.
|
|
1977
|
+
// Save URL checkpoints after successful navigation for rollback.
|
|
1977
1978
|
const postUrl = this.driver.getPage?.()?.url() || '';
|
|
1978
1979
|
const lastCheckpointUrl = runState.checkpoints[runState.checkpoints.length - 1]?.url;
|
|
1979
1980
|
if (postUrl && postUrl !== 'about:blank' && postUrl !== lastCheckpointUrl) {
|
|
@@ -2182,9 +2183,7 @@ export class BrowserAgent {
|
|
|
2182
2183
|
try {
|
|
2183
2184
|
if (this.knowledge && scenario && result) {
|
|
2184
2185
|
this.knowledge.recordSession(buildSession(scenario, result));
|
|
2185
|
-
//
|
|
2186
|
-
// Patterns gain confidence with repeated observation and auto-decay
|
|
2187
|
-
// when contradicted. Low-confidence facts are pruned automatically.
|
|
2186
|
+
// Extract reusable patterns from successful runs.
|
|
2188
2187
|
if (result.success && turns && turns.length > 0) {
|
|
2189
2188
|
const domain = safeHostname(scenario.startUrl || '') || '';
|
|
2190
2189
|
if (domain) {
|
|
@@ -2225,7 +2224,7 @@ export class BrowserAgent {
|
|
|
2225
2224
|
* - Generic text match -> check if text appears in snapshot
|
|
2226
2225
|
*/
|
|
2227
2226
|
/**
|
|
2228
|
-
*
|
|
2227
|
+
* Execute a Plan deterministically without re-entering the LLM
|
|
2229
2228
|
* between steps. Each step:
|
|
2230
2229
|
* 1. Drives the action via driver.execute (existing path, gets bus events)
|
|
2231
2230
|
* 2. Verifies the post-condition via verifyExpectedEffect
|
|
@@ -2251,25 +2250,17 @@ export class BrowserAgent {
|
|
|
2251
2250
|
* one plan call per N steps. To make the run-level cost tally honest,
|
|
2252
2251
|
* we attribute the plan call to the FIRST step's Turn artifact so the
|
|
2253
2252
|
* downstream sum (in baseline-summary.json / report.json) reflects the
|
|
2254
|
-
* real LLM spend.
|
|
2255
|
-
* report $0 cost while Gen 7 baseline runs reported $0.50.
|
|
2253
|
+
* real LLM spend.
|
|
2256
2254
|
*/
|
|
2257
2255
|
planCallTokens) {
|
|
2258
2256
|
let currentTurnIndex = startingTurnIndex;
|
|
2259
2257
|
let lastState = turns[turns.length - 1]?.state
|
|
2260
2258
|
?? { url: '', title: '', snapshot: '' };
|
|
2261
|
-
//
|
|
2262
|
-
//
|
|
2263
|
-
// "<from prior step>", etc.) can be substituted with the real script
|
|
2264
|
-
// output. The planner has to commit to its `complete.result` text BEFORE
|
|
2265
|
-
// runScript runs, so on extraction tasks it fabricates placeholders.
|
|
2266
|
-
// This deterministic substitution fixes that without an extra LLM call.
|
|
2259
|
+
// Track the last successful runScript output so placeholder complete
|
|
2260
|
+
// results can be substituted with the real script output.
|
|
2267
2261
|
let lastRunScriptOutput = null;
|
|
2268
|
-
//
|
|
2269
|
-
//
|
|
2270
|
-
// must read the formatted match list and pick by index. When the plan
|
|
2271
|
-
// ends with extractWithIndex (or runs out of valid steps), we fall
|
|
2272
|
-
// through to the per-action loop with the match list as feedback.
|
|
2262
|
+
// Track extractWithIndex matches for per-action fallback; the LLM must
|
|
2263
|
+
// read the list and pick by index.
|
|
2273
2264
|
let lastExtractOutput = null;
|
|
2274
2265
|
for (let stepIdx = 0; stepIdx < plan.steps.length; stepIdx++) {
|
|
2275
2266
|
if (scenario.signal?.aborted) {
|
|
@@ -2296,10 +2287,8 @@ export class BrowserAgent {
|
|
|
2296
2287
|
// timeline regardless of whether the runner used the planner or the
|
|
2297
2288
|
// per-action loop.
|
|
2298
2289
|
//
|
|
2299
|
-
// Token attribution: the
|
|
2300
|
-
//
|
|
2301
|
-
// in plan-mode (Gen 7.1) report $0 cost while their Brain.plan()
|
|
2302
|
-
// calls actually spent real tokens.
|
|
2290
|
+
// Token attribution: the first step carries the Brain.plan() LLM call's
|
|
2291
|
+
// token usage so run-level cost includes planning.
|
|
2303
2292
|
const isFirstStep = stepIdx === 0;
|
|
2304
2293
|
const turn = {
|
|
2305
2294
|
turn: turnNumber,
|
|
@@ -2319,21 +2308,16 @@ export class BrowserAgent {
|
|
|
2319
2308
|
// Terminal actions: complete and abort don't go through driver.execute
|
|
2320
2309
|
// — the runner handles them as the end of the plan.
|
|
2321
2310
|
if (step.action.action === 'complete') {
|
|
2322
|
-
//
|
|
2323
|
-
// with placeholder values AND we have a real runScript output from
|
|
2324
|
-
// earlier in the plan, use the runScript output as the final result.
|
|
2325
|
-
// Detection is conservative: only substitute when the planner clearly
|
|
2326
|
-
// didn't know real values at planning time (null literals, "<from
|
|
2327
|
-
// prior step>", "{{...}}" templates, "<placeholder>", etc.).
|
|
2311
|
+
// Substitute placeholder complete results with prior runScript output.
|
|
2328
2312
|
let resolvedResult = step.action.result;
|
|
2329
2313
|
if (lastRunScriptOutput
|
|
2330
2314
|
&& typeof resolvedResult === 'string'
|
|
2331
2315
|
&& hasPlaceholderPattern(resolvedResult)) {
|
|
2332
2316
|
if (this.config.debug) {
|
|
2333
|
-
console.log(`[Runner]
|
|
2317
|
+
console.log(`[Runner] Substituting placeholder complete.result with runScript output (${lastRunScriptOutput.length} chars)`);
|
|
2334
2318
|
}
|
|
2335
2319
|
resolvedResult = lastRunScriptOutput;
|
|
2336
|
-
turn.reasoning = `${turn.reasoning ?? ''} [
|
|
2320
|
+
turn.reasoning = `${turn.reasoning ?? ''} [substituted runScript output]`.trim();
|
|
2337
2321
|
}
|
|
2338
2322
|
turn.durationMs = Date.now() - stepStartedAt;
|
|
2339
2323
|
turns.push(turn);
|
|
@@ -2380,12 +2364,8 @@ export class BrowserAgent {
|
|
|
2380
2364
|
// execute-started / execute-completed events on the bus exactly
|
|
2381
2365
|
// like the per-action loop does.
|
|
2382
2366
|
//
|
|
2383
|
-
//
|
|
2384
|
-
//
|
|
2385
|
-
// the snapshot at planning time — a missing element should fail
|
|
2386
|
-
// FAST and trigger fallback to per-action mode, NOT block the run for
|
|
2387
|
-
// 30s. Batch verbs already enforce a 5s per-field cap internally,
|
|
2388
|
-
// but single-step type/click/press/select use the full 30s default.
|
|
2367
|
+
// Cap each plan step at 10s so missing selectors fail quickly and hand
|
|
2368
|
+
// control back to per-action mode.
|
|
2389
2369
|
this.bus.emitNow({ type: 'execute-started', runId, turn: turnNumber, action: step.action });
|
|
2390
2370
|
const execStartedAt = Date.now();
|
|
2391
2371
|
const planStepTimeoutMs = 10_000;
|
|
@@ -2450,18 +2430,14 @@ export class BrowserAgent {
|
|
|
2450
2430
|
runState.clearConsecutiveErrors();
|
|
2451
2431
|
if (execResult.bounds)
|
|
2452
2432
|
turn.actionBounds = execResult.bounds;
|
|
2453
|
-
//
|
|
2454
|
-
// with placeholder values can be substituted with the real output.
|
|
2455
|
-
// This is the supply side of the placeholder-substitution fix above.
|
|
2433
|
+
// Capture runScript output for placeholder substitution and evidence.
|
|
2456
2434
|
if (step.action.action === 'runScript' && typeof execResult.data === 'string' && execResult.data.length > 0) {
|
|
2457
2435
|
lastRunScriptOutput = execResult.data;
|
|
2458
2436
|
if (execResult.data.length > 10) {
|
|
2459
2437
|
runState.recordEvidence(`EXTRACTED (turn ${currentTurnIndex}): ${execResult.data.slice(0, 500)}`);
|
|
2460
2438
|
}
|
|
2461
2439
|
}
|
|
2462
|
-
//
|
|
2463
|
-
// per-action loop. The LLM must read the list and pick by index — we
|
|
2464
|
-
// do not auto-complete with the raw match list.
|
|
2440
|
+
// Capture extractWithIndex output for per-action fallback.
|
|
2465
2441
|
if (step.action.action === 'extractWithIndex' && typeof execResult.data === 'string' && execResult.data.length > 0) {
|
|
2466
2442
|
lastExtractOutput = execResult.data;
|
|
2467
2443
|
// Also push as goal verification evidence so the verifier sees what
|
|
@@ -2475,8 +2451,7 @@ export class BrowserAgent {
|
|
|
2475
2451
|
// Verify the post-condition. We re-observe to get the post-action
|
|
2476
2452
|
// state, then run the same verifyExpectedEffect helper the per-action
|
|
2477
2453
|
// loop uses. The fresh observe is also stashed in cachedPostState so
|
|
2478
|
-
// the next step's pre-step observe
|
|
2479
|
-
// observe optimization).
|
|
2454
|
+
// the next step's pre-step observe can reuse it.
|
|
2480
2455
|
this.bus.emitNow({
|
|
2481
2456
|
type: 'verify-started',
|
|
2482
2457
|
runId,
|
|
@@ -2600,18 +2575,7 @@ export class BrowserAgent {
|
|
|
2600
2575
|
};
|
|
2601
2576
|
}
|
|
2602
2577
|
}
|
|
2603
|
-
//
|
|
2604
|
-
// explicit complete BUT the last successful step was a runScript with
|
|
2605
|
-
// non-empty output, treat the runScript output as the final result and
|
|
2606
|
-
// synthesize a complete turn. This handles the planner-prompt path where
|
|
2607
|
-
// the planner correctly emits ONLY runScript on extraction tasks (per
|
|
2608
|
-
// rule #7) — without this, we'd fall through to a 4-5 turn per-action
|
|
2609
|
-
// loop that's much slower than necessary.
|
|
2610
|
-
//
|
|
2611
|
-
// Detection: the LAST step in the plan was a `runScript` AND we captured
|
|
2612
|
-
// a non-empty output for it. We don't check intermediate steps because
|
|
2613
|
-
// a plan like [navigate, click, runScript] where runScript is last is
|
|
2614
|
-
// exactly the extraction-task shape we want to short-circuit.
|
|
2578
|
+
// Auto-complete when the plan ends with meaningful runScript output.
|
|
2615
2579
|
const lastStep = plan.steps[plan.steps.length - 1];
|
|
2616
2580
|
if (lastStep
|
|
2617
2581
|
&& lastStep.action.action === 'runScript'
|
|
@@ -2621,7 +2585,7 @@ export class BrowserAgent {
|
|
|
2621
2585
|
turn: synthTurnNumber,
|
|
2622
2586
|
state: lastState,
|
|
2623
2587
|
action: { action: 'complete', result: lastRunScriptOutput },
|
|
2624
|
-
reasoning: '
|
|
2588
|
+
reasoning: 'Auto-complete: plan ended after runScript, runner emitted complete with the runScript output',
|
|
2625
2589
|
durationMs: 0,
|
|
2626
2590
|
};
|
|
2627
2591
|
turns.push(synthTurn);
|
|
@@ -2638,7 +2602,7 @@ export class BrowserAgent {
|
|
|
2638
2602
|
durationMs: 0,
|
|
2639
2603
|
});
|
|
2640
2604
|
if (this.config.debug) {
|
|
2641
|
-
console.log(`[Runner]
|
|
2605
|
+
console.log(`[Runner] Auto-emitted complete with runScript output (${lastRunScriptOutput.length} chars) after plan exhausted`);
|
|
2642
2606
|
}
|
|
2643
2607
|
return {
|
|
2644
2608
|
kind: 'completed',
|
|
@@ -2647,12 +2611,8 @@ export class BrowserAgent {
|
|
|
2647
2611
|
turnsConsumed: plan.steps.length + 1,
|
|
2648
2612
|
};
|
|
2649
2613
|
}
|
|
2650
|
-
//
|
|
2651
|
-
//
|
|
2652
|
-
// the matches and pick by index — we do NOT auto-complete with the raw
|
|
2653
|
-
// match list. This is the planner-emits-extract path for extraction
|
|
2654
|
-
// tasks like npm/mdn/python-docs where the planner used the new
|
|
2655
|
-
// extractWithIndex action.
|
|
2614
|
+
// If the plan produced extractWithIndex matches, fall through with the
|
|
2615
|
+
// match list so the LLM can choose the correct index.
|
|
2656
2616
|
if (lastExtractOutput) {
|
|
2657
2617
|
return {
|
|
2658
2618
|
kind: 'deviated',
|
|
@@ -2662,20 +2622,13 @@ export class BrowserAgent {
|
|
|
2662
2622
|
turnsConsumed: plan.steps.length,
|
|
2663
2623
|
};
|
|
2664
2624
|
}
|
|
2665
|
-
//
|
|
2666
|
-
//
|
|
2667
|
-
// auto-complete with garbage. Fall through to the per-action loop with
|
|
2668
|
-
// a deviation reason that names the empty output. In Gen 10 the per-
|
|
2669
|
-
// action loop has TWO new tools that make this recovery actually work:
|
|
2670
|
-
// 1. extractWithIndex (the wide-query content-match action) — see
|
|
2671
|
-
// data-extraction rule #25
|
|
2672
|
-
// 2. cost cap (100k tokens) — bounds any death-spiral if the LLM
|
|
2673
|
-
// can't recover, preventing the Gen 9.1 reddit failure mode
|
|
2625
|
+
// If the plan ends with runScript but output is empty or placeholder-like,
|
|
2626
|
+
// fall through to per-action mode instead of completing with bad data.
|
|
2674
2627
|
if (lastStep
|
|
2675
2628
|
&& lastStep.action.action === 'runScript'
|
|
2676
2629
|
&& !isMeaningfulRunScriptOutput(lastRunScriptOutput)) {
|
|
2677
2630
|
if (this.config.debug) {
|
|
2678
|
-
console.log(`[Runner]
|
|
2631
|
+
console.log(`[Runner] runScript returned no meaningful output (${JSON.stringify(lastRunScriptOutput).slice(0, 100)}); falling through to per-action loop for two-pass extraction`);
|
|
2679
2632
|
}
|
|
2680
2633
|
return {
|
|
2681
2634
|
kind: 'deviated',
|