@tangle-network/browser-agent-driver 0.31.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artifacts/filesystem-sink.d.ts +1 -1
- package/dist/artifacts/filesystem-sink.js +1 -1
- package/dist/brain/index.d.ts +16 -9
- package/dist/brain/index.d.ts.map +1 -1
- package/dist/brain/index.js +75 -72
- package/dist/brain/index.js.map +1 -1
- package/dist/browser-launch.js +1 -3
- package/dist/browser-launch.js.map +1 -1
- package/dist/captcha.d.ts.map +1 -1
- package/dist/captcha.js +1 -3
- package/dist/captcha.js.map +1 -1
- package/dist/cli-attach.d.ts +2 -2
- package/dist/cli-attach.js +2 -2
- package/dist/cli-design-audit.d.ts.map +1 -1
- package/dist/cli-design-audit.js +14 -24
- package/dist/cli-design-audit.js.map +1 -1
- package/dist/cli-jobs.js +4 -13
- package/dist/cli-jobs.js.map +1 -1
- package/dist/cli-patterns.d.ts +1 -1
- package/dist/cli-patterns.d.ts.map +1 -1
- package/dist/cli.js +26 -28
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +5 -3
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/design/audit/attribution/types.d.ts +3 -3
- package/dist/design/audit/attribution/types.d.ts.map +1 -1
- package/dist/design/audit/attribution/types.js +2 -2
- package/dist/design/audit/build-result.d.ts +71 -0
- package/dist/design/audit/build-result.d.ts.map +1 -0
- package/dist/design/audit/build-result.js +251 -0
- package/dist/design/audit/build-result.js.map +1 -0
- package/dist/design/audit/classify-ensemble.d.ts +1 -1
- package/dist/design/audit/classify-ensemble.d.ts.map +1 -1
- package/dist/design/audit/ethics/check.d.ts +2 -2
- package/dist/design/audit/ethics/check.d.ts.map +1 -1
- package/dist/design/audit/ethics/check.js +1 -1
- package/dist/design/audit/ethics/loader.d.ts +1 -1
- package/dist/design/audit/ethics/loader.d.ts.map +1 -1
- package/dist/design/audit/evaluate.d.ts +5 -2
- package/dist/design/audit/evaluate.d.ts.map +1 -1
- package/dist/design/audit/evaluate.js +10 -4
- package/dist/design/audit/evaluate.js.map +1 -1
- package/dist/design/audit/first-principles-mode.d.ts +1 -1
- package/dist/design/audit/first-principles-mode.d.ts.map +1 -1
- package/dist/design/audit/modality/android.d.ts +1 -1
- package/dist/design/audit/modality/android.d.ts.map +1 -1
- package/dist/design/audit/modality/html.d.ts +1 -1
- package/dist/design/audit/modality/html.d.ts.map +1 -1
- package/dist/design/audit/modality/ios.d.ts +1 -1
- package/dist/design/audit/modality/ios.d.ts.map +1 -1
- package/dist/design/audit/modality/types.d.ts +2 -2
- package/dist/design/audit/modality/types.d.ts.map +1 -1
- package/dist/design/audit/modality/types.js +1 -1
- package/dist/design/audit/patches/generate.d.ts +57 -0
- package/dist/design/audit/patches/generate.d.ts.map +1 -0
- package/dist/design/audit/patches/generate.js +168 -0
- package/dist/design/audit/patches/generate.js.map +1 -0
- package/dist/design/audit/patches/parse.d.ts +1 -1
- package/dist/design/audit/patches/parse.d.ts.map +1 -1
- package/dist/design/audit/patches/render.d.ts +1 -1
- package/dist/design/audit/patches/render.d.ts.map +1 -1
- package/dist/design/audit/patches/severity-enforcement.d.ts +1 -1
- package/dist/design/audit/patches/severity-enforcement.d.ts.map +1 -1
- package/dist/design/audit/patches/validate.d.ts +7 -1
- package/dist/design/audit/patches/validate.d.ts.map +1 -1
- package/dist/design/audit/patches/validate.js +8 -1
- package/dist/design/audit/patches/validate.js.map +1 -1
- package/dist/design/audit/patterns/match.d.ts +1 -1
- package/dist/design/audit/patterns/match.d.ts.map +1 -1
- package/dist/design/audit/patterns/types.d.ts +2 -2
- package/dist/design/audit/patterns/types.d.ts.map +1 -1
- package/dist/design/audit/pipeline.d.ts +4 -4
- package/dist/design/audit/pipeline.d.ts.map +1 -1
- package/dist/design/audit/pipeline.js +45 -30
- package/dist/design/audit/pipeline.js.map +1 -1
- package/dist/design/audit/rubric/loader.d.ts +1 -1
- package/dist/design/audit/rubric/loader.d.ts.map +1 -1
- package/dist/design/audit/rubric/rollup-weights.d.ts +1 -1
- package/dist/design/audit/rubric/rollup-weights.d.ts.map +1 -1
- package/dist/design/audit/{v2/types.d.ts → score-types.d.ts} +10 -10
- package/dist/design/audit/score-types.d.ts.map +1 -0
- package/dist/design/audit/{v2/types.js → score-types.js} +2 -2
- package/dist/design/audit/score-types.js.map +1 -0
- package/dist/design/audit/{v2/score.d.ts → score.d.ts} +9 -9
- package/dist/design/audit/score.d.ts.map +1 -0
- package/dist/design/audit/{v2/score.js → score.js} +17 -17
- package/dist/design/audit/score.js.map +1 -0
- package/dist/design/audit/types.d.ts +7 -7
- package/dist/design/audit/types.d.ts.map +1 -1
- package/dist/drivers/cursor-overlay.d.ts +1 -1
- package/dist/drivers/cursor-overlay.d.ts.map +1 -1
- package/dist/drivers/cursor-overlay.js +2 -2
- package/dist/drivers/extract-with-index.d.ts +4 -7
- package/dist/drivers/extract-with-index.d.ts.map +1 -1
- package/dist/drivers/extract-with-index.js +4 -7
- package/dist/drivers/extract-with-index.js.map +1 -1
- package/dist/drivers/overlay-label.d.ts +3 -4
- package/dist/drivers/overlay-label.d.ts.map +1 -1
- package/dist/drivers/overlay-label.js.map +1 -1
- package/dist/drivers/playwright.d.ts +2 -2
- package/dist/drivers/playwright.d.ts.map +1 -1
- package/dist/drivers/playwright.js +10 -24
- package/dist/drivers/playwright.js.map +1 -1
- package/dist/drivers/som-overlay.d.ts +2 -3
- package/dist/drivers/som-overlay.d.ts.map +1 -1
- package/dist/drivers/som-overlay.js +2 -3
- package/dist/drivers/som-overlay.js.map +1 -1
- package/dist/drivers/types.d.ts +3 -9
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/jobs/anti-bot.js +1 -1
- package/dist/jobs/anti-bot.js.map +1 -1
- package/dist/jobs/types.d.ts +1 -1
- package/dist/jobs/types.d.ts.map +1 -1
- package/dist/multi-actor.d.ts.map +1 -1
- package/dist/multi-actor.js +2 -3
- package/dist/multi-actor.js.map +1 -1
- package/dist/reports/aggregate.js +6 -6
- package/dist/reports/aggregate.js.map +1 -1
- package/dist/reports/tokens.d.ts +0 -8
- package/dist/reports/tokens.d.ts.map +1 -1
- package/dist/reports/tokens.js +1 -15
- package/dist/reports/tokens.js.map +1 -1
- package/dist/reports/types.d.ts +2 -2
- package/dist/reports/types.d.ts.map +1 -1
- package/dist/run-state.d.ts +21 -15
- package/dist/run-state.d.ts.map +1 -1
- package/dist/run-state.js +23 -22
- package/dist/run-state.js.map +1 -1
- package/dist/runner/completion-language.d.ts +2 -0
- package/dist/runner/completion-language.d.ts.map +1 -0
- package/dist/runner/completion-language.js +5 -0
- package/dist/runner/completion-language.js.map +1 -0
- package/dist/runner/direct-start.d.ts +19 -0
- package/dist/runner/direct-start.d.ts.map +1 -0
- package/dist/runner/direct-start.js +295 -0
- package/dist/runner/direct-start.js.map +1 -0
- package/dist/runner/events.d.ts +3 -6
- package/dist/runner/events.d.ts.map +1 -1
- package/dist/runner/events.js.map +1 -1
- package/dist/runner/fan-out.d.ts +6 -9
- package/dist/runner/fan-out.d.ts.map +1 -1
- package/dist/runner/fan-out.js +7 -15
- package/dist/runner/fan-out.js.map +1 -1
- package/dist/runner/goal-decomposer.d.ts +0 -1
- package/dist/runner/goal-decomposer.d.ts.map +1 -1
- package/dist/runner/goal-decomposer.js +0 -1
- package/dist/runner/goal-decomposer.js.map +1 -1
- package/dist/runner/google-flights-preflight.d.ts +35 -0
- package/dist/runner/google-flights-preflight.d.ts.map +1 -0
- package/dist/runner/google-flights-preflight.js +411 -0
- package/dist/runner/google-flights-preflight.js.map +1 -0
- package/dist/runner/interrupt-controller.d.ts +1 -1
- package/dist/runner/interrupt-controller.js +1 -1
- package/dist/runner/overlay-narration.d.ts +1 -1
- package/dist/runner/overlay-narration.js +1 -1
- package/dist/runner/parallel-runner.d.ts +3 -6
- package/dist/runner/parallel-runner.d.ts.map +1 -1
- package/dist/runner/parallel-runner.js +2 -3
- package/dist/runner/parallel-runner.js.map +1 -1
- package/dist/runner/pattern-extractor.d.ts +2 -2
- package/dist/runner/pattern-extractor.js +2 -2
- package/dist/runner/runner.d.ts +15 -39
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +254 -231
- package/dist/runner/runner.js.map +1 -1
- package/dist/runner/stream-webhook.d.ts +1 -1
- package/dist/runner/stream-webhook.js +1 -1
- package/dist/skills/domain-loader.d.ts +1 -2
- package/dist/skills/domain-loader.d.ts.map +1 -1
- package/dist/skills/domain-loader.js +1 -2
- package/dist/skills/domain-loader.js.map +1 -1
- package/dist/telemetry/schema.d.ts +2 -2
- package/dist/telemetry/schema.js +2 -2
- package/dist/test-runner.d.ts +4 -5
- package/dist/test-runner.d.ts.map +1 -1
- package/dist/test-runner.js +2 -3
- package/dist/test-runner.js.map +1 -1
- package/dist/types.d.ts +28 -21
- package/dist/types.d.ts.map +1 -1
- package/package.json +8 -2
- package/dist/design/audit/v2/build-result.d.ts +0 -47
- package/dist/design/audit/v2/build-result.d.ts.map +0 -1
- package/dist/design/audit/v2/build-result.js +0 -163
- package/dist/design/audit/v2/build-result.js.map +0 -1
- package/dist/design/audit/v2/score.d.ts.map +0 -1
- package/dist/design/audit/v2/score.js.map +0 -1
- package/dist/design/audit/v2/types.d.ts.map +0 -1
- package/dist/design/audit/v2/types.js.map +0 -1
package/dist/brain/index.d.ts
CHANGED
|
@@ -84,6 +84,17 @@ export declare class Brain {
|
|
|
84
84
|
private resolveModelName;
|
|
85
85
|
private shouldSendTemperature;
|
|
86
86
|
private generationOptions;
|
|
87
|
+
/**
|
|
88
|
+
* True iff we're hitting an OpenAI-compatible *proxy* (router.tangle.tools,
|
|
89
|
+
* LiteLLM, etc.) rather than OpenAI direct. Used to downshift to the lowest
|
|
90
|
+
* common denominator API surface — chat-completions, no Responses API,
|
|
91
|
+
* non-streaming — that every OpenAI-compatible proxy is guaranteed to serve.
|
|
92
|
+
*
|
|
93
|
+
* The single source of truth for "we're talking to a proxy, be conservative
|
|
94
|
+
* about API features"; both `createForceNonStreamingFetch()` and the
|
|
95
|
+
* `forceReasoning` gate route through this predicate.
|
|
96
|
+
*/
|
|
97
|
+
private isProxiedOpenAI;
|
|
87
98
|
/** Get a LLM model instance, optionally with provider/model override (e.g. for CAPTCHA fallback) */
|
|
88
99
|
getLanguageModel(selection?: {
|
|
89
100
|
provider?: 'openai' | 'anthropic' | 'google' | 'cli-bridge';
|
|
@@ -203,21 +214,17 @@ export declare class Brain {
|
|
|
203
214
|
forceVision?: boolean;
|
|
204
215
|
}): Promise<BrainDecision>;
|
|
205
216
|
/**
|
|
206
|
-
*
|
|
217
|
+
* Vision-first decision path. The screenshot is the primary
|
|
207
218
|
* observation; DOM snapshot is minimal context (URL, title only in pure
|
|
208
219
|
* vision mode, or compact DOM in hybrid mode). The LLM outputs
|
|
209
220
|
* coordinate-based actions (clickAt, typeAt) in 1024×768 virtual space.
|
|
210
221
|
*/
|
|
211
222
|
private decideVision;
|
|
212
223
|
/**
|
|
213
|
-
*
|
|
224
|
+
* Generate a structured plan for the entire task with one LLM call.
|
|
214
225
|
*
|
|
215
226
|
* The runner executes the plan deterministically (no LLM between steps),
|
|
216
227
|
* falling back to per-action `decide()` only when verification fails.
|
|
217
|
-
* This is the architectural shift that breaks the "1 LLM call per
|
|
218
|
-
* action" assumption — a 9-action task becomes 1 plan call + 9
|
|
219
|
-
* deterministic executes instead of 9 LLM calls.
|
|
220
|
-
*
|
|
221
228
|
* Returns null when:
|
|
222
229
|
* - the LLM response is unparseable JSON (fall through to per-action)
|
|
223
230
|
* - the plan has zero steps
|
|
@@ -268,6 +275,7 @@ export declare class Brain {
|
|
|
268
275
|
raw: string;
|
|
269
276
|
tokensUsed?: number;
|
|
270
277
|
designSystemScore?: Record<string, unknown>;
|
|
278
|
+
parseError?: string;
|
|
271
279
|
}>;
|
|
272
280
|
/**
|
|
273
281
|
* Extract reusable knowledge from a completed trajectory.
|
|
@@ -303,9 +311,8 @@ export declare class Brain {
|
|
|
303
311
|
* like MDN, Python docs, W3C spec where the value the agent needs lives in
|
|
304
312
|
* a `<dl>/<code>/<pre>` block, not in an interactive element)
|
|
305
313
|
*
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
* unchanged because by then the LLM has already seen the full snapshot once.
|
|
314
|
+
* New-page callers use a larger default budget; same-page callers can pass a
|
|
315
|
+
* tighter budget after the model has already seen the full snapshot once.
|
|
309
316
|
*/
|
|
310
317
|
export declare function budgetSnapshot(snapshot: string, maxChars?: number): string;
|
|
311
318
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/brain/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAsB,MAAM,IAAI,CAAC;AAC1E,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,gBAAgB,EAAE,IAAI,EAAY,MAAM,aAAa,CAAC;AA2ZnH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,6EAA6E;IAC7E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAmBD,qBAAa,KAAK;IAChB,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,QAAQ,CAAyH;IACzI,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,oBAAoB,CAAU;IACtC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAC,CAAyH;IAC7I,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,KAAK,CAAU;IACvB,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,eAAe,CAA8B;IACrD,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,aAAa,CAAC,CAAyH;IAC/I,OAAO,CAAC,cAAc,CAAU;IAEhC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAC,CAAS;IAGxC,OAAO,CAAC,cAAc,CAAC,CAAgF;IACvG,OAAO,CAAC,oBAAoB,CAAC,CAA0C;IACvE,sGAAsG;IACtG,OAAO,CAAC,gBAAgB,CAAM;gBAElB,MAAM,GAAE,WAAgB;IAsCpC,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,iBAAiB;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/brain/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAsB,MAAM,IAAI,CAAC;AAC1E,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,gBAAgB,EAAE,IAAI,EAAY,MAAM,aAAa,CAAC;AA2ZnH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,6EAA6E;IAC7E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAmBD,qBAAa,KAAK;IAChB,OAAO,CAAC,UAAU,CAAoC;IACtD,OAAO,CAAC,QAAQ,CAAyH;IACzI,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,oBAAoB,CAAU;IACtC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAC,CAAyH;IAC7I,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,KAAK,CAAU;IACvB,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,eAAe,CAA8B;IACrD,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,cAAc,CAAC,CAAS;IAChC,OAAO,CAAC,aAAa,CAAC,CAAyH;IAC/I,OAAO,CAAC,cAAc,CAAU;IAEhC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,kBAAkB,CAAC,CAAS;IACpC,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAC,CAAS;IAGxC,OAAO,CAAC,cAAc,CAAC,CAAgF;IACvG,OAAO,CAAC,oBAAoB,CAAC,CAA0C;IACvE,sGAAsG;IACtG,OAAO,CAAC,gBAAgB,CAAM;gBAElB,MAAM,GAAE,WAAgB;IAsCpC,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,qBAAqB;IAK7B,OAAO,CAAC,iBAAiB;IAuCzB;;;;;;;;;OASG;IACH,OAAO,CAAC,eAAe;IAIvB,oGAAoG;IAC9F,gBAAgB,CAAC,SAAS,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,YAAY,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,CAAC;IAI3I,oEAAoE;YACtD,QAAQ;YA6KR,QAAQ;IAuHtB;;;;;;;;;OASG;IACH,OAAO,CAAC,wBAAwB;IAehC;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;;;;;;;OASG;IACH,OAAO,CAAC,wBAAwB;IA4ChC;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;;OAGG;IACH,iBAAiB,CACf,YAAY,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAC5F,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GACpD,IAAI;IAKP;6EACyE;IACzE,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAIxC;;;;;;;;;;OAUG;IACH,OAAO,CAAC,oBAAoB;IA0B5B,0DAA0D;IAC1D,KAAK,IAAI,IAAI;IAIb;;;;;;;;;;;;OAYG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAwB7B,uCAAuC;IACvC,UAAU,IAAI,YAAY,EAAE;IAI5B,0DAA0D;IAC1D,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAItC;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAmBxB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAoBpC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAgBnB,MAAM,CACV,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,EAChB,YAAY,CAAC,EAAE,MAAM,EACrB,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAC3C,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,GAClC,OAAO,CAAC,aAAa,CAAC;IAoMzB;;;;;OAKG;YACW,YAAY;IAoI1B;;;;;;;;;;;;OAYG;IACG,IAAI,CACR,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,EAChB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GACrD,OAAO,CAAC;QACT,IAAI,EAAE,IAAI,GAAG,IAAI,CAAA;QACjB,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,EAAE,MAAM,CAAA;QAClB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,oBAAoB,CAAC,EAAE,MAAM,CAAA;QAC7B,wBAAwB,CAAC,EAAE,MAAM,CAAA;QACjC,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IA4OF;;;;OAIG;IACG,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAsDpE,sBAAsB,CAC1B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,SAAS,EAChB,UAAU,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,EAC/D,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,uBAAuB,CAAC;IAsEnC;;;;OAIG;IACG,oBAAoB,CACxB,KAAK,EAAE,SAAS,EAChB,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,gBAAgB,CAAC;IAkE5B;;;;OAIG;IACG,WAAW,CACf,KAAK,EAAE,SAAS,EAChB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE,EACrB,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,aAAa,EAAE,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAsG5J;;;;OAIG;IACG,gBAAgB,CACpB,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,QAAQ,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAqDpG;;;;;;;OAOG;IACG,QAAQ,CACZ,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAA;KAAO,GACzC,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAUjD,OAAO,CAAC,KAAK;CAqFd;AAsHD;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,SAAS,GAAG,MAAM,CAgE1E"}
|
package/dist/brain/index.js
CHANGED
|
@@ -79,13 +79,13 @@ const DATA_EXTRACTION_RULES = `
|
|
|
79
79
|
21. EFFICIENT COMPLETION: When you have enough data to answer the goal, complete immediately. Do not navigate to additional pages for "confirmation" if the data was already extracted or is visible in the current a11y tree. Include all extracted data in the completion result
|
|
80
80
|
22. EXTRACT BEFORE NAVIGATING: On search results, directory listings, or any page showing multiple items, ALWAYS extract ALL needed data BEFORE clicking into individual items. Use extractWithIndex with a wide query for unknown structure, or runScript with document.querySelectorAll('.result-card') if the structure is well-known. Many sites use anti-bot protection on detail pages but leave listing pages accessible. If you can answer the goal from list-level data, do so without navigating deeper.
|
|
81
81
|
23. FILTER vs SEARCH: When a goal asks to filter results (e.g., "under $50", "4+ stars"), look for filter controls (sliders, dropdowns, checkboxes in a sidebar or toolbar) rather than typing filter values into the search box. Search boxes are for keyword queries, not numeric filters. After applying a filter: (1) wait 2-3 seconds for results to update, (2) verify the filter took effect by checking the updated results, (3) extract the filtered data. Do NOT keep searching for more filter controls after one is applied — extract and complete
|
|
82
|
-
25. EXTRACTWITHINDEX RECOVERY: If a previous runScript returned null/empty/{x:null} on an extraction task, the selector was wrong. DO NOT retry the same runScript or guess a similar selector — the LLM cannot guess CSS class names that aren't visible in the snapshot. Switch to extractWithIndex with a WIDE query: \`'p, span, dd, code, strong, em'\` plus a \`contains\` filter naming the expected text fragment (e.g. contains: "downloads" for npm download counts, contains: "callbackFn" for MDN method signatures). The response shows you the actual text per element so you can pick by content match.
|
|
82
|
+
25. EXTRACTWITHINDEX RECOVERY: If a previous runScript returned null/empty/{x:null} on an extraction task, the selector was wrong. DO NOT retry the same runScript or guess a similar selector — the LLM cannot guess CSS class names that aren't visible in the snapshot. Switch to extractWithIndex with a WIDE query: \`'p, span, dd, code, strong, em'\` plus a \`contains\` filter naming the expected text fragment (e.g. contains: "downloads" for npm download counts, contains: "callbackFn" for MDN method signatures). The response shows you the actual text per element so you can pick by content match. Pick-by-content beats pick-by-selector on pages where the planner couldn't see the data at plan time.`;
|
|
83
83
|
/** Heavy page rules (19-20, 24): injected when snapshot is large or turn count is high */
|
|
84
84
|
const HEAVY_PAGE_RULES = `
|
|
85
85
|
19. FORM FIELD TARGETING: Before typing, verify you are targeting the correct input field using its @ref from the ELEMENTS list. If multiple inputs are visible (e.g., search box + price filter), ensure you select the right one by checking its label or placeholder text in the a11y tree. Never assume focus — always specify the exact @ref
|
|
86
86
|
20. SECTION NAVIGATION: When you need to find a specific section (e.g., rugby, sports, travel) and the nav links aren't in the truncated a11y tree, use runScript to discover navigation: JSON.stringify(Array.from(document.querySelectorAll('nav a, header a, [role="navigation"] a, .nav a')).slice(0, 30).map(a => ({text: a.textContent.trim(), href: a.href}))). Then navigate directly to the matching section URL
|
|
87
87
|
24. HEAVY PAGE RECOVERY: If a page takes very long to load or seems stuck, do NOT wait — use runScript to check document.readyState and extract whatever content is already in the DOM. Partial data is better than a timeout. If the page is completely blank, try navigating to a simpler version (mobile site, search page) instead of waiting`;
|
|
88
|
-
/**
|
|
88
|
+
/** URL-first navigation rules for complex forms and search pages.
|
|
89
89
|
* Teaches the agent to construct search/results URLs from goal text
|
|
90
90
|
* instead of fighting form UIs. Works on any site with URL parameters. */
|
|
91
91
|
const URL_FIRST_RULES = `
|
|
@@ -139,7 +139,7 @@ EXAMPLE 3 — Batch fill a multi-field form (one turn instead of ten):
|
|
|
139
139
|
{"plan":["Fill all visible Personal Info fields","Click Next","Fill Contact step","Submit"],"currentStep":0,"action":{"action":"fill","fields":{"@firstname":"Jordan","@lastname":"Rivera","@dob":"1990-04-15"},"selects":{"@gender":"other"}},"reasoning":"Step 1 of the form has 3 text fields and 1 select all visible at once. Filling them in a single batch action saves 7 turns vs typing each individually.","expectedEffect":"All four Step 1 fields populated with the supplied values"}`;
|
|
140
140
|
/** Full static prompt (all rules) — used as default when config.systemPrompt is not set */
|
|
141
141
|
const SYSTEM_PROMPT = CORE_RULES + SEARCH_RULES + DATA_EXTRACTION_RULES + HEAVY_PAGE_RULES + REASONING_SUFFIX;
|
|
142
|
-
//
|
|
142
|
+
// Vision-first system prompt for pure coordinate actions.
|
|
143
143
|
const VISION_FIRST_PROMPT = `You are a browser automation agent. You operate by looking at screenshots and clicking on elements using pixel coordinates.
|
|
144
144
|
|
|
145
145
|
The screenshot shows the current page state at 1024×768 resolution. You identify elements visually and specify where to click using (x, y) coordinates in this coordinate space.
|
|
@@ -239,7 +239,7 @@ REASONING FRAMEWORK:
|
|
|
239
239
|
2. Where is the element I need to interact with? Estimate its (x, y) coordinates.
|
|
240
240
|
3. What is the smallest action that makes progress toward the goal?
|
|
241
241
|
4. If my last action failed, WHY did it fail? Try a different location or strategy.`;
|
|
242
|
-
//
|
|
242
|
+
// Unified vision+DOM prompt. The model sees BOTH the screenshot AND
|
|
243
243
|
// the ARIA snapshot with @refs. It can use EITHER coordinate actions (clickAt/
|
|
244
244
|
// typeAt for visual targets) OR ref actions (click/type/fill for DOM elements).
|
|
245
245
|
// This lets it pick the best tool per interaction: vision for visual layout,
|
|
@@ -427,7 +427,7 @@ export class Brain {
|
|
|
427
427
|
scoutModelName;
|
|
428
428
|
scoutProvider;
|
|
429
429
|
scoutUseVision;
|
|
430
|
-
//
|
|
430
|
+
// Per-role model overrides.
|
|
431
431
|
plannerModel;
|
|
432
432
|
plannerProvider;
|
|
433
433
|
verifierModel;
|
|
@@ -466,7 +466,7 @@ export class Brain {
|
|
|
466
466
|
this.scoutModelName = config.scout?.model;
|
|
467
467
|
this.scoutProvider = config.scout?.provider;
|
|
468
468
|
this.scoutUseVision = config.scout?.useVision === true;
|
|
469
|
-
//
|
|
469
|
+
// Per-role model overrides.
|
|
470
470
|
this.plannerModel = config.models?.planner?.model;
|
|
471
471
|
this.plannerProvider = config.models?.planner?.provider;
|
|
472
472
|
this.verifierModel = config.models?.verifier?.model;
|
|
@@ -503,7 +503,15 @@ export class Brain {
|
|
|
503
503
|
return {
|
|
504
504
|
...(this.shouldSendTemperature(modelName) ? { temperature: 0 } : {}),
|
|
505
505
|
...(isCliSpawning || omitsLegacyMaxTokens ? {} : { maxOutputTokens }),
|
|
506
|
-
|
|
506
|
+
// forceReasoning routes the AI SDK to OpenAI's Responses API
|
|
507
|
+
// (`/v1/responses`). Most third-party OpenAI-compatible proxies
|
|
508
|
+
// (router.tangle.tools, LiteLLM, Together, vLLM, etc.) only implement
|
|
509
|
+
// /v1/chat/completions — Responses API requests come back 503 / 4xx
|
|
510
|
+
// and the SDK throws "Invalid JSON response". Disable on proxied
|
|
511
|
+
// openai routes; only OpenAI direct supports the Responses API today.
|
|
512
|
+
...(providerName === 'openai'
|
|
513
|
+
&& /(^|\/)gpt-5(?:[.-]|$)/i.test(modelName)
|
|
514
|
+
&& !this.isProxiedOpenAI(providerName)
|
|
507
515
|
? {
|
|
508
516
|
providerOptions: {
|
|
509
517
|
openai: {
|
|
@@ -515,6 +523,19 @@ export class Brain {
|
|
|
515
523
|
: {}),
|
|
516
524
|
};
|
|
517
525
|
}
|
|
526
|
+
/**
|
|
527
|
+
* True iff we're hitting an OpenAI-compatible *proxy* (router.tangle.tools,
|
|
528
|
+
* LiteLLM, etc.) rather than OpenAI direct. Used to downshift to the lowest
|
|
529
|
+
* common denominator API surface — chat-completions, no Responses API,
|
|
530
|
+
* non-streaming — that every OpenAI-compatible proxy is guaranteed to serve.
|
|
531
|
+
*
|
|
532
|
+
* The single source of truth for "we're talking to a proxy, be conservative
|
|
533
|
+
* about API features"; both `createForceNonStreamingFetch()` and the
|
|
534
|
+
* `forceReasoning` gate route through this predicate.
|
|
535
|
+
*/
|
|
536
|
+
isProxiedOpenAI(providerName) {
|
|
537
|
+
return providerName === 'openai' && Boolean(this.baseUrl);
|
|
538
|
+
}
|
|
518
539
|
/** Get a LLM model instance, optionally with provider/model override (e.g. for CAPTCHA fallback) */
|
|
519
540
|
async getLanguageModel(selection) {
|
|
520
541
|
return this.getModel(selection);
|
|
@@ -673,16 +694,18 @@ export class Brain {
|
|
|
673
694
|
// chat factory explicitly so we hit a path the upstream actually
|
|
674
695
|
// serves.
|
|
675
696
|
const { createOpenAI } = await import('@ai-sdk/openai');
|
|
697
|
+
const usingProxy = this.isProxiedOpenAI(providerName);
|
|
676
698
|
const provider = createOpenAI({
|
|
677
699
|
apiKey: apiKey || '',
|
|
678
700
|
...(this.baseUrl ? { baseURL: this.baseUrl } : {}),
|
|
679
|
-
//
|
|
680
|
-
// default to SSE
|
|
681
|
-
//
|
|
682
|
-
// "Invalid JSON response" on SSE. Force
|
|
683
|
-
// chat-completions body
|
|
684
|
-
//
|
|
685
|
-
|
|
701
|
+
// Proxy downshift — see isProxiedOpenAI(). Routes like
|
|
702
|
+
// router.tangle.tools default chat-completions to SSE when the
|
|
703
|
+
// client omits `stream`, and the AI SDK's generateText errors with
|
|
704
|
+
// "Invalid JSON response" on SSE. Force stream: false on every
|
|
705
|
+
// chat-completions body. Paired with the forceReasoning gate in
|
|
706
|
+
// generationOptions() so all "talk to a proxy" downshifts share
|
|
707
|
+
// one predicate.
|
|
708
|
+
...(usingProxy ? { fetch: createForceNonStreamingFetch() } : {}),
|
|
686
709
|
});
|
|
687
710
|
model = provider.chat(modelName);
|
|
688
711
|
break;
|
|
@@ -808,9 +831,8 @@ export class Brain {
|
|
|
808
831
|
shouldUseNavigationModel(state, extraContext, turnInfo) {
|
|
809
832
|
if (!this.adaptiveModelRouting || !this.navModelName)
|
|
810
833
|
return false;
|
|
811
|
-
//
|
|
812
|
-
//
|
|
813
|
-
// pages, and error recovery where reasoning matters.
|
|
834
|
+
// Use the navigation model for DOM-only same-page turns. Keep the primary
|
|
835
|
+
// model for first turns, new pages, and error recovery.
|
|
814
836
|
const isFirstTurn = !turnInfo || turnInfo.current <= 1;
|
|
815
837
|
const samePageAsPrevious = this.lastDecisionUrl === state.url;
|
|
816
838
|
const hasError = extraContext?.includes('REJECTED') || extraContext?.includes('ERROR');
|
|
@@ -1039,8 +1061,7 @@ export class Brain {
|
|
|
1039
1061
|
// Zone 1 (intact): last 2 turns — full content
|
|
1040
1062
|
// Zone 2 (standard): turns 3-5 back — ELEMENTS stripped from user msgs
|
|
1041
1063
|
// Zone 3 (deep compact): turns 6+ back — both user and assistant ultra-compacted
|
|
1042
|
-
//
|
|
1043
|
-
// Google Flights to lose essential history on 30-turn runs.
|
|
1064
|
+
// Keep enough older history for long multi-step travel workflows.
|
|
1044
1065
|
const deepCompactBefore = Math.max(0, this.history.length - 10);
|
|
1045
1066
|
return this.history.map((msg, idx) => {
|
|
1046
1067
|
// Zone 1: keep recent turns intact
|
|
@@ -1158,7 +1179,7 @@ export class Brain {
|
|
|
1158
1179
|
});
|
|
1159
1180
|
}
|
|
1160
1181
|
async decide(goal, state, extraContext, turnInfo, options) {
|
|
1161
|
-
//
|
|
1182
|
+
// Vision-first and hybrid modes delegate to the vision path.
|
|
1162
1183
|
if (this.observationMode === 'vision' || this.observationMode === 'hybrid') {
|
|
1163
1184
|
return this.decideVision(goal, state, extraContext, turnInfo);
|
|
1164
1185
|
}
|
|
@@ -1177,10 +1198,8 @@ export class Brain {
|
|
|
1177
1198
|
&& diffChanges > 0
|
|
1178
1199
|
&& diffTotal > 0
|
|
1179
1200
|
&& diffChanges / diffTotal < 0.3;
|
|
1180
|
-
// Tighter snapshot budget on same-page turns
|
|
1181
|
-
//
|
|
1182
|
-
// docs/W3C spec) get the full <dl>/<code>/<pre> content the LLM needs to write
|
|
1183
|
-
// a working runScript on the first try.
|
|
1201
|
+
// Tighter snapshot budget on same-page turns; new pages keep enough
|
|
1202
|
+
// content for extraction from docs/spec pages.
|
|
1184
1203
|
const snapshotBudget = samePageAsPrevious ? 8_000 : 24_000;
|
|
1185
1204
|
let visibleSnapshot;
|
|
1186
1205
|
let elementsHeader;
|
|
@@ -1338,17 +1357,15 @@ ${visibleSnapshot}`;
|
|
|
1338
1357
|
};
|
|
1339
1358
|
}
|
|
1340
1359
|
/**
|
|
1341
|
-
*
|
|
1360
|
+
* Vision-first decision path. The screenshot is the primary
|
|
1342
1361
|
* observation; DOM snapshot is minimal context (URL, title only in pure
|
|
1343
1362
|
* vision mode, or compact DOM in hybrid mode). The LLM outputs
|
|
1344
1363
|
* coordinate-based actions (clickAt, typeAt) in 1024×768 virtual space.
|
|
1345
1364
|
*/
|
|
1346
1365
|
async decideVision(goal, state, extraContext, turnInfo) {
|
|
1347
1366
|
this.lastDecisionUrl = state.url;
|
|
1348
|
-
//
|
|
1349
|
-
//
|
|
1350
|
-
// expanded, content loaded), the DIFF is the signal. Send only what
|
|
1351
|
-
// changed instead of the full snapshot. Saves 3-5k tokens per turn.
|
|
1367
|
+
// Adaptive observation: on same-page hybrid turns, send only changed
|
|
1368
|
+
// elements when the diff is small.
|
|
1352
1369
|
const isHybrid = this.observationMode === 'hybrid';
|
|
1353
1370
|
const samePageAsPrevious = this.lastDecisionUrl === state.url;
|
|
1354
1371
|
const isFirstTurn = !turnInfo || turnInfo.current <= 1;
|
|
@@ -1413,10 +1430,8 @@ Title: ${state.title}`;
|
|
|
1413
1430
|
{ type: 'text', text: textContent },
|
|
1414
1431
|
{ type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
|
|
1415
1432
|
];
|
|
1416
|
-
//
|
|
1417
|
-
//
|
|
1418
|
-
// dead weight that pushes cumulative token count past the cost cap.
|
|
1419
|
-
// This alone fixes 3/5 Gen 13 failures (cost_cap at 101-106k).
|
|
1433
|
+
// Strip old screenshots from history; the current screenshot is the only
|
|
1434
|
+
// image the model needs for this turn.
|
|
1420
1435
|
const compacted = this.compactHistory().map((msg) => {
|
|
1421
1436
|
if (msg.role !== 'user' || !Array.isArray(msg.content))
|
|
1422
1437
|
return msg;
|
|
@@ -1430,13 +1445,12 @@ Title: ${state.title}`;
|
|
|
1430
1445
|
...compacted,
|
|
1431
1446
|
{ role: 'user', content: userContent },
|
|
1432
1447
|
];
|
|
1433
|
-
//
|
|
1434
|
-
//
|
|
1435
|
-
// differently, hitting cost cap in 3 turns. Vision turns stay on main model.
|
|
1448
|
+
// Vision turns stay on the main model because smaller model routes can
|
|
1449
|
+
// count image tokens differently and exhaust the token budget.
|
|
1436
1450
|
const modelOpts = { provider: this.provider, model: this.modelName };
|
|
1437
1451
|
const nearingEnd = turnInfo && turnInfo.current >= turnInfo.max - 3;
|
|
1438
1452
|
const maxTokens = nearingEnd ? 1200 : 600;
|
|
1439
|
-
//
|
|
1453
|
+
// Hybrid mode uses the unified prompt with both action vocabularies.
|
|
1440
1454
|
const systemPrompt = isHybrid ? UNIFIED_VISION_DOM_PROMPT : VISION_FIRST_PROMPT;
|
|
1441
1455
|
const result = await this.generate(systemPrompt, messages, modelOpts, maxTokens);
|
|
1442
1456
|
const raw = result.text;
|
|
@@ -1465,14 +1479,10 @@ Title: ${state.title}`;
|
|
|
1465
1479
|
};
|
|
1466
1480
|
}
|
|
1467
1481
|
/**
|
|
1468
|
-
*
|
|
1482
|
+
* Generate a structured plan for the entire task with one LLM call.
|
|
1469
1483
|
*
|
|
1470
1484
|
* The runner executes the plan deterministically (no LLM between steps),
|
|
1471
1485
|
* falling back to per-action `decide()` only when verification fails.
|
|
1472
|
-
* This is the architectural shift that breaks the "1 LLM call per
|
|
1473
|
-
* action" assumption — a 9-action task becomes 1 plan call + 9
|
|
1474
|
-
* deterministic executes instead of 9 LLM calls.
|
|
1475
|
-
*
|
|
1476
1486
|
* Returns null when:
|
|
1477
1487
|
* - the LLM response is unparseable JSON (fall through to per-action)
|
|
1478
1488
|
* - the plan has zero steps
|
|
@@ -1485,11 +1495,8 @@ Title: ${state.title}`;
|
|
|
1485
1495
|
const startedAt = Date.now();
|
|
1486
1496
|
const maxSteps = options?.maxSteps ?? 12;
|
|
1487
1497
|
const extraContext = options?.extraContext;
|
|
1488
|
-
//
|
|
1489
|
-
//
|
|
1490
|
-
// writes the runScript that runs on the first observation. Without enough
|
|
1491
|
-
// snapshot context (especially `<dl>/<dt>/<code>/<pre>` content lines that
|
|
1492
|
-
// Gen 10C now preserves), the planner emits selectors that don't exist.
|
|
1498
|
+
// Planner snapshots keep enough context for extraction tasks, especially
|
|
1499
|
+
// docs/spec pages with data in `<dl>`, `<code>`, and `<pre>` blocks.
|
|
1493
1500
|
const snapshot = budgetSnapshot(state.snapshot, 24_000);
|
|
1494
1501
|
const planSystemPrompt = `You are a planning engine for a browser automation agent.
|
|
1495
1502
|
|
|
@@ -1560,10 +1567,8 @@ ELEMENTS:
|
|
|
1560
1567
|
${snapshot}
|
|
1561
1568
|
${extraContext ? `\n${extraContext}\n` : ''}
|
|
1562
1569
|
What is the complete plan?`;
|
|
1563
|
-
//
|
|
1564
|
-
//
|
|
1565
|
-
// the planner can see the visual layout. This helps on sites like
|
|
1566
|
-
// Google Flights where the DOM doesn't convey form structure well.
|
|
1570
|
+
// In vision-capable modes, include the screenshot so the planner can use
|
|
1571
|
+
// visual layout when the DOM does not capture form structure.
|
|
1567
1572
|
const isVisionPlanner = (this.observationMode === 'hybrid' || this.observationMode === 'vision') && !!state.screenshot;
|
|
1568
1573
|
const userContent = isVisionPlanner
|
|
1569
1574
|
? [
|
|
@@ -1571,7 +1576,7 @@ What is the complete plan?`;
|
|
|
1571
1576
|
{ type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
|
|
1572
1577
|
]
|
|
1573
1578
|
: userText;
|
|
1574
|
-
//
|
|
1579
|
+
// Planner can use its own model override.
|
|
1575
1580
|
const planModelOpts = this.plannerModel
|
|
1576
1581
|
? { provider: (this.plannerProvider || this.provider), model: this.plannerModel }
|
|
1577
1582
|
: { provider: this.provider, model: this.modelName };
|
|
@@ -1831,7 +1836,7 @@ ${budgetSnapshot(state.snapshot)}${siteBoundaryNote ? `\n\n${siteBoundaryNote}`
|
|
|
1831
1836
|
|
|
1832
1837
|
Was the goal actually achieved? Analyze the current page state carefully.`;
|
|
1833
1838
|
const userContent = this.buildUserContent(textContent, state.screenshot, true);
|
|
1834
|
-
//
|
|
1839
|
+
// Verifier can use its own model, then the navigation model, then main.
|
|
1835
1840
|
const verifyProvider = this.verifierProvider
|
|
1836
1841
|
? this.verifierProvider
|
|
1837
1842
|
: (this.adaptiveModelRouting && this.navModelName ? (this.navProvider || this.provider) : undefined);
|
|
@@ -1930,12 +1935,16 @@ Audit this page for design quality, UX issues, and visual bugs.`;
|
|
|
1930
1935
|
suggestion: String(f.suggestion ?? ''),
|
|
1931
1936
|
...(f.cssSelector ? { cssSelector: String(f.cssSelector) } : {}),
|
|
1932
1937
|
...(f.cssFix ? { cssFix: String(f.cssFix) } : {}),
|
|
1933
|
-
//
|
|
1938
|
+
// Optional ROI fields.
|
|
1934
1939
|
...(clampScore(f.impact) !== undefined ? { impact: clampScore(f.impact) } : {}),
|
|
1935
1940
|
...(clampScore(f.effort) !== undefined ? { effort: clampScore(f.effort) } : {}),
|
|
1936
1941
|
...(VALID_BLAST.has(f.blast)
|
|
1937
1942
|
? { blast: f.blast }
|
|
1938
1943
|
: {}),
|
|
1944
|
+
// Layer 2 — preserve raw patches array (untyped passthrough). The
|
|
1945
|
+
// parsePatches/validatePatch pipeline in build-result.ts converts
|
|
1946
|
+
// these into typed, validated Patch objects.
|
|
1947
|
+
...(Array.isArray(f.patches) ? { rawPatches: f.patches } : {}),
|
|
1939
1948
|
}))
|
|
1940
1949
|
: [];
|
|
1941
1950
|
const designSystemScore = parsed.designSystemScore && typeof parsed.designSystemScore === 'object'
|
|
@@ -1950,12 +1959,13 @@ Audit this page for design quality, UX issues, and visual bugs.`;
|
|
|
1950
1959
|
designSystemScore,
|
|
1951
1960
|
};
|
|
1952
1961
|
}
|
|
1953
|
-
catch {
|
|
1962
|
+
catch (err) {
|
|
1954
1963
|
return {
|
|
1955
1964
|
score: 5,
|
|
1956
1965
|
findings: [],
|
|
1957
1966
|
raw,
|
|
1958
1967
|
tokensUsed,
|
|
1968
|
+
parseError: err instanceof Error ? err.message : String(err),
|
|
1959
1969
|
};
|
|
1960
1970
|
}
|
|
1961
1971
|
}
|
|
@@ -2035,11 +2045,9 @@ Only include facts that are genuinely useful. Quality over quantity. Max 10 fact
|
|
|
2035
2045
|
'fill', 'clickSequence',
|
|
2036
2046
|
'clickAt', 'typeAt',
|
|
2037
2047
|
'clickLabel', 'typeLabel',
|
|
2038
|
-
//
|
|
2039
|
-
// execute time; here we just accept the shape.
|
|
2048
|
+
// Macro dispatch. The driver validates the macro name at execute time.
|
|
2040
2049
|
'macro',
|
|
2041
|
-
//
|
|
2042
|
-
// only checks shape (subGoals[] or baseUrl+goalTemplate+items).
|
|
2050
|
+
// Parallel fan-out. Runner handles dispatch; validator checks shape.
|
|
2043
2051
|
'fanOut',
|
|
2044
2052
|
]);
|
|
2045
2053
|
// Parse strategy: exact → first-{/last-} extraction.
|
|
@@ -2223,9 +2231,8 @@ function deduplicateSnapshot(snapshot) {
|
|
|
2223
2231
|
* like MDN, Python docs, W3C spec where the value the agent needs lives in
|
|
2224
2232
|
* a `<dl>/<code>/<pre>` block, not in an interactive element)
|
|
2225
2233
|
*
|
|
2226
|
-
*
|
|
2227
|
-
*
|
|
2228
|
-
* unchanged because by then the LLM has already seen the full snapshot once.
|
|
2234
|
+
* New-page callers use a larger default budget; same-page callers can pass a
|
|
2235
|
+
* tighter budget after the model has already seen the full snapshot once.
|
|
2229
2236
|
*/
|
|
2230
2237
|
export function budgetSnapshot(snapshot, maxChars = 24_000) {
|
|
2231
2238
|
// Skip dedup on small snapshots — not enough repetition to justify the O(n) scan
|
|
@@ -2409,19 +2416,18 @@ function validateAction(actionType, data) {
|
|
|
2409
2416
|
...(typeof data.intervalMs === 'number' ? { intervalMs: data.intervalMs } : {}),
|
|
2410
2417
|
};
|
|
2411
2418
|
}
|
|
2412
|
-
//
|
|
2419
|
+
// Vision-first coordinate actions.
|
|
2413
2420
|
case 'clickAt':
|
|
2414
2421
|
return { action: 'clickAt', x: num(data.x, 0), y: num(data.y, 0) };
|
|
2415
2422
|
case 'typeAt':
|
|
2416
2423
|
return { action: 'typeAt', x: num(data.x, 0), y: num(data.y, 0), text: optStr('text') };
|
|
2417
|
-
//
|
|
2424
|
+
// Set-of-Marks label-based actions.
|
|
2418
2425
|
case 'clickLabel':
|
|
2419
2426
|
return { action: 'clickLabel', label: num(data.label, 0) };
|
|
2420
2427
|
case 'typeLabel':
|
|
2421
2428
|
return { action: 'typeLabel', label: num(data.label, 0), text: optStr('text') };
|
|
2422
|
-
//
|
|
2423
|
-
//
|
|
2424
|
-
// legitimately be omitted for macros that declare no params.
|
|
2429
|
+
// Macro invocation. The driver validates the name and required args at
|
|
2430
|
+
// execute time; macros without params may omit args.
|
|
2425
2431
|
case 'macro': {
|
|
2426
2432
|
const args = {};
|
|
2427
2433
|
if (data.args && typeof data.args === 'object' && !Array.isArray(data.args)) {
|
|
@@ -2483,11 +2489,8 @@ function isStringRecord(value) {
|
|
|
2483
2489
|
return true;
|
|
2484
2490
|
}
|
|
2485
2491
|
/**
|
|
2486
|
-
*
|
|
2487
|
-
*
|
|
2488
|
-
* default to SSE streaming when the client omits the field (observed on
|
|
2489
|
-
* router.tangle.tools). The AI SDK's generateText path does not handle
|
|
2490
|
-
* SSE, and we can't rely on the gateway respecting the absence of `stream`.
|
|
2492
|
+
* Build a fetch replacement that forces `"stream": false` on chat completions
|
|
2493
|
+
* bodies for OpenAI-compatible gateways that default to SSE streaming.
|
|
2491
2494
|
*/
|
|
2492
2495
|
function createForceNonStreamingFetch() {
|
|
2493
2496
|
return async (input, init) => {
|