@tangle-network/browser-agent-driver 0.33.3 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -1
- package/discovery/llms.txt +27 -0
- package/discovery/tangle-agent.json +36 -0
- package/dist/brain/action-parse.d.ts +16 -0
- package/dist/brain/action-parse.d.ts.map +1 -0
- package/dist/brain/action-parse.js +191 -0
- package/dist/brain/action-parse.js.map +1 -0
- package/dist/brain/decide.d.ts +58 -0
- package/dist/brain/decide.d.ts.map +1 -0
- package/dist/brain/decide.js +418 -0
- package/dist/brain/decide.js.map +1 -0
- package/dist/brain/history-compact.d.ts +19 -0
- package/dist/brain/history-compact.d.ts.map +1 -0
- package/dist/brain/history-compact.js +153 -0
- package/dist/brain/history-compact.js.map +1 -0
- package/dist/brain/index.d.ts +78 -152
- package/dist/brain/index.d.ts.map +1 -1
- package/dist/brain/index.js +62 -2268
- package/dist/brain/index.js.map +1 -1
- package/dist/brain/model-client.d.ts +101 -0
- package/dist/brain/model-client.d.ts.map +1 -0
- package/dist/brain/model-client.js +391 -0
- package/dist/brain/model-client.js.map +1 -0
- package/dist/brain/plan.d.ts +57 -0
- package/dist/brain/plan.d.ts.map +1 -0
- package/dist/brain/plan.js +194 -0
- package/dist/brain/plan.js.map +1 -0
- package/dist/brain/prompts.d.ts +37 -0
- package/dist/brain/prompts.d.ts.map +1 -0
- package/dist/brain/prompts.js +455 -0
- package/dist/brain/prompts.js.map +1 -0
- package/dist/brain/provider-fetch.d.ts +26 -0
- package/dist/brain/provider-fetch.d.ts.map +1 -0
- package/dist/brain/provider-fetch.js +45 -0
- package/dist/brain/provider-fetch.js.map +1 -0
- package/dist/brain/snapshot-budget.d.ts +20 -0
- package/dist/brain/snapshot-budget.d.ts.map +1 -0
- package/dist/brain/snapshot-budget.js +202 -0
- package/dist/brain/snapshot-budget.js.map +1 -0
- package/dist/brain/system-prompt.d.ts +67 -0
- package/dist/brain/system-prompt.d.ts.map +1 -0
- package/dist/brain/system-prompt.js +128 -0
- package/dist/brain/system-prompt.js.map +1 -0
- package/dist/brain/tasks/design-audit.d.ts +33 -0
- package/dist/brain/tasks/design-audit.d.ts.map +1 -0
- package/dist/brain/tasks/design-audit.js +100 -0
- package/dist/brain/tasks/design-audit.js.map +1 -0
- package/dist/brain/tasks/evaluate.d.ts +25 -0
- package/dist/brain/tasks/evaluate.d.ts.map +1 -0
- package/dist/brain/tasks/evaluate.js +55 -0
- package/dist/brain/tasks/evaluate.js.map +1 -0
- package/dist/brain/tasks/goal-verification.d.ts +32 -0
- package/dist/brain/tasks/goal-verification.d.ts.map +1 -0
- package/dist/brain/tasks/goal-verification.js +68 -0
- package/dist/brain/tasks/goal-verification.js.map +1 -0
- package/dist/brain/tasks/knowledge.d.ts +26 -0
- package/dist/brain/tasks/knowledge.d.ts.map +1 -0
- package/dist/brain/tasks/knowledge.js +56 -0
- package/dist/brain/tasks/knowledge.js.map +1 -0
- package/dist/brain/tasks/link-scout.d.ts +37 -0
- package/dist/brain/tasks/link-scout.d.ts.map +1 -0
- package/dist/brain/tasks/link-scout.js +69 -0
- package/dist/brain/tasks/link-scout.js.map +1 -0
- package/dist/brain/types.d.ts +55 -0
- package/dist/brain/types.d.ts.map +1 -0
- package/dist/brain/types.js +7 -0
- package/dist/brain/types.js.map +1 -0
- package/dist/browser/stealth-init-script.d.ts +9 -0
- package/dist/browser/stealth-init-script.d.ts.map +1 -0
- package/dist/browser/stealth-init-script.js +128 -0
- package/dist/browser/stealth-init-script.js.map +1 -0
- package/dist/browser/storage-state.d.ts +29 -0
- package/dist/browser/storage-state.d.ts.map +1 -0
- package/dist/browser/storage-state.js +47 -0
- package/dist/browser/storage-state.js.map +1 -0
- package/dist/browser-launch.d.ts +6 -0
- package/dist/browser-launch.d.ts.map +1 -1
- package/dist/browser-launch.js +3 -0
- package/dist/browser-launch.js.map +1 -1
- package/dist/cli/args.d.ts +141 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +167 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli/benchmark-sync.d.ts +9 -0
- package/dist/cli/benchmark-sync.d.ts.map +1 -0
- package/dist/cli/benchmark-sync.js +43 -0
- package/dist/cli/benchmark-sync.js.map +1 -0
- package/dist/cli/build-config.d.ts +11 -0
- package/dist/cli/build-config.d.ts.map +1 -0
- package/dist/cli/build-config.js +355 -0
- package/dist/cli/build-config.js.map +1 -0
- package/dist/cli/commands/auth.d.ts +3 -0
- package/dist/cli/commands/auth.d.ts.map +1 -0
- package/dist/cli/commands/auth.js +36 -0
- package/dist/cli/commands/auth.js.map +1 -0
- package/dist/cli/commands/chrome-debug.d.ts +7 -0
- package/dist/cli/commands/chrome-debug.d.ts.map +1 -0
- package/dist/cli/commands/chrome-debug.js +11 -0
- package/dist/cli/commands/chrome-debug.js.map +1 -0
- package/dist/cli/commands/design-audit.d.ts +3 -0
- package/dist/cli/commands/design-audit.d.ts.map +1 -0
- package/dist/cli/commands/design-audit.js +68 -0
- package/dist/cli/commands/design-audit.js.map +1 -0
- package/dist/cli/commands/preview.d.ts +14 -0
- package/dist/cli/commands/preview.d.ts.map +1 -0
- package/dist/cli/commands/preview.js +31 -0
- package/dist/cli/commands/preview.js.map +1 -0
- package/dist/cli/commands/run.d.ts +3 -0
- package/dist/cli/commands/run.d.ts.map +1 -0
- package/dist/cli/commands/run.js +740 -0
- package/dist/cli/commands/run.js.map +1 -0
- package/dist/cli/commands/runs.d.ts +8 -0
- package/dist/cli/commands/runs.d.ts.map +1 -0
- package/dist/cli/commands/runs.js +35 -0
- package/dist/cli/commands/runs.js.map +1 -0
- package/dist/cli/commands/share.d.ts +10 -0
- package/dist/cli/commands/share.d.ts.map +1 -0
- package/dist/cli/commands/share.js +32 -0
- package/dist/cli/commands/share.js.map +1 -0
- package/dist/cli/commands/showcase.d.ts +17 -0
- package/dist/cli/commands/showcase.d.ts.map +1 -0
- package/dist/cli/commands/showcase.js +20 -0
- package/dist/cli/commands/showcase.js.map +1 -0
- package/dist/cli/commands/snapshot.d.ts +12 -0
- package/dist/cli/commands/snapshot.d.ts.map +1 -0
- package/dist/cli/commands/snapshot.js +24 -0
- package/dist/cli/commands/snapshot.js.map +1 -0
- package/dist/cli/commands/view.d.ts +7 -0
- package/dist/cli/commands/view.d.ts.map +1 -0
- package/dist/cli/commands/view.js +23 -0
- package/dist/cli/commands/view.js.map +1 -0
- package/dist/cli/constants.d.ts +6 -0
- package/dist/cli/constants.d.ts.map +1 -0
- package/dist/cli/constants.js +3 -0
- package/dist/cli/constants.js.map +1 -0
- package/dist/cli/version.d.ts +7 -0
- package/dist/cli/version.d.ts.map +1 -0
- package/dist/cli/version.js +16 -0
- package/dist/cli/version.js.map +1 -0
- package/dist/cli-design-audit.d.ts +44 -36
- package/dist/cli-design-audit.d.ts.map +1 -1
- package/dist/cli-design-audit.js +137 -1646
- package/dist/cli-design-audit.js.map +1 -1
- package/dist/cli-jobs.js +2 -1
- package/dist/cli-jobs.js.map +1 -1
- package/dist/cli.js +66 -1697
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +20 -2
- package/dist/config.js.map +1 -1
- package/dist/design/audit/evolve/agent.d.ts +23 -0
- package/dist/design/audit/evolve/agent.d.ts.map +1 -0
- package/dist/design/audit/evolve/agent.js +273 -0
- package/dist/design/audit/evolve/agent.js.map +1 -0
- package/dist/design/audit/evolve/css.d.ts +10 -0
- package/dist/design/audit/evolve/css.d.ts.map +1 -0
- package/dist/design/audit/evolve/css.js +206 -0
- package/dist/design/audit/evolve/css.js.map +1 -0
- package/dist/design/audit/evolve/index.d.ts +13 -0
- package/dist/design/audit/evolve/index.d.ts.map +1 -0
- package/dist/design/audit/evolve/index.js +12 -0
- package/dist/design/audit/evolve/index.js.map +1 -0
- package/dist/design/audit/evolve/report.d.ts +3 -0
- package/dist/design/audit/evolve/report.d.ts.map +1 -0
- package/dist/design/audit/evolve/report.js +45 -0
- package/dist/design/audit/evolve/report.js.map +1 -0
- package/dist/design/audit/evolve/types.d.ts +14 -0
- package/dist/design/audit/evolve/types.d.ts.map +1 -0
- package/dist/design/audit/evolve/types.js +2 -0
- package/dist/design/audit/evolve/types.js.map +1 -0
- package/dist/design/audit/pipeline.d.ts +12 -0
- package/dist/design/audit/pipeline.d.ts.map +1 -1
- package/dist/design/audit/pipeline.js +43 -27
- package/dist/design/audit/pipeline.js.map +1 -1
- package/dist/design/audit/reference/artifact/build.d.ts +33 -0
- package/dist/design/audit/reference/artifact/build.d.ts.map +1 -0
- package/dist/design/audit/reference/artifact/build.js +64 -0
- package/dist/design/audit/reference/artifact/build.js.map +1 -0
- package/dist/design/audit/reference/artifact/render.d.ts +39 -0
- package/dist/design/audit/reference/artifact/render.d.ts.map +1 -0
- package/dist/design/audit/reference/artifact/render.js +213 -0
- package/dist/design/audit/reference/artifact/render.js.map +1 -0
- package/dist/design/audit/reference/artifact/text.d.ts +11 -0
- package/dist/design/audit/reference/artifact/text.d.ts.map +1 -0
- package/dist/design/audit/reference/artifact/text.js +20 -0
- package/dist/design/audit/reference/artifact/text.js.map +1 -0
- package/dist/design/audit/reference/artifact/to-findings.d.ts +57 -0
- package/dist/design/audit/reference/artifact/to-findings.d.ts.map +1 -0
- package/dist/design/audit/reference/artifact/to-findings.js +314 -0
- package/dist/design/audit/reference/artifact/to-findings.js.map +1 -0
- package/dist/design/audit/reference/config.d.ts +50 -0
- package/dist/design/audit/reference/config.d.ts.map +1 -0
- package/dist/design/audit/reference/config.js +108 -0
- package/dist/design/audit/reference/config.js.map +1 -0
- package/dist/design/audit/reference/contracts.d.ts +1030 -0
- package/dist/design/audit/reference/contracts.d.ts.map +1 -0
- package/dist/design/audit/reference/contracts.js +28 -0
- package/dist/design/audit/reference/contracts.js.map +1 -0
- package/dist/design/audit/reference/corpus/build.d.ts +115 -0
- package/dist/design/audit/reference/corpus/build.d.ts.map +1 -0
- package/dist/design/audit/reference/corpus/build.js +152 -0
- package/dist/design/audit/reference/corpus/build.js.map +1 -0
- package/dist/design/audit/reference/corpus/schema.d.ts +33 -0
- package/dist/design/audit/reference/corpus/schema.d.ts.map +1 -0
- package/dist/design/audit/reference/corpus/schema.js +249 -0
- package/dist/design/audit/reference/corpus/schema.js.map +1 -0
- package/dist/design/audit/reference/corpus/store.d.ts +26 -0
- package/dist/design/audit/reference/corpus/store.d.ts.map +1 -0
- package/dist/design/audit/reference/corpus/store.js +104 -0
- package/dist/design/audit/reference/corpus/store.js.map +1 -0
- package/dist/design/audit/reference/dna/delta.d.ts +21 -0
- package/dist/design/audit/reference/dna/delta.d.ts.map +1 -0
- package/dist/design/audit/reference/dna/delta.js +126 -0
- package/dist/design/audit/reference/dna/delta.js.map +1 -0
- package/dist/design/audit/reference/dna/derive.d.ts +31 -0
- package/dist/design/audit/reference/dna/derive.d.ts.map +1 -0
- package/dist/design/audit/reference/dna/derive.js +381 -0
- package/dist/design/audit/reference/dna/derive.js.map +1 -0
- package/dist/design/audit/reference/dna/descriptor.d.ts +36 -0
- package/dist/design/audit/reference/dna/descriptor.d.ts.map +1 -0
- package/dist/design/audit/reference/dna/descriptor.js +209 -0
- package/dist/design/audit/reference/dna/descriptor.js.map +1 -0
- package/dist/design/audit/reference/dna/page-adapter.d.ts +17 -0
- package/dist/design/audit/reference/dna/page-adapter.d.ts.map +1 -0
- package/dist/design/audit/reference/dna/page-adapter.js +35 -0
- package/dist/design/audit/reference/dna/page-adapter.js.map +1 -0
- package/dist/design/audit/reference/dna/scroll-capture.d.ts +93 -0
- package/dist/design/audit/reference/dna/scroll-capture.d.ts.map +1 -0
- package/dist/design/audit/reference/dna/scroll-capture.js +374 -0
- package/dist/design/audit/reference/dna/scroll-capture.js.map +1 -0
- package/dist/design/audit/reference/engine/budget.d.ts +63 -0
- package/dist/design/audit/reference/engine/budget.d.ts.map +1 -0
- package/dist/design/audit/reference/engine/budget.js +110 -0
- package/dist/design/audit/reference/engine/budget.js.map +1 -0
- package/dist/design/audit/reference/engine/core.d.ts +32 -0
- package/dist/design/audit/reference/engine/core.d.ts.map +1 -0
- package/dist/design/audit/reference/engine/core.js +261 -0
- package/dist/design/audit/reference/engine/core.js.map +1 -0
- package/dist/design/audit/reference/engine/guard.d.ts +43 -0
- package/dist/design/audit/reference/engine/guard.d.ts.map +1 -0
- package/dist/design/audit/reference/engine/guard.js +43 -0
- package/dist/design/audit/reference/engine/guard.js.map +1 -0
- package/dist/design/audit/reference/engine/score-core.d.ts +48 -0
- package/dist/design/audit/reference/engine/score-core.d.ts.map +1 -0
- package/dist/design/audit/reference/engine/score-core.js +134 -0
- package/dist/design/audit/reference/engine/score-core.js.map +1 -0
- package/dist/design/audit/reference/engine/wiring.d.ts +35 -0
- package/dist/design/audit/reference/engine/wiring.d.ts.map +1 -0
- package/dist/design/audit/reference/engine/wiring.js +80 -0
- package/dist/design/audit/reference/engine/wiring.js.map +1 -0
- package/dist/design/audit/reference/eval/taste-core.d.ts +77 -0
- package/dist/design/audit/reference/eval/taste-core.d.ts.map +1 -0
- package/dist/design/audit/reference/eval/taste-core.js +95 -0
- package/dist/design/audit/reference/eval/taste-core.js.map +1 -0
- package/dist/design/audit/reference/generate/generator.d.ts +49 -0
- package/dist/design/audit/reference/generate/generator.d.ts.map +1 -0
- package/dist/design/audit/reference/generate/generator.js +100 -0
- package/dist/design/audit/reference/generate/generator.js.map +1 -0
- package/dist/design/audit/reference/generate/parse.d.ts +28 -0
- package/dist/design/audit/reference/generate/parse.d.ts.map +1 -0
- package/dist/design/audit/reference/generate/parse.js +220 -0
- package/dist/design/audit/reference/generate/parse.js.map +1 -0
- package/dist/design/audit/reference/generate/prompt.d.ts +36 -0
- package/dist/design/audit/reference/generate/prompt.d.ts.map +1 -0
- package/dist/design/audit/reference/generate/prompt.js +172 -0
- package/dist/design/audit/reference/generate/prompt.js.map +1 -0
- package/dist/design/audit/reference/index.d.ts +26 -0
- package/dist/design/audit/reference/index.d.ts.map +1 -0
- package/dist/design/audit/reference/index.js +31 -0
- package/dist/design/audit/reference/index.js.map +1 -0
- package/dist/design/audit/reference/judge/image-clamp.d.ts +34 -0
- package/dist/design/audit/reference/judge/image-clamp.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/image-clamp.js +88 -0
- package/dist/design/audit/reference/judge/image-clamp.js.map +1 -0
- package/dist/design/audit/reference/judge/pairwise.d.ts +34 -0
- package/dist/design/audit/reference/judge/pairwise.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/pairwise.js +105 -0
- package/dist/design/audit/reference/judge/pairwise.js.map +1 -0
- package/dist/design/audit/reference/judge/parse.d.ts +31 -0
- package/dist/design/audit/reference/judge/parse.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/parse.js +113 -0
- package/dist/design/audit/reference/judge/parse.js.map +1 -0
- package/dist/design/audit/reference/judge/prompt.d.ts +38 -0
- package/dist/design/audit/reference/judge/prompt.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/prompt.js +123 -0
- package/dist/design/audit/reference/judge/prompt.js.map +1 -0
- package/dist/design/audit/reference/judge/quality.d.ts +28 -0
- package/dist/design/audit/reference/judge/quality.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/quality.js +62 -0
- package/dist/design/audit/reference/judge/quality.js.map +1 -0
- package/dist/design/audit/reference/judge/rank.d.ts +41 -0
- package/dist/design/audit/reference/judge/rank.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/rank.js +196 -0
- package/dist/design/audit/reference/judge/rank.js.map +1 -0
- package/dist/design/audit/reference/judge/text-judge.d.ts +31 -0
- package/dist/design/audit/reference/judge/text-judge.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/text-judge.js +43 -0
- package/dist/design/audit/reference/judge/text-judge.js.map +1 -0
- package/dist/design/audit/reference/judge/vision-judge.d.ts +59 -0
- package/dist/design/audit/reference/judge/vision-judge.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/vision-judge.js +150 -0
- package/dist/design/audit/reference/judge/vision-judge.js.map +1 -0
- package/dist/design/audit/reference/judge/vision-model.d.ts +65 -0
- package/dist/design/audit/reference/judge/vision-model.d.ts.map +1 -0
- package/dist/design/audit/reference/judge/vision-model.js +110 -0
- package/dist/design/audit/reference/judge/vision-model.js.map +1 -0
- package/dist/design/audit/reference/pipeline/evaluate-reference.d.ts +68 -0
- package/dist/design/audit/reference/pipeline/evaluate-reference.d.ts.map +1 -0
- package/dist/design/audit/reference/pipeline/evaluate-reference.js +52 -0
- package/dist/design/audit/reference/pipeline/evaluate-reference.js.map +1 -0
- package/dist/design/audit/reference/reference-context.d.ts +31 -0
- package/dist/design/audit/reference/reference-context.d.ts.map +1 -0
- package/dist/design/audit/reference/reference-context.js +73 -0
- package/dist/design/audit/reference/reference-context.js.map +1 -0
- package/dist/design/audit/reference/retrieval/embedding-hash.d.ts +30 -0
- package/dist/design/audit/reference/retrieval/embedding-hash.d.ts.map +1 -0
- package/dist/design/audit/reference/retrieval/embedding-hash.js +87 -0
- package/dist/design/audit/reference/retrieval/embedding-hash.js.map +1 -0
- package/dist/design/audit/reference/retrieval/embedding-openai.d.ts +38 -0
- package/dist/design/audit/reference/retrieval/embedding-openai.d.ts.map +1 -0
- package/dist/design/audit/reference/retrieval/embedding-openai.js +73 -0
- package/dist/design/audit/reference/retrieval/embedding-openai.js.map +1 -0
- package/dist/design/audit/reference/retrieval/matcher.d.ts +34 -0
- package/dist/design/audit/reference/retrieval/matcher.d.ts.map +1 -0
- package/dist/design/audit/reference/retrieval/matcher.js +107 -0
- package/dist/design/audit/reference/retrieval/matcher.js.map +1 -0
- package/dist/design/audit/reference/run.d.ts +59 -0
- package/dist/design/audit/reference/run.d.ts.map +1 -0
- package/dist/design/audit/reference/run.js +99 -0
- package/dist/design/audit/reference/run.js.map +1 -0
- package/dist/design/audit/report.d.ts +35 -0
- package/dist/design/audit/report.d.ts.map +1 -0
- package/dist/design/audit/report.js +195 -0
- package/dist/design/audit/report.js.map +1 -0
- package/dist/design/audit/tokens/extract.d.ts +71 -0
- package/dist/design/audit/tokens/extract.d.ts.map +1 -0
- package/dist/design/audit/tokens/extract.js +1034 -0
- package/dist/design/audit/tokens/extract.js.map +1 -0
- package/dist/design/audit/types.d.ts +8 -0
- package/dist/design/audit/types.d.ts.map +1 -1
- package/dist/design/compare.d.ts.map +1 -1
- package/dist/design/compare.js +2 -6
- package/dist/design/compare.js.map +1 -1
- package/dist/design/cookie-consent.d.ts +3 -0
- package/dist/design/cookie-consent.d.ts.map +1 -0
- package/dist/design/cookie-consent.js +26 -0
- package/dist/design/cookie-consent.js.map +1 -0
- package/dist/design/viewports.d.ts +18 -0
- package/dist/design/viewports.d.ts.map +1 -0
- package/dist/design/viewports.js +10 -0
- package/dist/design/viewports.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/memory/store.d.ts +19 -1
- package/dist/memory/store.d.ts.map +1 -1
- package/dist/memory/store.js +47 -0
- package/dist/memory/store.js.map +1 -1
- package/dist/provider-defaults.d.ts +18 -0
- package/dist/provider-defaults.d.ts.map +1 -1
- package/dist/provider-defaults.js +33 -0
- package/dist/provider-defaults.js.map +1 -1
- package/dist/runner/allowed-domains.d.ts +11 -0
- package/dist/runner/allowed-domains.d.ts.map +1 -0
- package/dist/runner/allowed-domains.js +14 -0
- package/dist/runner/allowed-domains.js.map +1 -0
- package/dist/runner/batch-fill.d.ts +28 -0
- package/dist/runner/batch-fill.d.ts.map +1 -0
- package/dist/runner/batch-fill.js +80 -0
- package/dist/runner/batch-fill.js.map +1 -0
- package/dist/runner/completion-policy.d.ts +26 -0
- package/dist/runner/completion-policy.d.ts.map +1 -0
- package/dist/runner/completion-policy.js +76 -0
- package/dist/runner/completion-policy.js.map +1 -0
- package/dist/runner/constants.d.ts +14 -0
- package/dist/runner/constants.d.ts.map +1 -0
- package/dist/runner/constants.js +23 -0
- package/dist/runner/constants.js.map +1 -0
- package/dist/runner/decision-screenshot.d.ts +24 -0
- package/dist/runner/decision-screenshot.d.ts.map +1 -0
- package/dist/runner/decision-screenshot.js +23 -0
- package/dist/runner/decision-screenshot.js.map +1 -0
- package/dist/runner/domain-boundary.d.ts +36 -0
- package/dist/runner/domain-boundary.d.ts.map +1 -0
- package/dist/runner/domain-boundary.js +81 -0
- package/dist/runner/domain-boundary.js.map +1 -0
- package/dist/runner/effect-verify.d.ts +28 -0
- package/dist/runner/effect-verify.d.ts.map +1 -0
- package/dist/runner/effect-verify.js +42 -0
- package/dist/runner/effect-verify.js.map +1 -0
- package/dist/runner/execute-plan.d.ts +58 -0
- package/dist/runner/execute-plan.d.ts.map +1 -0
- package/dist/runner/execute-plan.js +426 -0
- package/dist/runner/execute-plan.js.map +1 -0
- package/dist/runner/max-turns-extension.d.ts +19 -0
- package/dist/runner/max-turns-extension.d.ts.map +1 -0
- package/dist/runner/max-turns-extension.js +25 -0
- package/dist/runner/max-turns-extension.js.map +1 -0
- package/dist/runner/micro-plan.d.ts +11 -0
- package/dist/runner/micro-plan.d.ts.map +1 -0
- package/dist/runner/micro-plan.js +29 -0
- package/dist/runner/micro-plan.js.map +1 -0
- package/dist/runner/prompt-snippets.d.ts +5 -0
- package/dist/runner/prompt-snippets.d.ts.map +1 -0
- package/dist/runner/prompt-snippets.js +13 -0
- package/dist/runner/prompt-snippets.js.map +1 -0
- package/dist/runner/replay/contracts.d.ts +300 -0
- package/dist/runner/replay/contracts.d.ts.map +1 -0
- package/dist/runner/replay/contracts.js +42 -0
- package/dist/runner/replay/contracts.js.map +1 -0
- package/dist/runner/replay/controller.d.ts +27 -0
- package/dist/runner/replay/controller.d.ts.map +1 -0
- package/dist/runner/replay/controller.js +197 -0
- package/dist/runner/replay/controller.js.map +1 -0
- package/dist/runner/replay/guard.d.ts +28 -0
- package/dist/runner/replay/guard.d.ts.map +1 -0
- package/dist/runner/replay/guard.js +100 -0
- package/dist/runner/replay/guard.js.map +1 -0
- package/dist/runner/runner.d.ts +44 -51
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +155 -851
- package/dist/runner/runner.js.map +1 -1
- package/dist/runner/scout-feedback.d.ts +51 -0
- package/dist/runner/scout-feedback.d.ts.map +1 -0
- package/dist/runner/scout-feedback.js +149 -0
- package/dist/runner/scout-feedback.js.map +1 -0
- package/dist/skills/macro-loader.d.ts +1 -1
- package/dist/test-runner.d.ts.map +1 -1
- package/dist/test-runner.js +27 -1
- package/dist/test-runner.js.map +1 -1
- package/dist/types/actions.d.ts +241 -0
- package/dist/types/actions.d.ts.map +1 -0
- package/dist/types/actions.js +5 -0
- package/dist/types/actions.js.map +1 -0
- package/dist/types/config.d.ts +211 -0
- package/dist/types/config.d.ts.map +1 -0
- package/dist/types/config.js +2 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/design-audit.d.ts +92 -0
- package/dist/types/design-audit.d.ts.map +1 -0
- package/dist/types/design-audit.js +5 -0
- package/dist/types/design-audit.js.map +1 -0
- package/dist/types/design-tokens.d.ts +138 -0
- package/dist/types/design-tokens.d.ts.map +1 -0
- package/dist/types/design-tokens.js +5 -0
- package/dist/types/design-tokens.js.map +1 -0
- package/dist/types/page.d.ts +16 -0
- package/dist/types/page.d.ts.map +1 -0
- package/dist/types/page.js +2 -0
- package/dist/types/page.js.map +1 -0
- package/dist/types/plan.d.ts +50 -0
- package/dist/types/plan.d.ts.map +1 -0
- package/dist/types/plan.js +2 -0
- package/dist/types/plan.js.map +1 -0
- package/dist/types/preview.d.ts +9 -0
- package/dist/types/preview.d.ts.map +1 -0
- package/dist/types/preview.js +5 -0
- package/dist/types/preview.js.map +1 -0
- package/dist/types/result.d.ts +62 -0
- package/dist/types/result.d.ts.map +1 -0
- package/dist/types/result.js +2 -0
- package/dist/types/result.js.map +1 -0
- package/dist/types/scenario.d.ts +20 -0
- package/dist/types/scenario.d.ts.map +1 -0
- package/dist/types/scenario.js +5 -0
- package/dist/types/scenario.js.map +1 -0
- package/dist/types/test-runner.d.ts +145 -0
- package/dist/types/test-runner.d.ts.map +1 -0
- package/dist/types/test-runner.js +10 -0
- package/dist/types/test-runner.js.map +1 -0
- package/dist/types/trajectory.d.ts +38 -0
- package/dist/types/trajectory.d.ts.map +1 -0
- package/dist/types/trajectory.js +2 -0
- package/dist/types/trajectory.js.map +1 -0
- package/dist/types/turn.d.ts +50 -0
- package/dist/types/turn.d.ts.map +1 -0
- package/dist/types/turn.js +2 -0
- package/dist/types/turn.js.map +1 -0
- package/dist/types.d.ts +16 -1017
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +15 -8
- package/dist/types.js.map +1 -1
- package/dist/wallet/rpc-interception.d.ts +15 -0
- package/dist/wallet/rpc-interception.d.ts.map +1 -0
- package/dist/wallet/rpc-interception.js +95 -0
- package/dist/wallet/rpc-interception.js.map +1 -0
- package/package.json +23 -5
package/dist/brain/index.js
CHANGED
|
@@ -4,411 +4,28 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Uses Vercel AI SDK for multi-provider support (OpenAI, Anthropic, Google, Codex CLI, Claude Code).
|
|
6
6
|
*/
|
|
7
|
-
import { generateText
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
- {"action": "click", "selector": "@REF"}
|
|
20
|
-
- {"action": "type", "selector": "@REF", "text": "text to type"}
|
|
21
|
-
- {"action": "press", "selector": "@REF", "key": "Enter"} (or Tab, Escape, ArrowDown, etc.)
|
|
22
|
-
- {"action": "hover", "selector": "@REF"}
|
|
23
|
-
- {"action": "select", "selector": "@REF", "value": "option-value"}
|
|
24
|
-
- {"action": "scroll", "direction": "up" | "down", "amount": 500} — add "selector": "@REF" to scroll a specific container
|
|
25
|
-
- {"action": "navigate", "url": "https://..."}
|
|
26
|
-
- {"action": "wait", "ms": 1000}
|
|
27
|
-
- {"action": "evaluate", "criteria": "Is the layout professional? Are colors consistent?"}
|
|
28
|
-
- {"action": "runScript", "script": "document.querySelector('.count').textContent"} — run JS in page context and get the result. Use for reading content not in the a11y tree (canvas, computed styles, hidden state).
|
|
29
|
-
- {"action": "extractWithIndex", "query": "p, span, dd, code", "contains": "downloads"} — return a NUMBERED list of every visible element matching \`query\`, with each element's tag, full textContent, key attributes, and a stable selector. PREFER THIS OVER runScript when you need to find data inside the page but don't know the exact selector. The wide query (e.g. \`'p, span, strong'\`) finds candidates and the response shows the actual text so you can pick by content match. Optional \`contains\` filters matches to those whose text contains a substring (case-insensitive). After this action, your next turn can complete with the picked element's text or click its selector.
|
|
30
|
-
- {"action": "verifyPreview"} — after the app builds, inspect the preview iframe. Returns URL, title, a11y tree, and errors. Use this AFTER you see a preview iframe on the page.
|
|
31
|
-
- {"action": "fill", "fields": {"@t1": "Jordan", "@t2": "Rivera"}, "selects": {"@s1": "WA"}, "checks": ["@c1", "@c2"]} — BATCH fill multiple form fields, dropdowns, and checkboxes in ONE turn. Use this whenever you can see 2+ form fields you need to fill — it's dramatically faster than per-field type/click. fields/selects/checks are all optional but at least one must be non-empty.
|
|
32
|
-
- {"action": "clickSequence", "refs": ["@r1", "@r2", "@r3"]} — click a known sequence of refs in order. Use for multi-step UI navigation chains where the click order is obvious from the page structure.
|
|
33
|
-
- {"action": "complete", "result": "description of what was accomplished"}
|
|
34
|
-
- {"action": "abort", "reason": "why you cannot continue"}
|
|
35
|
-
|
|
36
|
-
SELECTOR FORMAT:
|
|
37
|
-
- CRITICAL: Replace @REF with an actual ref from the ELEMENTS list below (e.g., @b3cee, @t1f2a)
|
|
38
|
-
- NEVER invent or guess ref IDs — only use refs that appear as [ref=XXX] in the ELEMENTS list
|
|
39
|
-
- Refs are deterministic — same element keeps the same ref across observations
|
|
40
|
-
- Fallback: [data-testid="..."], [aria-label="..."], text="...", role=button[name="..."]
|
|
41
|
-
|
|
42
|
-
RESPONSE FORMAT — respond with ONLY a JSON object:
|
|
43
|
-
{
|
|
44
|
-
"plan": ["step 1", "step 2", ...],
|
|
45
|
-
"currentStep": 0,
|
|
46
|
-
"action": { "action": "click", "selector": "@REF_FROM_ELEMENTS" },
|
|
47
|
-
"nextActions": [{ "action": "type", "selector": "@REF_FROM_ELEMENTS", "text": "..." }],
|
|
48
|
-
"reasoning": "Why I chose this action based on what I see",
|
|
49
|
-
"expectedEffect": "What should change (e.g., 'URL should contain /chat/', 'modal should close')"
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
RULES:
|
|
53
|
-
1. Respond with ONLY valid JSON, no markdown or extra text
|
|
54
|
-
2. Use @ref selectors from the ELEMENTS list — they are stable across turns
|
|
55
|
-
3. Include plan, currentStep, reasoning, and expectedEffect in every response
|
|
56
|
-
4. Primary action must be in "action". Optional "nextActions" can contain up to 2 safe follow-ups (click/type/press/hover/select/scroll/wait) only when deterministic
|
|
57
|
-
5. When the goal is achieved, use "complete" with a detailed result description
|
|
58
|
-
6. If stuck after multiple attempts, use "abort" — don't loop forever
|
|
59
|
-
7. LOOK at the screenshot — it shows visual state the a11y tree may miss
|
|
60
|
-
8. If an action failed, try a DIFFERENT approach (different selector, different strategy)
|
|
61
|
-
9. For complex goals, break them into clear plan steps and track progress
|
|
62
|
-
10. Use "evaluate" when you need to assess visual quality, layout, or design
|
|
63
|
-
11. After the app builds and a preview is visible, use "verifyPreview" to check for errors before completing
|
|
64
|
-
12. BLOCKER-FIRST POLICY: if a modal, limit, quota, permission, or error dialog blocks progress, resolve THAT first before continuing the main goal
|
|
65
|
-
13. For quota/limit blockers, use an unblock ladder: open manage path -> clean up old test resources if needed -> retry the original action
|
|
66
|
-
14. If the same action triggers the same blocker twice, switch strategy immediately (different button/path), do not repeat blind retries
|
|
67
|
-
15. BATCH FILL FOR MULTI-FIELD FORMS: when you can see 2+ form fields that need to be filled, ALWAYS use a single "fill" action with all the fields at once instead of multiple type/click turns. A 5-field form takes 1 turn with fill, not 10 turns with type. Same for dropdowns (use selects map) and checkboxes (use checks array). The page rarely cares which order fields are filled — batch them.
|
|
68
|
-
- CRITICAL: every key in fields/selects/checks MUST be an @ref taken VERBATIM from the ELEMENTS list (e.g., "@t1f2a"), or a simple [data-testid="..."] selector copied from the DATA-TESTID SELECTORS section. NEVER invent CSS combinators like "[data-testid=\"x\"] input" or "@refXXX child". If a target doesn't appear in the snapshot, use single-step type/click for it instead.
|
|
69
|
-
- Date inputs (type="date") and spinbuttons (year/month/day) typically need single-step "type" actions, NOT batch fill. They have non-text input behavior that confuses Playwright's fill(). Skip them in your batch and handle them with type after.
|
|
70
|
-
- If a batch fill fails, do NOT retry the same batch on the next turn. The error message will tell you which target failed — switch to single-step type/click for that target and shrink your next batch to just the targets that work.`;
|
|
71
|
-
/** Search-related rules (15-17): injected when page has search elements or /search URL */
|
|
72
|
-
const SEARCH_RULES = `
|
|
73
|
-
15. SEARCH FORMS: Always interact with the form (type in search box, then click Search or press Enter). Do NOT navigate to a URL with search query parameters — many sites require form submission to trigger filtering. If a search yields no results, try the page's own search box rather than the site-wide search
|
|
74
|
-
16. CONTENT DISCOVERY: If the ELEMENTS list doesn't show the link/content you need (e.g., the page has many links but the a11y tree is truncated), use runScript to find it: document.querySelectorAll('a[href]') filtered by keyword. Navigate to the discovered URL directly instead of clicking blindly through menus
|
|
75
|
-
17. EXTERNAL SEARCH REDIRECTS: If a site's search form redirects to an external search engine (e.g., search.usa.gov for .gov sites), the results still link back to the original site. Click a relevant search result link — it will take you to the target domain. Do NOT abandon search results to navigate the target site manually`;
|
|
76
|
-
/** Data extraction rules (18, 21-23, 25): injected when goal involves extracting data */
|
|
77
|
-
const DATA_EXTRACTION_RULES = `
|
|
78
|
-
18. DATA EXTRACTION: When the goal asks for specific data (prices, ratings, counts, names) from a list or search results page, prefer extractWithIndex with a wide query (e.g. \`'p, span, dd, code, strong'\`) over runScript when you don't already see the value in the snapshot. extractWithIndex returns the actual textContent of every match so you can pick the right one by content. Use runScript only when you need a transformation the LLM can't do from text alone.
|
|
79
|
-
21. EFFICIENT COMPLETION: When you have enough data to answer the goal, complete immediately. Do not navigate to additional pages for "confirmation" if the data was already extracted or is visible in the current a11y tree. Include all extracted data in the completion result
|
|
80
|
-
22. EXTRACT BEFORE NAVIGATING: On search results, directory listings, or any page showing multiple items, ALWAYS extract ALL needed data BEFORE clicking into individual items. Use extractWithIndex with a wide query for unknown structure, or runScript with document.querySelectorAll('.result-card') if the structure is well-known. Many sites use anti-bot protection on detail pages but leave listing pages accessible. If you can answer the goal from list-level data, do so without navigating deeper.
|
|
81
|
-
23. FILTER vs SEARCH: When a goal asks to filter results (e.g., "under $50", "4+ stars"), look for filter controls (sliders, dropdowns, checkboxes in a sidebar or toolbar) rather than typing filter values into the search box. Search boxes are for keyword queries, not numeric filters. After applying a filter: (1) wait 2-3 seconds for results to update, (2) verify the filter took effect by checking the updated results, (3) extract the filtered data. Do NOT keep searching for more filter controls after one is applied — extract and complete
|
|
82
|
-
25. EXTRACTWITHINDEX RECOVERY: If a previous runScript returned null/empty/{x:null} on an extraction task, the selector was wrong. DO NOT retry the same runScript or guess a similar selector — the LLM cannot guess CSS class names that aren't visible in the snapshot. Switch to extractWithIndex with a WIDE query: \`'p, span, dd, code, strong, em'\` plus a \`contains\` filter naming the expected text fragment (e.g. contains: "downloads" for npm download counts, contains: "callbackFn" for MDN method signatures). The response shows you the actual text per element so you can pick by content match. Pick-by-content beats pick-by-selector on pages where the planner couldn't see the data at plan time.`;
|
|
83
|
-
/** Heavy page rules (19-20, 24): injected when snapshot is large or turn count is high */
|
|
84
|
-
const HEAVY_PAGE_RULES = `
|
|
85
|
-
19. FORM FIELD TARGETING: Before typing, verify you are targeting the correct input field using its @ref from the ELEMENTS list. If multiple inputs are visible (e.g., search box + price filter), ensure you select the right one by checking its label or placeholder text in the a11y tree. Never assume focus — always specify the exact @ref
|
|
86
|
-
20. SECTION NAVIGATION: When you need to find a specific section (e.g., rugby, sports, travel) and the nav links aren't in the truncated a11y tree, use runScript to discover navigation: JSON.stringify(Array.from(document.querySelectorAll('nav a, header a, [role="navigation"] a, .nav a')).slice(0, 30).map(a => ({text: a.textContent.trim(), href: a.href}))). Then navigate directly to the matching section URL
|
|
87
|
-
24. HEAVY PAGE RECOVERY: If a page takes very long to load or seems stuck, do NOT wait — use runScript to check document.readyState and extract whatever content is already in the DOM. Partial data is better than a timeout. If the page is completely blank, try navigating to a simpler version (mobile site, search page) instead of waiting`;
|
|
88
|
-
/** URL-first navigation rules for complex forms and search pages.
|
|
89
|
-
* Teaches the agent to construct search/results URLs from goal text
|
|
90
|
-
* instead of fighting form UIs. Works on any site with URL parameters. */
|
|
91
|
-
const URL_FIRST_RULES = `
|
|
92
|
-
URL-FIRST NAVIGATION: When a search form is complex (date pickers, multi-step dropdowns, dynamic widgets), try constructing a results URL directly instead of interacting with the form.
|
|
93
|
-
|
|
94
|
-
STRATEGY:
|
|
95
|
-
1. Look at the current URL structure. Most search sites encode parameters: ?q=query, ?checkin=date, ?dest=city, etc.
|
|
96
|
-
2. Construct a URL with the goal's parameters filled in. Use the site's own URL pattern.
|
|
97
|
-
3. Navigate directly to that URL — skip the form entirely.
|
|
98
|
-
4. If the URL doesn't work (wrong page, error), fall back to form interaction.
|
|
99
|
-
|
|
100
|
-
HOW TO DISCOVER URL PATTERNS:
|
|
101
|
-
- If you're on a search results page, the URL already shows the pattern. Modify the parameters for your goal.
|
|
102
|
-
- Most sites accept ?q= or ?search= for keyword queries.
|
|
103
|
-
- Travel sites typically use: checkin/checkout dates, destination/origin, adults count.
|
|
104
|
-
- Use runScript to read window.location.href if the URL isn't visible in the snapshot.
|
|
105
|
-
- ENCODED PARAMETERS: If a URL contains encoded parameters (base64, protobuf), you may be able to replicate them from a previous successful URL, but do NOT invent new encodings. If a navigate with encoded parameters lands on the wrong page, do not retry with a different encoding — it will waste turns.
|
|
106
|
-
|
|
107
|
-
IMPORTANT EXCEPTIONS — some sites BLOCK direct URL navigation:
|
|
108
|
-
- If a direct URL navigate lands on the homepage or an error page instead of results, the site blocks URL manipulation. STOP trying URLs and use the site's form/search UI instead.
|
|
109
|
-
- After ONE failed URL attempt, switch to form interaction immediately. Do NOT retry different URL patterns — you will waste turns.
|
|
110
|
-
|
|
111
|
-
FORM RESET DETECTION: Some sites (especially SPAs) silently reset form fields after filling. After batch-filling a form:
|
|
112
|
-
1. Use runScript to verify values stuck: document.querySelector('[aria-label="From"]')?.value or similar.
|
|
113
|
-
2. If fields reset to defaults (wrong city, blank dates), do NOT re-fill with the same approach — it will reset again.
|
|
114
|
-
3. Instead, switch to keyboard-only interaction: click the field, type the value character by character, wait for autocomplete dropdown, press Enter to confirm. Then Tab to the next field.
|
|
115
|
-
|
|
116
|
-
DATE PICKER STRATEGY: Calendar widgets often ignore programmatic fill/type. When a date field opens a calendar popup that blocks further input:
|
|
117
|
-
1. Try typing the date directly into the field in the site's format (e.g., "Jan 25, 2026" or "01/25/2026"). Press Escape first if the calendar covers the input.
|
|
118
|
-
2. If typing doesn't stick, use runScript to find clickable date elements: document.querySelectorAll('[data-iso],[aria-label*="January"],[aria-label*="25"]') and click the matching element.
|
|
119
|
-
3. NEVER spend more than 4 turns on a single date field.
|
|
120
|
-
|
|
121
|
-
WHY: Complex forms with date pickers, calendar widgets, and multi-step dropdowns consume many turns and often time out. A single "navigate" action replaces 5-10 form interaction turns. But only use this on sites that support it.`;
|
|
122
|
-
/** Reasoning framework and examples (always appended after rules) */
|
|
123
|
-
const REASONING_SUFFIX = `
|
|
124
|
-
|
|
125
|
-
REASONING FRAMEWORK — before choosing an action:
|
|
126
|
-
1. What is the current state vs. the goal state? What is missing?
|
|
127
|
-
2. What is the smallest action that makes progress toward the goal?
|
|
128
|
-
3. If multiple elements could match, prefer the one closest to the user-visible label
|
|
129
|
-
4. If an action just failed, identify WHY it failed before trying again
|
|
130
|
-
5. Ask: "Is there a blocker preventing progress right now?" If yes, clear blocker first, then continue goal plan
|
|
131
|
-
|
|
132
|
-
EXAMPLE 1 — Multi-step form fill (use actual refs from ELEMENTS, not these placeholders):
|
|
133
|
-
{"plan":["Navigate to signup page","Fill email field","Fill password field","Click submit","Verify success"],"currentStep":1,"action":{"action":"type","selector":"@REF","text":"user@example.com"},"reasoning":"I see the signup form with email input [ref=...] and password input [ref=...]. Starting with email since it is the first required field.","expectedEffect":"Email field should show 'user@example.com'"}
|
|
134
|
-
|
|
135
|
-
EXAMPLE 2 — Recovery after failure:
|
|
136
|
-
{"plan":["Click the send button","Wait for response"],"currentStep":0,"action":{"action":"scroll","direction":"down","amount":300},"reasoning":"My last click failed because the element was not visible in the viewport. I can see from the screenshot that the send button is below the fold. Scrolling down to bring it into view before retrying.","expectedEffect":"The send button should become visible in the viewport"}
|
|
137
|
-
|
|
138
|
-
EXAMPLE 3 — Batch fill a multi-field form (one turn instead of ten):
|
|
139
|
-
{"plan":["Fill all visible Personal Info fields","Click Next","Fill Contact step","Submit"],"currentStep":0,"action":{"action":"fill","fields":{"@firstname":"Jordan","@lastname":"Rivera","@dob":"1990-04-15"},"selects":{"@gender":"other"}},"reasoning":"Step 1 of the form has 3 text fields and 1 select all visible at once. Filling them in a single batch action saves 7 turns vs typing each individually.","expectedEffect":"All four Step 1 fields populated with the supplied values"}`;
|
|
140
|
-
/** Full static prompt (all rules) — used as default when config.systemPrompt is not set */
|
|
141
|
-
const SYSTEM_PROMPT = CORE_RULES + SEARCH_RULES + DATA_EXTRACTION_RULES + HEAVY_PAGE_RULES + REASONING_SUFFIX;
|
|
142
|
-
// Vision-first system prompt for pure coordinate actions.
|
|
143
|
-
const VISION_FIRST_PROMPT = `You are a browser automation agent. You operate by looking at screenshots and clicking on elements using pixel coordinates.
|
|
144
|
-
|
|
145
|
-
The screenshot shows the current page state at 1024×768 resolution. You identify elements visually and specify where to click using (x, y) coordinates in this coordinate space.
|
|
146
|
-
|
|
147
|
-
ACTIONS:
|
|
148
|
-
- {"action": "clickAt", "x": 512, "y": 384} — click at pixel coordinates (x, y) in 1024×768 space
|
|
149
|
-
- {"action": "typeAt", "x": 300, "y": 200, "text": "search query"} — click at coordinates then type text
|
|
150
|
-
- {"action": "scroll", "direction": "up" | "down", "amount": 500}
|
|
151
|
-
- {"action": "navigate", "url": "https://..."}
|
|
152
|
-
- {"action": "wait", "ms": 1000}
|
|
153
|
-
- {"action": "complete", "result": "description of what was accomplished"}
|
|
154
|
-
- {"action": "abort", "reason": "why you cannot continue"}
|
|
155
|
-
- {"action": "runScript", "script": "document.querySelector('.count').textContent"} — run JS in page context for data the screenshot can't show
|
|
156
|
-
- {"action": "extractWithIndex", "query": "p, span, dd", "contains": "keyword"} — find text in the DOM by content match
|
|
157
|
-
- {"action": "fanOut", "subGoals": [...]} — spawn up to 8 parallel sub-agents in separate tabs (same session/cookies). See FAN-OUT section below for full rules + worked example.
|
|
158
|
-
|
|
159
|
-
FAN-OUT — PARALLEL INVESTIGATION:
|
|
160
|
-
fanOut is the way to investigate N independent candidates in PARALLEL instead of serially. When the current page shows a list of candidates or you have a queue of independent sub-tasks, emit ONE fanOut action instead of processing them one at a time. The system spawns N sub-agents in fresh tabs of the same session; they run concurrently; their results are merged and returned to you as structured JSON in the NEXT turn's feedback.
|
|
161
|
-
|
|
162
|
-
STRONGLY PREFER THE SHORTHAND FORM when every branch shares a URL + instruction template. It emits tiny JSON that can't malform:
|
|
163
|
-
|
|
164
|
-
{"action":"fanOut","baseUrl":"https://site.example/","goalTemplate":"Investigate {item} — report outcome","items":["X","Y","Z"]}
|
|
165
|
-
|
|
166
|
-
The string {item} in goalTemplate is replaced with each array entry. Labels default to the item. This is the RIGHT shape for N>=3 branches.
|
|
167
|
-
|
|
168
|
-
USE fanOut WHEN:
|
|
169
|
-
- A search returned multiple results and each needs investigation (click in / extract / return verdict per row).
|
|
170
|
-
- A batch job has ≥3 independent sub-tasks (screen multiple customers, check multiple products, compare multiple pages).
|
|
171
|
-
- The task is obviously parallelizable and sequential execution would 3x+ the wall-clock time.
|
|
172
|
-
|
|
173
|
-
DO NOT use fanOut for:
|
|
174
|
-
- A truly sequential task where step 2 depends on step 1's outcome.
|
|
175
|
-
- A single-target investigation (use regular click/type).
|
|
176
|
-
- Cases where fewer than 3 branches would run (overhead not worth it).
|
|
177
|
-
|
|
178
|
-
SHAPE:
|
|
179
|
-
{
|
|
180
|
-
"action": "fanOut",
|
|
181
|
-
"subGoals": [
|
|
182
|
-
{
|
|
183
|
-
"url": "https://target.site/",
|
|
184
|
-
"goal": "Full natural-language instruction for this branch. Must be self-contained because the sub-agent starts from the url with no other context. End with 'Complete with a structured verdict of {schema}.'",
|
|
185
|
-
"label": "SHORT-LABEL-FOR-OVERLAY",
|
|
186
|
-
"maxTurns": 8
|
|
187
|
-
},
|
|
188
|
-
... (1-8 entries)
|
|
189
|
-
]
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
WORKED EXAMPLE (OFAC batch screening, after C-001 is done via regular actions):
|
|
193
|
-
{
|
|
194
|
-
"action": "fanOut",
|
|
195
|
-
"subGoals": [
|
|
196
|
-
{"url":"https://sanctionssearch.ofac.treas.gov/","goal":"Click the Reset button, then type 'SMITH' into Last Name and 'JOHN' into First Name. Leave score at 95. Click Search. If 0 matches, complete with result 'CLEARED'. If 1+ exact matches with score 95-100, click the top row, read SDN program + list + DOB, complete with result 'POSITIVE MATCH: <program>/<list>'. Else complete with 'NEEDS REVIEW'.","label":"C-002 SMITH"},
|
|
197
|
-
{"url":"https://sanctionssearch.ofac.treas.gov/","goal":"Click Reset, type 'MADURO' into Last Name and 'NICOLAS' into First Name. Leave score at 95. Click Search. [same disposition rules]","label":"C-003 MADURO"},
|
|
198
|
-
... (up to 8 per fanOut)
|
|
199
|
-
]
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
AFTER fanOut RETURNS, you receive FAN-OUT RESULTS as feedback — a JSON payload with {label, success, verdict, turnsUsed} per branch. Update your progress ledger with each branch's verdict, then either fire another fanOut for the next batch OR complete() if all sub-tasks are done.
|
|
203
|
-
|
|
204
|
-
EFFICIENCY: one fanOut with 8 branches running in parallel finishes in ~the time of ONE sequential customer (not 8×). For a 10-customer batch, prefer: C-001 sequential (learn the form) → fanOut C-002..C-009 (8 parallel) → C-010 sequential or 2nd fanOut. Target: ~12 parent turns instead of ~50.
|
|
205
|
-
|
|
206
|
-
COORDINATE SYSTEM:
|
|
207
|
-
- (0, 0) is the top-left corner of the viewport
|
|
208
|
-
- (1024, 768) is the bottom-right corner
|
|
209
|
-
- Click the CENTER of the target element, not its edge
|
|
210
|
-
- For text inputs, click the middle of the input field
|
|
211
|
-
- For buttons, click the center of the button text or icon
|
|
212
|
-
|
|
213
|
-
RESPONSE FORMAT — respond with ONLY a JSON object:
|
|
214
|
-
{
|
|
215
|
-
"plan": ["step 1", "step 2", ...],
|
|
216
|
-
"currentStep": 0,
|
|
217
|
-
"action": { "action": "clickAt", "x": 512, "y": 384 },
|
|
218
|
-
"reasoning": "I see [element description] at approximately (x, y). Clicking it to [purpose].",
|
|
219
|
-
"expectedEffect": "What should change after this action"
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
RULES:
|
|
223
|
-
1. Respond with ONLY valid JSON, no markdown or extra text
|
|
224
|
-
2. LOOK at the screenshot carefully — it is your primary information source
|
|
225
|
-
3. Include plan, currentStep, reasoning, and expectedEffect in every response
|
|
226
|
-
4. When the goal is achieved, use "complete" with a detailed result description
|
|
227
|
-
5. If stuck after multiple attempts, use "abort" — don't loop forever
|
|
228
|
-
6. If an action failed, try a DIFFERENT approach (different location, different strategy)
|
|
229
|
-
7. For search: click the search box, type your query, then press Enter
|
|
230
|
-
8. For navigation: click visible links or use the "navigate" action for direct URLs
|
|
231
|
-
9. BLOCKER-FIRST: if a modal, cookie banner, or error dialog blocks progress, dismiss it first
|
|
232
|
-
10. Use runScript or extractWithIndex when you need to extract data that isn't clearly visible in the screenshot
|
|
233
|
-
11. BATCH: when filling forms, you can type in one field, then immediately use clickAt on the next field. Plan multiple actions per turn when they are sequential and obvious.
|
|
234
|
-
12. VERIFY BEFORE COMPLETING: Before using "complete", re-read the GOAL and check: does your result ACTUALLY answer what was asked? If the goal asks for "5 beauty salons with ratings > 4.8" and you only found 3, do NOT complete — keep searching. If the goal asks for a specific date/price/name and your result doesn't contain it, do NOT complete. Premature completion with wrong data is worse than using another turn.
|
|
235
|
-
13. DATE PICKER BYPASS: If you encounter a complex date picker widget (calendar popup, date spinner) that is hard to interact with, DO NOT spend multiple turns clicking through calendar months. Instead, use "navigate" to construct a URL with the date parameters encoded. For Google Flights: navigate to google.com/travel/flights with search params. For Booking: navigate to booking.com/searchresults with checkin/checkout params. URL-based date setting is faster and more reliable than fighting date picker UIs.
|
|
236
|
-
|
|
237
|
-
REASONING FRAMEWORK:
|
|
238
|
-
1. What do I see in the screenshot? Describe the visual layout.
|
|
239
|
-
2. Where is the element I need to interact with? Estimate its (x, y) coordinates.
|
|
240
|
-
3. What is the smallest action that makes progress toward the goal?
|
|
241
|
-
4. If my last action failed, WHY did it fail? Try a different location or strategy.`;
|
|
242
|
-
// Unified vision+DOM prompt. The model sees BOTH the screenshot AND
|
|
243
|
-
// the ARIA snapshot with @refs. It can use EITHER coordinate actions (clickAt/
|
|
244
|
-
// typeAt for visual targets) OR ref actions (click/type/fill for DOM elements).
|
|
245
|
-
// This lets it pick the best tool per interaction: vision for visual layout,
|
|
246
|
-
// DOM for precise form interaction.
|
|
247
|
-
const UNIFIED_VISION_DOM_PROMPT = `You are a browser automation agent with TWO input modalities: a screenshot showing the visual page state, and a structured ELEMENTS list with interactive element refs.
|
|
248
|
-
|
|
249
|
-
Use BOTH together:
|
|
250
|
-
- The screenshot shows layout, visual state, images, icons — what a human sees
|
|
251
|
-
- The ELEMENTS list shows interactive elements with @ref IDs for precise targeting
|
|
252
|
-
|
|
253
|
-
ACTIONS — pick the best tool for each interaction:
|
|
254
|
-
|
|
255
|
-
LABEL ACTIONS (PREFERRED — use the [N] numbered labels visible on the screenshot):
|
|
256
|
-
- {"action": "clickLabel", "label": 3} — click element labeled [3] in the screenshot
|
|
257
|
-
- {"action": "typeLabel", "label": 5, "text": "query"} — click [5] then type text
|
|
258
|
-
The screenshot has numbered red badges on interactive elements. Use these labels — they're MORE ACCURATE than coordinate guessing.
|
|
259
|
-
|
|
260
|
-
REF ACTIONS (use for form fields, buttons, links with clear @refs from ELEMENTS):
|
|
261
|
-
- {"action": "click", "selector": "@REF"}
|
|
262
|
-
- {"action": "type", "selector": "@REF", "text": "text"}
|
|
263
|
-
- {"action": "press", "selector": "@REF", "key": "Enter"}
|
|
264
|
-
- {"action": "select", "selector": "@REF", "value": "option"}
|
|
265
|
-
- {"action": "fill", "fields": {"@REF1": "val1", "@REF2": "val2"}} — batch fill multiple form fields
|
|
266
|
-
|
|
267
|
-
COORDINATE ACTIONS (fallback when no label or ref is available):
|
|
268
|
-
- {"action": "clickAt", "x": 512, "y": 384} — click at pixel (x, y) in 1024×768 space
|
|
269
|
-
- {"action": "typeAt", "x": 300, "y": 200, "text": "query"} — click + type
|
|
270
|
-
|
|
271
|
-
SHARED ACTIONS:
|
|
272
|
-
- {"action": "scroll", "direction": "up" | "down", "amount": 500}
|
|
273
|
-
- {"action": "navigate", "url": "https://..."}
|
|
274
|
-
- {"action": "wait", "ms": 1000}
|
|
275
|
-
- {"action": "runScript", "script": "..."} — run JS in page context
|
|
276
|
-
- {"action": "extractWithIndex", "query": "p, span", "contains": "keyword"}
|
|
277
|
-
- {"action": "complete", "result": "description"}
|
|
278
|
-
- {"action": "abort", "reason": "why"}
|
|
279
|
-
|
|
280
|
-
WHEN TO USE WHICH (priority order):
|
|
281
|
-
1. Element has a [N] label in the screenshot → use clickLabel/typeLabel (most accurate)
|
|
282
|
-
2. Element has an @ref in ELEMENTS → use click/type/fill (fast and precise)
|
|
283
|
-
3. Element is visible but has no label or ref → use clickAt/typeAt (coordinate fallback)
|
|
284
|
-
- Date pickers, dropdown items rendered dynamically → use clickLabel if labeled, else clickAt
|
|
285
|
-
|
|
286
|
-
RESPONSE FORMAT — respond with ONLY a JSON object:
|
|
287
|
-
{
|
|
288
|
-
"plan": ["step 1", "step 2", ...],
|
|
289
|
-
"currentStep": 0,
|
|
290
|
-
"action": { "action": "click", "selector": "@REF" },
|
|
291
|
-
"nextActions": [{ "action": "type", "selector": "@REF2", "text": "query" }],
|
|
292
|
-
"reasoning": "Why I chose this action",
|
|
293
|
-
"expectedEffect": "What should change"
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
NOTE: "nextActions" is optional — include up to 3 safe follow-up actions (click, type, press, clickAt, typeAt, scroll) that are DETERMINISTIC given the current state. For example: click a search box THEN type a query. This saves turns.
|
|
297
|
-
|
|
298
|
-
RULES:
|
|
299
|
-
1. Respond with ONLY valid JSON
|
|
300
|
-
2. Use @ref selectors from ELEMENTS when available — they are stable and precise
|
|
301
|
-
3. Fall back to clickAt coordinates when the target has no ref or is visual-only
|
|
302
|
-
4. LOOK at the screenshot — it shows visual state the ELEMENTS list may miss
|
|
303
|
-
5. When the goal is achieved, use "complete" with a detailed result
|
|
304
|
-
6. BLOCKER-FIRST: dismiss modals, cookie banners, login walls before continuing
|
|
305
|
-
7. BATCH FILL: when 2+ form fields are visible with refs, use a single "fill" action
|
|
306
|
-
8. If stuck after multiple attempts, use "abort"
|
|
307
|
-
9. VERIFY BEFORE COMPLETING: Before using "complete", re-read the GOAL and check: does your result ACTUALLY answer what was asked? If the goal asks for specific data (prices, names, ratings, counts) and your result doesn't contain ALL of them, keep going. Premature completion with partial data is worse than using another turn.
|
|
308
|
-
10. FORM RESET DETECTION: After batch-filling a form, verify values stuck via runScript. If fields reset to defaults, switch to keyboard-only: click field → type value → wait for autocomplete → press Enter → Tab to next. Do NOT re-fill with the same approach if it reset once.
|
|
309
|
-
11. DATE PICKER STRATEGY: When a calendar popup opens over a date field:
|
|
310
|
-
a. Press Escape to dismiss, then type the date directly (e.g., "Jan 25, 2026").
|
|
311
|
-
b. If typing doesn't stick, use runScript to find clickable dates: document.querySelectorAll('[data-iso],[aria-label*="25"]').
|
|
312
|
-
c. NEVER spend more than 4 turns on a single date field.`;
|
|
313
|
-
/** Pattern for detecting data-extraction keywords in goal text */
|
|
314
|
-
const DATA_EXTRACTION_PATTERN = /\b(extract|list|find|data|price|pric|names?|rating|cost|count)\b/i;
|
|
315
|
-
/** Pattern for detecting search-related roles in snapshot text */
|
|
316
|
-
const SEARCH_SNAPSHOT_PATTERN = /^\s*-\s+(?:searchbox|combobox)\s/m;
|
|
317
|
-
const FIRST_TURN_COMPACT_PROMPT = `You are a browser agent choosing the fastest safe next action.
|
|
318
|
-
|
|
319
|
-
Return ONLY valid JSON with:
|
|
320
|
-
{
|
|
321
|
-
"plan": ["step 1", "step 2"],
|
|
322
|
-
"currentStep": 0,
|
|
323
|
-
"action": { "action": "click", "selector": "@REF" },
|
|
324
|
-
"nextActions": [],
|
|
325
|
-
"reasoning": "brief reason",
|
|
326
|
-
"expectedEffect": "what should change"
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
Rules:
|
|
330
|
-
1. Use exact @ref selectors from ELEMENTS. Never invent refs.
|
|
331
|
-
2. Prefer the smallest high-signal action.
|
|
332
|
-
3. On landing pages, prefer site search, primary navigation, or an obvious goal-matching link.
|
|
333
|
-
4. If a blocker is visible, resolve it first.
|
|
334
|
-
5. Do not over-explore on the first turn.
|
|
335
|
-
6. Respond with JSON only.`;
|
|
336
|
-
const LINK_SCOUT_PROMPT = `Pick the best link from CANDIDATES to advance the GOAL. Respond with ONLY JSON:
|
|
337
|
-
{"selector":"@ref","reasoning":"brief reason","confidence":0.82}
|
|
338
|
-
Rules: use exact candidate ref, pick one, confidence 0-1, prefer first-party and text-matching links.`;
|
|
339
|
-
const DESIGN_AUDIT_PROMPT = `You are a senior product designer and UX engineer auditing a web application.
|
|
340
|
-
|
|
341
|
-
Analyze the screenshot and accessibility tree for design quality, UX issues, and visual bugs.
|
|
342
|
-
|
|
343
|
-
CHECK FOR:
|
|
344
|
-
- Layout: misaligned elements, broken grids, inconsistent spacing, overflow/clipping
|
|
345
|
-
- Typography: inconsistent font sizes, poor hierarchy, text overflow, unreadable text
|
|
346
|
-
- Colors: poor contrast (WCAG AA requires 4.5:1 for text), inconsistent color palette
|
|
347
|
-
- Spacing: inconsistent padding/margins, crowded elements, excessive whitespace
|
|
348
|
-
- Alignment: elements not vertically/horizontally aligned with their siblings
|
|
349
|
-
- Accessibility: missing labels, unclear focus indicators, keyboard traps
|
|
350
|
-
- UX: confusing navigation, hidden actions, missing feedback states, dead-end flows
|
|
351
|
-
- Visual bugs: z-index issues, overlapping elements, broken images, rendering artifacts
|
|
352
|
-
|
|
353
|
-
You will also receive CHECKPOINTS — specific conditions to verify. Include a finding for each checkpoint that fails.
|
|
354
|
-
|
|
355
|
-
For each issue found, categorize it and rate its severity:
|
|
356
|
-
- critical: blocks user flow or causes data loss
|
|
357
|
-
- major: significantly impacts usability or looks unprofessional
|
|
358
|
-
- minor: cosmetic issue, polish improvement
|
|
359
|
-
|
|
360
|
-
RESPOND WITH ONLY a JSON object:
|
|
361
|
-
{
|
|
362
|
-
"score": 7,
|
|
363
|
-
"findings": [
|
|
364
|
-
{
|
|
365
|
-
"category": "layout",
|
|
366
|
-
"severity": "major",
|
|
367
|
-
"description": "Navigation sidebar overlaps main content on narrower viewports",
|
|
368
|
-
"location": "Left sidebar, main content area",
|
|
369
|
-
"suggestion": "Add responsive breakpoint or collapse sidebar below 1024px"
|
|
370
|
-
}
|
|
371
|
-
]
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
Categories: visual-bug, layout, contrast, alignment, spacing, typography, accessibility, ux
|
|
375
|
-
Score: 1-3 = poor, 4-5 = needs work, 6-7 = acceptable, 8-9 = good, 10 = excellent`;
|
|
376
|
-
const EVALUATE_PROMPT = `You are evaluating the quality of a web page or application output.
|
|
377
|
-
|
|
378
|
-
Look at the screenshot and assess:
|
|
379
|
-
1. Visual design quality (layout, spacing, colors, typography)
|
|
380
|
-
2. Functionality completeness (does it match the intended goal?)
|
|
381
|
-
3. Professional polish (would this be acceptable in production?)
|
|
382
|
-
4. Accessibility (readable text, good contrast, clear labels)
|
|
383
|
-
5. Responsiveness indicators (proper scaling, no overflow)
|
|
384
|
-
|
|
385
|
-
Respond with ONLY a JSON object:
|
|
386
|
-
{
|
|
387
|
-
"score": 8,
|
|
388
|
-
"assessment": "Brief overall assessment",
|
|
389
|
-
"strengths": ["strength 1", "strength 2"],
|
|
390
|
-
"issues": ["issue 1", "issue 2"],
|
|
391
|
-
"suggestions": ["improvement 1", "improvement 2"]
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
Score: 1-3 = poor, 4-5 = needs work, 6-7 = acceptable, 8-9 = good, 10 = excellent`;
|
|
395
|
-
const JSON_TEXT_OUTPUT = {
|
|
396
|
-
name: 'json-text',
|
|
397
|
-
responseFormat: Promise.resolve({ type: 'json' }),
|
|
398
|
-
async parseCompleteOutput({ text }) {
|
|
399
|
-
return text;
|
|
400
|
-
},
|
|
401
|
-
async parsePartialOutput({ text }) {
|
|
402
|
-
return { partial: text };
|
|
403
|
-
},
|
|
404
|
-
createElementStreamTransform() {
|
|
405
|
-
return undefined;
|
|
406
|
-
},
|
|
407
|
-
};
|
|
7
|
+
import { generateText } from 'ai';
|
|
8
|
+
import { SYSTEM_PROMPT } from './prompts.js';
|
|
9
|
+
import { decideImpl, decideVisionImpl, parseDecision } from './decide.js';
|
|
10
|
+
import { planImpl } from './plan.js';
|
|
11
|
+
import { getModelImpl, getLanguageModelImpl, generateImpl, generationOptionsImpl, completeImpl, completeVisionImpl, } from './model-client.js';
|
|
12
|
+
import { composeSystemPromptPartsImpl, buildSystemForDecideImpl, } from './system-prompt.js';
|
|
13
|
+
import { evaluateImpl } from './tasks/evaluate.js';
|
|
14
|
+
import { recommendLinkCandidateImpl } from './tasks/link-scout.js';
|
|
15
|
+
import { verifyGoalCompletionImpl } from './tasks/goal-verification.js';
|
|
16
|
+
import { auditDesignImpl } from './tasks/design-audit.js';
|
|
17
|
+
import { extractKnowledgeImpl } from './tasks/knowledge.js';
|
|
18
|
+
export { budgetSnapshot } from './snapshot-budget.js';
|
|
408
19
|
export class Brain {
|
|
20
|
+
// Public fields below satisfy the extracted host interfaces (BrainModelHost,
|
|
21
|
+
// BrainSystemPromptHost). The free functions in model-client.ts and
|
|
22
|
+
// system-prompt.ts read them through those interfaces; `implements` makes
|
|
23
|
+
// tsc prove the surface is complete.
|
|
409
24
|
modelCache = new Map();
|
|
410
25
|
provider;
|
|
411
26
|
modelName;
|
|
27
|
+
// Public fields below additionally satisfy the extracted task host interfaces
|
|
28
|
+
// (BrainLinkScoutHost, BrainGoalVerificationHost).
|
|
412
29
|
adaptiveModelRouting;
|
|
413
30
|
navModelName;
|
|
414
31
|
navProvider;
|
|
@@ -420,6 +37,10 @@ export class Brain {
|
|
|
420
37
|
visionEnabled;
|
|
421
38
|
visionStrategy;
|
|
422
39
|
observationMode;
|
|
40
|
+
// Force streaming input on claude-code so completeVision images are not dropped
|
|
41
|
+
// (the Claude Code SDK omits image parts without it). Off by default; set only
|
|
42
|
+
// by the vision-judge wiring. See AgentConfig.claudeCodeStreamingInput.
|
|
43
|
+
claudeCodeStreamingInput;
|
|
423
44
|
llmTimeoutMs;
|
|
424
45
|
compactFirstTurn;
|
|
425
46
|
lastDecisionUrl;
|
|
@@ -459,6 +80,7 @@ export class Brain {
|
|
|
459
80
|
this.visionEnabled = config.vision !== false;
|
|
460
81
|
this.visionStrategy = config.visionStrategy ?? (this.visionEnabled ? 'always' : 'never');
|
|
461
82
|
this.observationMode = config.observationMode ?? 'dom';
|
|
83
|
+
this.claudeCodeStreamingInput = config.claudeCodeStreamingInput === true;
|
|
462
84
|
this.compactFirstTurn = config.compactFirstTurn === true;
|
|
463
85
|
this.sandboxBackendType = config.sandboxBackendType;
|
|
464
86
|
this.sandboxBackendProfile = config.sandboxBackendProfile;
|
|
@@ -480,449 +102,21 @@ export class Brain {
|
|
|
480
102
|
this.adaptiveModelRouting = true; // enable routing when executor model is set
|
|
481
103
|
}
|
|
482
104
|
}
|
|
483
|
-
resolveModelName(provider, requestedModel) {
|
|
484
|
-
return resolveProviderModelName(provider, requestedModel, {
|
|
485
|
-
sandboxBackendType: provider === 'sandbox-backend' ? this.sandboxBackendType : undefined,
|
|
486
|
-
});
|
|
487
|
-
}
|
|
488
|
-
shouldSendTemperature(modelName = this.modelName) {
|
|
489
|
-
// Reasoning models reject an explicit temperature — see the shared
|
|
490
|
-
// capability check (GPT-5, o-series, Opus 4.8+, Kimi K2.6+, DeepSeek
|
|
491
|
-
// reasoner).
|
|
492
|
-
return shouldSendTemperature(modelName);
|
|
493
|
-
}
|
|
494
105
|
generationOptions(maxOutputTokens, selection) {
|
|
495
|
-
|
|
496
|
-
const modelName = this.resolveModelName(providerName, selection?.model || this.modelName);
|
|
497
|
-
// CLI-spawning providers (codex-cli, claude-code, and zai-coding-plan
|
|
498
|
-
// when routed through claude-code) don't accept maxOutputTokens through
|
|
499
|
-
// the AI SDK shape — the subprocess controls its own output.
|
|
500
|
-
const isCliSpawning = providerName === 'codex-cli' ||
|
|
501
|
-
providerName === 'claude-code' ||
|
|
502
|
-
providerName === 'sandbox-backend' ||
|
|
503
|
-
(providerName === 'zai-coding-plan' && isClaudeCodeRoutedModel(modelName));
|
|
504
|
-
const omitsLegacyMaxTokens = providerName === 'cli-bridge' || /(^|\/)gpt-5(?:[.-]|$)/i.test(modelName);
|
|
505
|
-
return {
|
|
506
|
-
...(this.shouldSendTemperature(modelName) ? { temperature: 0 } : {}),
|
|
507
|
-
...(isCliSpawning || omitsLegacyMaxTokens ? {} : { maxOutputTokens }),
|
|
508
|
-
// forceReasoning routes the AI SDK to OpenAI's Responses API
|
|
509
|
-
// (`/v1/responses`). Most third-party OpenAI-compatible proxies
|
|
510
|
-
// (router.tangle.tools, LiteLLM, Together, vLLM, etc.) only implement
|
|
511
|
-
// /v1/chat/completions — Responses API requests come back 503 / 4xx
|
|
512
|
-
// and the SDK throws "Invalid JSON response". Disable on proxied
|
|
513
|
-
// openai routes; only OpenAI direct supports the Responses API today.
|
|
514
|
-
...(providerName === 'openai'
|
|
515
|
-
&& /(^|\/)gpt-5(?:[.-]|$)/i.test(modelName)
|
|
516
|
-
&& !this.isProxiedOpenAI(providerName)
|
|
517
|
-
? {
|
|
518
|
-
providerOptions: {
|
|
519
|
-
openai: {
|
|
520
|
-
forceReasoning: true,
|
|
521
|
-
maxCompletionTokens: maxOutputTokens,
|
|
522
|
-
},
|
|
523
|
-
},
|
|
524
|
-
}
|
|
525
|
-
: {}),
|
|
526
|
-
};
|
|
527
|
-
}
|
|
528
|
-
/**
|
|
529
|
-
* True iff we're hitting an OpenAI-compatible *proxy* (router.tangle.tools,
|
|
530
|
-
* LiteLLM, etc.) rather than OpenAI direct. Used to downshift to the lowest
|
|
531
|
-
* common denominator API surface — chat-completions, no Responses API,
|
|
532
|
-
* non-streaming — that every OpenAI-compatible proxy is guaranteed to serve.
|
|
533
|
-
*
|
|
534
|
-
* The single source of truth for "we're talking to a proxy, be conservative
|
|
535
|
-
* about API features"; both `createForceNonStreamingFetch()` and the
|
|
536
|
-
* `forceReasoning` gate route through this predicate.
|
|
537
|
-
*/
|
|
538
|
-
isProxiedOpenAI(providerName) {
|
|
539
|
-
return providerName === 'openai' && Boolean(this.baseUrl);
|
|
106
|
+
return generationOptionsImpl(this, maxOutputTokens, selection);
|
|
540
107
|
}
|
|
541
108
|
/** Get a LLM model instance, optionally with provider/model override (e.g. for CAPTCHA fallback) */
|
|
542
109
|
async getLanguageModel(selection) {
|
|
543
|
-
return this
|
|
110
|
+
return getLanguageModelImpl(this, selection);
|
|
544
111
|
}
|
|
545
112
|
/** Lazily create the LLM model instance based on provider config */
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const modelName = this.resolveModelName(providerName, selection?.model || this.modelName);
|
|
549
|
-
const apiKey = resolveProviderApiKey(providerName, this.explicitApiKey);
|
|
550
|
-
const cacheKey = `${providerName}:${modelName}`;
|
|
551
|
-
const cached = this.modelCache.get(cacheKey);
|
|
552
|
-
if (cached)
|
|
553
|
-
return cached;
|
|
554
|
-
let model;
|
|
555
|
-
switch (providerName) {
|
|
556
|
-
case 'anthropic': {
|
|
557
|
-
const { createAnthropic } = await import('@ai-sdk/anthropic');
|
|
558
|
-
const provider = createAnthropic({
|
|
559
|
-
apiKey,
|
|
560
|
-
...(this.baseUrl ? { baseURL: this.baseUrl } : {}),
|
|
561
|
-
});
|
|
562
|
-
model = provider(modelName);
|
|
563
|
-
break;
|
|
564
|
-
}
|
|
565
|
-
case 'google': {
|
|
566
|
-
const { createGoogleGenerativeAI } = await import('@ai-sdk/google');
|
|
567
|
-
const provider = createGoogleGenerativeAI({
|
|
568
|
-
apiKey,
|
|
569
|
-
...(this.baseUrl ? { baseURL: this.baseUrl } : {}),
|
|
570
|
-
});
|
|
571
|
-
model = provider(modelName);
|
|
572
|
-
break;
|
|
573
|
-
}
|
|
574
|
-
case 'cli-bridge': {
|
|
575
|
-
const { createOpenAI } = await import('@ai-sdk/openai');
|
|
576
|
-
const rawUrl = this.baseUrl || process.env.CLI_BRIDGE_URL;
|
|
577
|
-
if (!rawUrl) {
|
|
578
|
-
throw new Error('cli-bridge provider requires CLI_BRIDGE_URL or --base-url');
|
|
579
|
-
}
|
|
580
|
-
const baseURL = rawUrl.endsWith('/v1') ? rawUrl : `${rawUrl.replace(/\/+$/, '')}/v1`;
|
|
581
|
-
const provider = createOpenAI({
|
|
582
|
-
apiKey: apiKey || '',
|
|
583
|
-
baseURL,
|
|
584
|
-
});
|
|
585
|
-
model = provider.chat(modelName);
|
|
586
|
-
break;
|
|
587
|
-
}
|
|
588
|
-
case 'codex-cli': {
|
|
589
|
-
const { codexExec } = await import('ai-sdk-provider-codex-cli');
|
|
590
|
-
const env = {};
|
|
591
|
-
if (apiKey)
|
|
592
|
-
env.OPENAI_API_KEY = apiKey;
|
|
593
|
-
model = codexExec(modelName, {
|
|
594
|
-
allowNpx: process.env.CODEX_ALLOW_NPX !== '0',
|
|
595
|
-
skipGitRepoCheck: true,
|
|
596
|
-
...(process.env.CODEX_CLI_PATH ? { codexPath: process.env.CODEX_CLI_PATH } : {}),
|
|
597
|
-
...(Object.keys(env).length > 0 ? { env } : {}),
|
|
598
|
-
});
|
|
599
|
-
break;
|
|
600
|
-
}
|
|
601
|
-
case 'claude-code': {
|
|
602
|
-
const { createClaudeCode } = await import('ai-sdk-provider-claude-code');
|
|
603
|
-
const env = {};
|
|
604
|
-
if (apiKey)
|
|
605
|
-
env.ANTHROPIC_API_KEY = apiKey;
|
|
606
|
-
const provider = createClaudeCode({
|
|
607
|
-
defaultSettings: {
|
|
608
|
-
...(process.env.CLAUDE_CODE_CLI_PATH ? { pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_CLI_PATH } : {}),
|
|
609
|
-
permissionMode: 'default',
|
|
610
|
-
allowDangerouslySkipPermissions: false,
|
|
611
|
-
...(this.debug ? { verbose: true } : {}),
|
|
612
|
-
...(this.debug
|
|
613
|
-
? {
|
|
614
|
-
stderr: (chunk) => {
|
|
615
|
-
const line = chunk.trim();
|
|
616
|
-
if (line)
|
|
617
|
-
console.error(`[claude-code] ${line}`);
|
|
618
|
-
},
|
|
619
|
-
}
|
|
620
|
-
: {}),
|
|
621
|
-
...(Object.keys(env).length > 0 ? { env } : {}),
|
|
622
|
-
},
|
|
623
|
-
});
|
|
624
|
-
model = provider(modelName);
|
|
625
|
-
break;
|
|
626
|
-
}
|
|
627
|
-
case 'zai-coding-plan': {
|
|
628
|
-
// Z.ai coding plan: GLM models (glm-4.6, glm-4.5-air) at a fraction
|
|
629
|
-
// of Anthropic prices. Two backends:
|
|
630
|
-
// 1. Default — OpenAI-compatible endpoint, native GLM models
|
|
631
|
-
// 2. claude-code routed — spawn the Claude Code CLI subprocess
|
|
632
|
-
// with ANTHROPIC_BASE_URL/AUTH_TOKEN env vars pointed at Z.ai's
|
|
633
|
-
// Anthropic-compatible endpoint. The user gets the Claude Code
|
|
634
|
-
// agent loop with Z.ai pricing.
|
|
635
|
-
if (!apiKey) {
|
|
636
|
-
throw new Error('zai-coding-plan: API key required (set ZAI_API_KEY env var or pass --api-key)');
|
|
637
|
-
}
|
|
638
|
-
if (isClaudeCodeRoutedModel(modelName)) {
|
|
639
|
-
const { createClaudeCode } = await import('ai-sdk-provider-claude-code');
|
|
640
|
-
const env = {
|
|
641
|
-
// Override Claude Code's API base + auth at the env-var level.
|
|
642
|
-
// The CLI subprocess inherits these and routes all Anthropic
|
|
643
|
-
// API calls to Z.ai's Anthropic-compatible endpoint.
|
|
644
|
-
ANTHROPIC_BASE_URL: ZAI_ANTHROPIC_BASE_URL,
|
|
645
|
-
ANTHROPIC_AUTH_TOKEN: apiKey,
|
|
646
|
-
// Some Claude Code versions still read ANTHROPIC_API_KEY — set
|
|
647
|
-
// both so we don't depend on which env var name wins.
|
|
648
|
-
ANTHROPIC_API_KEY: apiKey,
|
|
649
|
-
};
|
|
650
|
-
const provider = createClaudeCode({
|
|
651
|
-
defaultSettings: {
|
|
652
|
-
...(process.env.CLAUDE_CODE_CLI_PATH ? { pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_CLI_PATH } : {}),
|
|
653
|
-
permissionMode: 'default',
|
|
654
|
-
allowDangerouslySkipPermissions: false,
|
|
655
|
-
...(this.debug ? { verbose: true } : {}),
|
|
656
|
-
...(this.debug
|
|
657
|
-
? {
|
|
658
|
-
stderr: (chunk) => {
|
|
659
|
-
const line = chunk.trim();
|
|
660
|
-
if (line)
|
|
661
|
-
console.error(`[zai-claude-code] ${line}`);
|
|
662
|
-
},
|
|
663
|
-
}
|
|
664
|
-
: {}),
|
|
665
|
-
env,
|
|
666
|
-
},
|
|
667
|
-
});
|
|
668
|
-
// The Claude Code SDK takes a model alias (sonnet/haiku/opus); when
|
|
669
|
-
// the user passes a glm-* name we just hand it through — Claude
|
|
670
|
-
// Code passes it as the model param to the upstream API, which is
|
|
671
|
-
// Z.ai's Anthropic-compat endpoint that maps it to the right glm.
|
|
672
|
-
// For 'claude-code'/'cc' we default to 'sonnet' which Z.ai aliases
|
|
673
|
-
// to its strongest glm model under the coding plan.
|
|
674
|
-
const ccModel = modelName === 'claude-code' || modelName === 'cc' ? 'sonnet' : modelName;
|
|
675
|
-
model = provider(ccModel);
|
|
676
|
-
break;
|
|
677
|
-
}
|
|
678
|
-
// Default path — direct OpenAI-compatible API to Z.ai. Z.ai's
|
|
679
|
-
// paas/v4 endpoint only implements `/chat/completions` (no
|
|
680
|
-
// `/responses`), so we MUST use `provider.chat(model)` rather than
|
|
681
|
-
// the bare `provider(model)` call which now defaults to the new
|
|
682
|
-
// OpenAI Responses API in @ai-sdk/openai v3+.
|
|
683
|
-
const { createOpenAI } = await import('@ai-sdk/openai');
|
|
684
|
-
const provider = createOpenAI({
|
|
685
|
-
apiKey,
|
|
686
|
-
baseURL: ZAI_OPENAI_BASE_URL,
|
|
687
|
-
});
|
|
688
|
-
model = provider.chat(modelName);
|
|
689
|
-
break;
|
|
690
|
-
}
|
|
691
|
-
default: {
|
|
692
|
-
// 'openai' or any OpenAI-compatible API (LiteLLM, Together, etc.).
|
|
693
|
-
// Same reason as the zai-coding-plan branch above: most third-party
|
|
694
|
-
// OpenAI-compatible proxies (LiteLLM, vLLM, Together) implement
|
|
695
|
-
// `/chat/completions` but not the new `/responses` API. Use the
|
|
696
|
-
// chat factory explicitly so we hit a path the upstream actually
|
|
697
|
-
// serves.
|
|
698
|
-
const { createOpenAI } = await import('@ai-sdk/openai');
|
|
699
|
-
const usingProxy = this.isProxiedOpenAI(providerName);
|
|
700
|
-
const provider = createOpenAI({
|
|
701
|
-
apiKey: apiKey || '',
|
|
702
|
-
...(this.baseUrl ? { baseURL: this.baseUrl } : {}),
|
|
703
|
-
// Proxy downshift — see isProxiedOpenAI(). Routes like
|
|
704
|
-
// router.tangle.tools default chat-completions to SSE when the
|
|
705
|
-
// client omits `stream`, and the AI SDK's generateText errors with
|
|
706
|
-
// "Invalid JSON response" on SSE. Force stream: false on every
|
|
707
|
-
// chat-completions body. Paired with the forceReasoning gate in
|
|
708
|
-
// generationOptions() so all "talk to a proxy" downshifts share
|
|
709
|
-
// one predicate.
|
|
710
|
-
...(usingProxy ? { fetch: createForceNonStreamingFetch() } : {}),
|
|
711
|
-
});
|
|
712
|
-
model = provider.chat(modelName);
|
|
713
|
-
break;
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
this.modelCache.set(cacheKey, model);
|
|
717
|
-
return model;
|
|
718
|
-
}
|
|
719
|
-
async generate(system, messages, selection, maxOutputTokens = 800) {
|
|
720
|
-
const providerName = selection?.provider || this.provider;
|
|
721
|
-
const modelName = this.resolveModelName(providerName, selection?.model || this.modelName);
|
|
722
|
-
// Sandbox backend doesn't accept structured system messages — flatten.
|
|
723
|
-
const systemForSandbox = typeof system === 'string'
|
|
724
|
-
? system
|
|
725
|
-
: system.map(m => m.content).join('\n\n');
|
|
726
|
-
if (providerName === 'sandbox-backend') {
|
|
727
|
-
const result = await generateWithSandboxBackend({
|
|
728
|
-
system: systemForSandbox,
|
|
729
|
-
messages,
|
|
730
|
-
model: modelName,
|
|
731
|
-
timeoutMs: this.llmTimeoutMs,
|
|
732
|
-
debug: this.debug,
|
|
733
|
-
backendType: this.sandboxBackendType,
|
|
734
|
-
backendProfile: this.sandboxBackendProfile,
|
|
735
|
-
backendModelProvider: this.sandboxBackendProvider,
|
|
736
|
-
});
|
|
737
|
-
return { text: result.text };
|
|
738
|
-
}
|
|
739
|
-
const model = await this.getModel({
|
|
740
|
-
provider: providerName,
|
|
741
|
-
model: modelName,
|
|
742
|
-
});
|
|
743
|
-
// Anthropic supports structured system messages with cache_control. For
|
|
744
|
-
// every other provider, flatten back to a string — they ignore the array
|
|
745
|
-
// form's per-message provider options anyway, and some (claude-code,
|
|
746
|
-
// codex-cli) wrap their own subprocess CLI which only takes plain text.
|
|
747
|
-
const systemForRequest = providerName === 'anthropic' || typeof system === 'string'
|
|
748
|
-
? system
|
|
749
|
-
: systemForSandbox;
|
|
750
|
-
const generationSettings = {
|
|
751
|
-
model,
|
|
752
|
-
system: systemForRequest,
|
|
753
|
-
messages,
|
|
754
|
-
...(providerName === 'cli-bridge' ? { output: JSON_TEXT_OUTPUT } : {}),
|
|
755
|
-
...this.generationOptions(maxOutputTokens, { provider: providerName, model: modelName }),
|
|
756
|
-
abortSignal: AbortSignal.timeout(this.llmTimeoutMs),
|
|
757
|
-
};
|
|
758
|
-
const result = providerName === 'cli-bridge'
|
|
759
|
-
? await (async () => {
|
|
760
|
-
const streamed = streamText(generationSettings);
|
|
761
|
-
const [text, usage, providerMetadata] = await Promise.all([
|
|
762
|
-
streamed.text,
|
|
763
|
-
streamed.totalUsage,
|
|
764
|
-
streamed.providerMetadata,
|
|
765
|
-
]);
|
|
766
|
-
return { text, usage, providerMetadata };
|
|
767
|
-
})()
|
|
768
|
-
: await generateText(generationSettings);
|
|
769
|
-
// Extract prompt-cache stats from the AI SDK's PROVIDER-AGNOSTIC fields:
|
|
770
|
-
// result.usage.inputTokenDetails.{cacheReadTokens, cacheWriteTokens}
|
|
771
|
-
//
|
|
772
|
-
// These flow uniformly from every provider that supports prompt caching:
|
|
773
|
-
// - OpenAI (gpt-5.4, gpt-4.1-mini): AUTOMATIC server-side caching for
|
|
774
|
-
// >1024 token prefixes, no markers needed. Returns cached_tokens.
|
|
775
|
-
// - Anthropic (claude-*): EXPLICIT cache_control markers required (we
|
|
776
|
-
// set them in buildSystemForDecide). Returns cache_read_input_tokens.
|
|
777
|
-
// - ZAI / GLM (zai-coding-plan via OpenAI-compatible endpoint): AUTOMATIC
|
|
778
|
-
// server-side caching, returns cached_tokens in prompt_tokens_details.
|
|
779
|
-
// - Google Gemini: explicit `cachedContent` ID-based caching (different
|
|
780
|
-
// paradigm — not currently exercised).
|
|
781
|
-
//
|
|
782
|
-
// The unified inputTokenDetails fields mean we get cache observability
|
|
783
|
-
// for free across providers without per-provider extraction code.
|
|
784
|
-
const inputDetails = result.usage?.inputTokenDetails;
|
|
785
|
-
let cacheReadInputTokens = typeof inputDetails?.cacheReadTokens === 'number'
|
|
786
|
-
? inputDetails.cacheReadTokens
|
|
787
|
-
: undefined;
|
|
788
|
-
let cacheCreationInputTokens = typeof inputDetails?.cacheWriteTokens === 'number'
|
|
789
|
-
? inputDetails.cacheWriteTokens
|
|
790
|
-
: undefined;
|
|
791
|
-
// Fallback: some providers expose cache stats only via providerMetadata.
|
|
792
|
-
// Anthropic uses cacheReadInputTokens / cacheCreationInputTokens directly,
|
|
793
|
-
// OpenAI sometimes lands cached_tokens under providerMetadata.openai.
|
|
794
|
-
if (cacheReadInputTokens === undefined || cacheCreationInputTokens === undefined) {
|
|
795
|
-
const meta = result.providerMetadata;
|
|
796
|
-
const anthropicMeta = meta?.anthropic;
|
|
797
|
-
const openaiMeta = meta?.openai;
|
|
798
|
-
if (cacheReadInputTokens === undefined) {
|
|
799
|
-
const fromAnthropic = typeof anthropicMeta?.cacheReadInputTokens === 'number'
|
|
800
|
-
? anthropicMeta.cacheReadInputTokens
|
|
801
|
-
: undefined;
|
|
802
|
-
const fromOpenAI = typeof openaiMeta?.cachedPromptTokens === 'number'
|
|
803
|
-
? openaiMeta.cachedPromptTokens
|
|
804
|
-
: undefined;
|
|
805
|
-
cacheReadInputTokens = fromAnthropic ?? fromOpenAI;
|
|
806
|
-
}
|
|
807
|
-
if (cacheCreationInputTokens === undefined) {
|
|
808
|
-
const fromAnthropic = typeof anthropicMeta?.cacheCreationInputTokens === 'number'
|
|
809
|
-
? anthropicMeta.cacheCreationInputTokens
|
|
810
|
-
: undefined;
|
|
811
|
-
cacheCreationInputTokens = fromAnthropic;
|
|
812
|
-
}
|
|
813
|
-
}
|
|
814
|
-
return {
|
|
815
|
-
text: result.text,
|
|
816
|
-
tokensUsed: result.usage?.totalTokens,
|
|
817
|
-
inputTokens: result.usage?.inputTokens ?? undefined,
|
|
818
|
-
outputTokens: result.usage?.outputTokens ?? undefined,
|
|
819
|
-
cacheReadInputTokens,
|
|
820
|
-
cacheCreationInputTokens,
|
|
821
|
-
};
|
|
822
|
-
}
|
|
823
|
-
/**
|
|
824
|
-
* Classify whether this turn should use the nav (cheap) model for decide().
|
|
825
|
-
*
|
|
826
|
-
* Empirically tested: routing early navigation turns to gpt-4.1-mini causes
|
|
827
|
-
* worse decisions that cascade into longer runs (more turns = more total cost).
|
|
828
|
-
* gpt-5.4 without routing is cheaper overall because it navigates more efficiently.
|
|
829
|
-
*
|
|
830
|
-
* Current strategy: nav model is ONLY used for verification (see verifyGoalCompletion),
|
|
831
|
-
* not for decide(). The flag is kept for future experiments with better routing signals.
|
|
832
|
-
*/
|
|
833
|
-
shouldUseNavigationModel(state, extraContext, turnInfo) {
|
|
834
|
-
if (!this.adaptiveModelRouting || !this.navModelName)
|
|
835
|
-
return false;
|
|
836
|
-
// Use the navigation model for DOM-only same-page turns. Keep the primary
|
|
837
|
-
// model for first turns, new pages, and error recovery.
|
|
838
|
-
const isFirstTurn = !turnInfo || turnInfo.current <= 1;
|
|
839
|
-
const samePageAsPrevious = this.lastDecisionUrl === state.url;
|
|
840
|
-
const hasError = extraContext?.includes('REJECTED') || extraContext?.includes('ERROR');
|
|
841
|
-
if (isFirstTurn || !samePageAsPrevious || hasError)
|
|
842
|
-
return false;
|
|
843
|
-
return true;
|
|
113
|
+
getModel(selection) {
|
|
114
|
+
return getModelImpl(this, selection);
|
|
844
115
|
}
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
*/
|
|
850
|
-
buildSystemPrompt(goal, state, turn) {
|
|
851
|
-
return this.composeSystemPromptParts(goal, state, turn).join('');
|
|
852
|
-
}
|
|
853
|
-
/**
|
|
854
|
-
* Same as buildSystemPrompt but returns the parts so the caller can decide
|
|
855
|
-
* how to send them. For Anthropic, decide() ships them as a SystemModelMessage[]
|
|
856
|
-
* with cache_control on the stable CORE_RULES prefix; other providers join.
|
|
857
|
-
*
|
|
858
|
-
* The first slot is ALWAYS CORE_RULES (or the user's custom override) so the
|
|
859
|
-
* cache breakpoint placement is deterministic. Extension-supplied rules are
|
|
860
|
-
* appended AFTER REASONING_SUFFIX so the cached prefix stays byte-stable
|
|
861
|
-
* across turns.
|
|
862
|
-
*/
|
|
863
|
-
composeSystemPromptParts(goal, state, turn) {
|
|
864
|
-
if (this.systemPrompt !== SYSTEM_PROMPT)
|
|
865
|
-
return [this.systemPrompt];
|
|
866
|
-
const parts = [CORE_RULES];
|
|
867
|
-
const snapshotSample = state.snapshot.length > 4000 ? state.snapshot.slice(0, 4000) : state.snapshot;
|
|
868
|
-
if (SEARCH_SNAPSHOT_PATTERN.test(snapshotSample) || /\/search\b/i.test(state.url)) {
|
|
869
|
-
parts.push(SEARCH_RULES);
|
|
870
|
-
if (this.extensionRules?.search) {
|
|
871
|
-
parts.push(`\n\nUSER RULES (search):\n${this.extensionRules.search}`);
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
if (DATA_EXTRACTION_PATTERN.test(goal)) {
|
|
875
|
-
parts.push(DATA_EXTRACTION_RULES);
|
|
876
|
-
if (this.extensionRules?.dataExtraction) {
|
|
877
|
-
parts.push(`\n\nUSER RULES (data extraction):\n${this.extensionRules.dataExtraction}`);
|
|
878
|
-
}
|
|
879
|
-
}
|
|
880
|
-
if (state.snapshot.length > 10_000 || turn > 10) {
|
|
881
|
-
parts.push(HEAVY_PAGE_RULES);
|
|
882
|
-
if (this.extensionRules?.heavy) {
|
|
883
|
-
parts.push(`\n\nUSER RULES (heavy page):\n${this.extensionRules.heavy}`);
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
parts.push(REASONING_SUFFIX);
|
|
887
|
-
// Global user rules + matching per-domain rules. Both are appended AFTER
|
|
888
|
-
// REASONING_SUFFIX so they don't pollute the byte-stable cached prefix.
|
|
889
|
-
if (this.extensionRules?.global) {
|
|
890
|
-
parts.push(`\n\nUSER RULES (global):\n${this.extensionRules.global}`);
|
|
891
|
-
}
|
|
892
|
-
if (this.extensionDomainRules) {
|
|
893
|
-
const domainRules = this.matchDomainRules(state.url);
|
|
894
|
-
if (domainRules) {
|
|
895
|
-
parts.push(`\n\nUSER RULES (domain match):\n${domainRules}`);
|
|
896
|
-
}
|
|
897
|
-
}
|
|
898
|
-
// Macros live AFTER the cached prefix so registering new macros
|
|
899
|
-
// doesn't bust the Anthropic cache.
|
|
900
|
-
if (this.macroPromptBlock) {
|
|
901
|
-
parts.push(`\n\n${this.macroPromptBlock}`);
|
|
902
|
-
}
|
|
903
|
-
return parts;
|
|
904
|
-
}
|
|
905
|
-
/**
|
|
906
|
-
* Find the per-domain extra rules whose domain key matches the URL host.
|
|
907
|
-
* Multiple matches are concatenated in registration order.
|
|
908
|
-
*/
|
|
909
|
-
matchDomainRules(url) {
|
|
910
|
-
if (!this.extensionDomainRules)
|
|
911
|
-
return undefined;
|
|
912
|
-
let host;
|
|
913
|
-
try {
|
|
914
|
-
host = new URL(url).hostname;
|
|
915
|
-
}
|
|
916
|
-
catch {
|
|
917
|
-
return undefined;
|
|
918
|
-
}
|
|
919
|
-
const matches = [];
|
|
920
|
-
for (const [domain, rules] of Object.entries(this.extensionDomainRules)) {
|
|
921
|
-
if (host.includes(domain) && rules.extraRules) {
|
|
922
|
-
matches.push(rules.extraRules);
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
|
-
return matches.length > 0 ? matches.join('\n\n') : undefined;
|
|
116
|
+
// Public so the extracted task impls (src/brain/tasks/*) can reach it through
|
|
117
|
+
// their host interfaces. The class still owns the single transport funnel.
|
|
118
|
+
generate(system, messages, selection, maxOutputTokens = 800) {
|
|
119
|
+
return generateImpl(this, system, messages, selection, maxOutputTokens);
|
|
926
120
|
}
|
|
927
121
|
/**
|
|
928
122
|
* Inject extension-supplied rules. Called by the runner after loading
|
|
@@ -937,6 +131,9 @@ export class Brain {
|
|
|
937
131
|
setMacroPromptBlock(block) {
|
|
938
132
|
this.macroPromptBlock = block ?? '';
|
|
939
133
|
}
|
|
134
|
+
composeSystemPromptParts(goal, state, turn) {
|
|
135
|
+
return composeSystemPromptPartsImpl(this, goal, state, turn);
|
|
136
|
+
}
|
|
940
137
|
/**
|
|
941
138
|
* Build the system prompt for `decide()` in the form best suited to the
|
|
942
139
|
* active provider:
|
|
@@ -949,25 +146,7 @@ export class Brain {
|
|
|
949
146
|
* is opt-in via the default prompt path only.
|
|
950
147
|
*/
|
|
951
148
|
buildSystemForDecide(goal, state, turn, providerName) {
|
|
952
|
-
|
|
953
|
-
if (providerName !== 'anthropic' || this.systemPrompt !== SYSTEM_PROMPT || parts.length === 0) {
|
|
954
|
-
return parts.join('');
|
|
955
|
-
}
|
|
956
|
-
// Anthropic path: first slot is CORE_RULES (cached), remaining parts ship
|
|
957
|
-
// as a separate uncached system message so the prefix stays byte-stable
|
|
958
|
-
// across turns and the cache hits.
|
|
959
|
-
const corePart = {
|
|
960
|
-
role: 'system',
|
|
961
|
-
content: parts[0],
|
|
962
|
-
providerOptions: {
|
|
963
|
-
anthropic: {
|
|
964
|
-
cacheControl: { type: 'ephemeral' },
|
|
965
|
-
},
|
|
966
|
-
},
|
|
967
|
-
};
|
|
968
|
-
if (parts.length === 1)
|
|
969
|
-
return [corePart];
|
|
970
|
-
return [corePart, { role: 'system', content: parts.slice(1).join('') }];
|
|
149
|
+
return buildSystemForDecideImpl(this, goal, state, turn, providerName);
|
|
971
150
|
}
|
|
972
151
|
/** Reset conversation history (call between scenarios) */
|
|
973
152
|
reset() {
|
|
@@ -1021,6 +200,8 @@ export class Brain {
|
|
|
1021
200
|
* Build the user message content parts — text + optional screenshot.
|
|
1022
201
|
* Multimodal when vision is enabled and screenshot is available.
|
|
1023
202
|
*/
|
|
203
|
+
// Public so the extracted task impls (src/brain/tasks/*) can reach it through
|
|
204
|
+
// their host interfaces.
|
|
1024
205
|
buildUserContent(text, screenshot, forceVision = false) {
|
|
1025
206
|
const shouldUseVision = !!screenshot && (this.visionStrategy === 'always'
|
|
1026
207
|
|| (this.visionStrategy === 'auto' && forceVision));
|
|
@@ -1036,327 +217,8 @@ export class Brain {
|
|
|
1036
217
|
},
|
|
1037
218
|
];
|
|
1038
219
|
}
|
|
1039
|
-
/**
|
|
1040
|
-
* Compact conversation history: strip ELEMENTS blocks and screenshots
|
|
1041
|
-
* from older observations, keeping the last 2 user messages intact.
|
|
1042
|
-
*
|
|
1043
|
-
* For older turns, replaces the full ELEMENTS block with a one-line
|
|
1044
|
-
* summary showing element count and the selectors the agent actually
|
|
1045
|
-
* used, extracted from the paired assistant response.
|
|
1046
|
-
*/
|
|
1047
|
-
compactHistory() {
|
|
1048
|
-
if (this.history.length === 0)
|
|
1049
|
-
return [];
|
|
1050
|
-
// Find indices of the last 2 user messages to keep intact
|
|
1051
|
-
const userIndices = [];
|
|
1052
|
-
for (let i = this.history.length - 1; i >= 0; i--) {
|
|
1053
|
-
if (this.history[i].role === 'user') {
|
|
1054
|
-
userIndices.push(i);
|
|
1055
|
-
if (userIndices.length === 2)
|
|
1056
|
-
break;
|
|
1057
|
-
}
|
|
1058
|
-
}
|
|
1059
|
-
const keepIntactFrom = userIndices.length > 0
|
|
1060
|
-
? userIndices[userIndices.length - 1]
|
|
1061
|
-
: this.history.length;
|
|
1062
|
-
// Three-tier compression:
|
|
1063
|
-
// Zone 1 (intact): last 2 turns — full content
|
|
1064
|
-
// Zone 2 (standard): turns 3-5 back — ELEMENTS stripped from user msgs
|
|
1065
|
-
// Zone 3 (deep compact): turns 6+ back — both user and assistant ultra-compacted
|
|
1066
|
-
// Keep enough older history for long multi-step travel workflows.
|
|
1067
|
-
const deepCompactBefore = Math.max(0, this.history.length - 10);
|
|
1068
|
-
return this.history.map((msg, idx) => {
|
|
1069
|
-
// Zone 1: keep recent turns intact
|
|
1070
|
-
if (idx >= keepIntactFrom)
|
|
1071
|
-
return msg;
|
|
1072
|
-
// Zone 3: ultra-compact for very old messages (user + assistant)
|
|
1073
|
-
if (idx < deepCompactBefore) {
|
|
1074
|
-
if (msg.role === 'assistant') {
|
|
1075
|
-
const raw = typeof msg.content === 'string' ? msg.content : '';
|
|
1076
|
-
return { ...msg, content: this.deepCompactAssistant(raw) };
|
|
1077
|
-
}
|
|
1078
|
-
if (msg.role === 'user') {
|
|
1079
|
-
return { ...msg, content: this.deepCompactUser(msg) };
|
|
1080
|
-
}
|
|
1081
|
-
return msg;
|
|
1082
|
-
}
|
|
1083
|
-
// Zone 2: standard compact — strip ELEMENTS from user messages only
|
|
1084
|
-
if (msg.role !== 'user')
|
|
1085
|
-
return msg;
|
|
1086
|
-
const assistantMsg = idx + 1 < this.history.length ? this.history[idx + 1] : undefined;
|
|
1087
|
-
const selectors = assistantMsg?.role === 'assistant'
|
|
1088
|
-
? this.extractSelectorsFromResponse(typeof assistantMsg.content === 'string' ? assistantMsg.content : '')
|
|
1089
|
-
: [];
|
|
1090
|
-
// Handle multimodal content (array of parts)
|
|
1091
|
-
if (Array.isArray(msg.content)) {
|
|
1092
|
-
const compacted = msg.content
|
|
1093
|
-
.filter((part) => part.type === 'text')
|
|
1094
|
-
.map((part) => ({
|
|
1095
|
-
...part,
|
|
1096
|
-
text: this.summarizeElements(part.text, selectors),
|
|
1097
|
-
}));
|
|
1098
|
-
return { ...msg, content: compacted };
|
|
1099
|
-
}
|
|
1100
|
-
// Handle string content
|
|
1101
|
-
if (typeof msg.content === 'string') {
|
|
1102
|
-
return { ...msg, content: this.summarizeElements(msg.content, selectors) };
|
|
1103
|
-
}
|
|
1104
|
-
return msg;
|
|
1105
|
-
});
|
|
1106
|
-
}
|
|
1107
|
-
deepCompactUser(msg) {
|
|
1108
|
-
const text = typeof msg.content === 'string'
|
|
1109
|
-
? msg.content
|
|
1110
|
-
: Array.isArray(msg.content)
|
|
1111
|
-
? msg.content
|
|
1112
|
-
.filter((p) => p.type === 'text')
|
|
1113
|
-
.map((p) => p.text)
|
|
1114
|
-
.join('\n')
|
|
1115
|
-
: '';
|
|
1116
|
-
const urlMatch = text.match(/URL:\s*(\S+)/);
|
|
1117
|
-
const titleMatch = text.match(/Title:\s*(.+?)(?:\n|$)/);
|
|
1118
|
-
const url = urlMatch?.[1] ?? 'unknown';
|
|
1119
|
-
const title = titleMatch?.[1]?.slice(0, 80) ?? '';
|
|
1120
|
-
return `[Prior turn — URL: ${url}${title ? ` | ${title}` : ''}]`;
|
|
1121
|
-
}
|
|
1122
|
-
deepCompactAssistant(raw) {
|
|
1123
|
-
try {
|
|
1124
|
-
let text = raw.trim();
|
|
1125
|
-
if (text.startsWith('```')) {
|
|
1126
|
-
text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
1127
|
-
}
|
|
1128
|
-
const parsed = JSON.parse(text);
|
|
1129
|
-
const action = parsed.action?.action ?? 'unknown';
|
|
1130
|
-
const selector = parsed.action?.selector ?? '';
|
|
1131
|
-
const parts = [action];
|
|
1132
|
-
if (selector)
|
|
1133
|
-
parts.push(selector);
|
|
1134
|
-
if (parsed.action?.url)
|
|
1135
|
-
parts.push(parsed.action.url.slice(0, 120));
|
|
1136
|
-
return `[${parts.join(' → ')}]`;
|
|
1137
|
-
}
|
|
1138
|
-
catch {
|
|
1139
|
-
return raw.slice(0, 100) + (raw.length > 100 ? '…' : '');
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
/**
|
|
1143
|
-
* Extract @ref selectors from an assistant JSON response.
|
|
1144
|
-
*/
|
|
1145
|
-
extractSelectorsFromResponse(raw) {
|
|
1146
|
-
const selectors = [];
|
|
1147
|
-
try {
|
|
1148
|
-
let text = raw.trim();
|
|
1149
|
-
if (text.startsWith('```')) {
|
|
1150
|
-
text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
1151
|
-
}
|
|
1152
|
-
const parsed = JSON.parse(text);
|
|
1153
|
-
if (parsed.action?.selector)
|
|
1154
|
-
selectors.push(parsed.action.selector);
|
|
1155
|
-
if (Array.isArray(parsed.nextActions)) {
|
|
1156
|
-
for (const na of parsed.nextActions) {
|
|
1157
|
-
if (na?.selector)
|
|
1158
|
-
selectors.push(na.selector);
|
|
1159
|
-
}
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
catch {
|
|
1163
|
-
// Best effort
|
|
1164
|
-
}
|
|
1165
|
-
return selectors;
|
|
1166
|
-
}
|
|
1167
|
-
/**
|
|
1168
|
-
* Replace the ELEMENTS block with a one-line action-only summary.
|
|
1169
|
-
*/
|
|
1170
|
-
summarizeElements(text, selectors) {
|
|
1171
|
-
return text.replace(/ELEMENTS[^:\n]*:\n[\s\S]*?(?=\n\n|What action should you take\?|$)/, (match) => {
|
|
1172
|
-
const snapshotStart = match.indexOf('\n');
|
|
1173
|
-
if (snapshotStart === -1)
|
|
1174
|
-
return 'ELEMENTS:\n[previous snapshot]';
|
|
1175
|
-
const snapshotText = match.slice(snapshotStart + 1);
|
|
1176
|
-
const elementCount = (snapshotText.match(/\[ref=\w+\]/g) || []).length;
|
|
1177
|
-
const selectorList = selectors.length > 0
|
|
1178
|
-
? selectors.join(', ')
|
|
1179
|
-
: 'none';
|
|
1180
|
-
return `ELEMENTS:\n[Page snapshot: ${elementCount} elements | agent used: ${selectorList}]`;
|
|
1181
|
-
});
|
|
1182
|
-
}
|
|
1183
220
|
async decide(goal, state, extraContext, turnInfo, options) {
|
|
1184
|
-
|
|
1185
|
-
if (this.observationMode === 'vision' || this.observationMode === 'hybrid') {
|
|
1186
|
-
return this.decideVision(goal, state, extraContext, turnInfo);
|
|
1187
|
-
}
|
|
1188
|
-
const useCompactFirstTurn = this.compactFirstTurn && turnInfo?.current === 1;
|
|
1189
|
-
const samePageAsPrevious = this.lastDecisionUrl === state.url;
|
|
1190
|
-
const isFirstTurn = !turnInfo || turnInfo.current <= 1;
|
|
1191
|
-
// Diff-only mode: on same-page turns with small diffs, send only changed
|
|
1192
|
-
// elements instead of the full snapshot. Saves 40-80% of input tokens on
|
|
1193
|
-
// form-fill / interaction-heavy pages where the page structure is stable.
|
|
1194
|
-
const rawDiff = state.snapshotDiffRaw;
|
|
1195
|
-
const diffChanges = rawDiff ? rawDiff.added.length + rawDiff.removed.length + rawDiff.changed.length : 0;
|
|
1196
|
-
const diffTotal = rawDiff ? diffChanges + rawDiff.unchangedCount : 0;
|
|
1197
|
-
const useDiffOnly = samePageAsPrevious
|
|
1198
|
-
&& !isFirstTurn
|
|
1199
|
-
&& rawDiff !== undefined
|
|
1200
|
-
&& diffChanges > 0
|
|
1201
|
-
&& diffTotal > 0
|
|
1202
|
-
&& diffChanges / diffTotal < 0.3;
|
|
1203
|
-
// Tighter snapshot budget on same-page turns; new pages keep enough
|
|
1204
|
-
// content for extraction from docs/spec pages.
|
|
1205
|
-
const snapshotBudget = samePageAsPrevious ? 8_000 : 24_000;
|
|
1206
|
-
let visibleSnapshot;
|
|
1207
|
-
let elementsHeader;
|
|
1208
|
-
if (useDiffOnly) {
|
|
1209
|
-
// Build compact diff-only view: changed/added elements with refs
|
|
1210
|
-
const lines = [];
|
|
1211
|
-
if (rawDiff.added.length)
|
|
1212
|
-
lines.push('ADDED:', ...rawDiff.added);
|
|
1213
|
-
if (rawDiff.changed.length)
|
|
1214
|
-
lines.push('CHANGED:', ...rawDiff.changed);
|
|
1215
|
-
if (rawDiff.removed.length)
|
|
1216
|
-
lines.push('REMOVED:', ...rawDiff.removed);
|
|
1217
|
-
lines.push(`(${rawDiff.unchangedCount} elements unchanged — refs from previous turn still valid)`);
|
|
1218
|
-
visibleSnapshot = lines.join('\n');
|
|
1219
|
-
elementsHeader = 'ELEMENTS (diff-only, previous refs still valid)';
|
|
1220
|
-
}
|
|
1221
|
-
else {
|
|
1222
|
-
visibleSnapshot = useCompactFirstTurn
|
|
1223
|
-
? compactFirstTurnSnapshot(state.snapshot)
|
|
1224
|
-
: budgetSnapshot(state.snapshot, snapshotBudget);
|
|
1225
|
-
elementsHeader = 'ELEMENTS';
|
|
1226
|
-
}
|
|
1227
|
-
this.lastDecisionUrl = state.url;
|
|
1228
|
-
// Build user message with stable prefix (GOAL) for prompt caching,
|
|
1229
|
-
// then dynamic per-turn content (turn budget, page state, elements).
|
|
1230
|
-
let textContent = `GOAL: ${goal}
|
|
1231
|
-
|
|
1232
|
-
CURRENT PAGE:
|
|
1233
|
-
URL: ${state.url}
|
|
1234
|
-
Title: ${state.title}
|
|
1235
|
-
|
|
1236
|
-
${elementsHeader}:
|
|
1237
|
-
${visibleSnapshot}`;
|
|
1238
|
-
if (turnInfo) {
|
|
1239
|
-
const remaining = turnInfo.max - turnInfo.current;
|
|
1240
|
-
const budgetUsed = turnInfo.current / turnInfo.max;
|
|
1241
|
-
textContent += `\n\nTURN: ${turnInfo.current}/${turnInfo.max} (${remaining} remaining)`;
|
|
1242
|
-
if (remaining === 1) {
|
|
1243
|
-
textContent += ` — FINAL TURN: return a terminal action only (complete or abort)`;
|
|
1244
|
-
}
|
|
1245
|
-
else if (remaining <= 3) {
|
|
1246
|
-
textContent += ` — RUNNING LOW, avoid exploratory navigation; prioritize completing the goal or aborting with a clear blocker reason`;
|
|
1247
|
-
}
|
|
1248
|
-
else if (budgetUsed >= 0.5) {
|
|
1249
|
-
textContent += ` — HALF BUDGET USED. If you have extracted useful data, try completing now. Do not navigate away from pages with relevant content without attempting completion first`;
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
// Append snapshot diff only when NOT using diff-only mode (avoid redundant info)
|
|
1253
|
-
if (!useDiffOnly && state.snapshotDiff && state.snapshotDiff.length < state.snapshot.length * 0.3) {
|
|
1254
|
-
textContent += `\n\nSNAPSHOT CHANGES (since last turn):\n${state.snapshotDiff}`;
|
|
1255
|
-
}
|
|
1256
|
-
if (extraContext) {
|
|
1257
|
-
textContent += `\n\n${extraContext}`;
|
|
1258
|
-
}
|
|
1259
|
-
textContent += '\n\nWhat action should you take?';
|
|
1260
|
-
const userContent = this.buildUserContent(textContent, state.screenshot, options?.forceVision === true);
|
|
1261
|
-
const useNavModel = this.shouldUseNavigationModel(state, extraContext, turnInfo);
|
|
1262
|
-
const effectiveProvider = useNavModel ? (this.navProvider || this.provider) : this.provider;
|
|
1263
|
-
const effectiveModel = useNavModel ? (this.navModelName || this.modelName) : this.modelName;
|
|
1264
|
-
if (this.debug) {
|
|
1265
|
-
const turnNum = Math.floor(this.history.length / 2) + 1;
|
|
1266
|
-
const usingVision = !!state.screenshot && (this.visionStrategy === 'always' || (this.visionStrategy === 'auto' && options?.forceVision === true));
|
|
1267
|
-
console.log(`[Brain] Turn ${turnNum} | URL: ${state.url} | Vision: ${usingVision}`);
|
|
1268
|
-
if (this.adaptiveModelRouting) {
|
|
1269
|
-
const mode = useNavModel ? 'nav-model' : 'primary-model';
|
|
1270
|
-
console.log(`[Brain] Model route: ${mode} (${effectiveProvider}/${effectiveModel}) turn=${turnInfo?.current}/${turnInfo?.max}`);
|
|
1271
|
-
}
|
|
1272
|
-
}
|
|
1273
|
-
const messages = [
|
|
1274
|
-
...this.compactHistory(),
|
|
1275
|
-
{ role: 'user', content: userContent },
|
|
1276
|
-
];
|
|
1277
|
-
const dynamicSystemPrompt = useCompactFirstTurn
|
|
1278
|
-
? FIRST_TURN_COMPACT_PROMPT
|
|
1279
|
-
: this.buildSystemForDecide(goal, state, turnInfo?.current ?? 1, effectiveProvider);
|
|
1280
|
-
const modelOpts = { provider: effectiveProvider, model: effectiveModel };
|
|
1281
|
-
// Bump output budget near max turns so data-heavy completions don't truncate
|
|
1282
|
-
const nearingEnd = turnInfo && turnInfo.current >= turnInfo.max - 3;
|
|
1283
|
-
const maxTokens = useCompactFirstTurn ? 500 : nearingEnd ? 1200 : 600;
|
|
1284
|
-
const result = await this.generate(dynamicSystemPrompt, messages, modelOpts, maxTokens);
|
|
1285
|
-
let raw = result.text;
|
|
1286
|
-
let tokensUsed = result.tokensUsed;
|
|
1287
|
-
let inputTokens = result.inputTokens;
|
|
1288
|
-
let outputTokens = result.outputTokens;
|
|
1289
|
-
let cacheReadInputTokens = result.cacheReadInputTokens;
|
|
1290
|
-
let cacheCreationInputTokens = result.cacheCreationInputTokens;
|
|
1291
|
-
if (!raw) {
|
|
1292
|
-
throw new Error('Brain.decide: LLM returned empty response — possible rate limit or model error');
|
|
1293
|
-
}
|
|
1294
|
-
if (this.debug) {
|
|
1295
|
-
console.log('[Brain] Response:', raw.slice(0, 300));
|
|
1296
|
-
}
|
|
1297
|
-
let parsed = this.parse(raw);
|
|
1298
|
-
// On malformed JSON, retry with minimal context (current page + correction
|
|
1299
|
-
// hint) instead of burning a full turn. Costs ~7K tokens vs ~25K for a
|
|
1300
|
-
// full-history retry on the next turn.
|
|
1301
|
-
if (parsed.reasoning?.startsWith('Malformed LLM JSON response') && !useCompactFirstTurn) {
|
|
1302
|
-
if (this.debug) {
|
|
1303
|
-
console.log('[Brain] Malformed JSON — retrying with format hint');
|
|
1304
|
-
}
|
|
1305
|
-
const retryMessages = [
|
|
1306
|
-
{ role: 'user', content: userContent },
|
|
1307
|
-
{ role: 'assistant', content: raw },
|
|
1308
|
-
{ role: 'user', content: 'Your previous response was not valid JSON. Respond with ONLY a valid JSON object matching the required schema.' },
|
|
1309
|
-
];
|
|
1310
|
-
try {
|
|
1311
|
-
const retryResult = await this.generate(dynamicSystemPrompt, retryMessages, modelOpts, maxTokens);
|
|
1312
|
-
if (retryResult.text) {
|
|
1313
|
-
const retryParsed = this.parse(retryResult.text);
|
|
1314
|
-
if (!retryParsed.reasoning?.startsWith('Malformed LLM JSON response')) {
|
|
1315
|
-
raw = retryResult.text;
|
|
1316
|
-
parsed = retryParsed;
|
|
1317
|
-
}
|
|
1318
|
-
else if (this.baseUrl) {
|
|
1319
|
-
// Both the initial parse and the format-hint retry failed while a
|
|
1320
|
-
// custom LLM_BASE_URL is set. Strong signal the gateway is
|
|
1321
|
-
// returning a shape the scout can't consume (e.g. SSE streams,
|
|
1322
|
-
// non-JSON wrappers). Surface the likely cause instead of
|
|
1323
|
-
// burning silent retries turn after turn.
|
|
1324
|
-
console.error(`[Brain] scout_json_parse_failed: LLM_BASE_URL=${this.baseUrl} returned a response the scout could not parse even after a format-hint retry. ` +
|
|
1325
|
-
`Suggestion: switch to an Anthropic-native endpoint, or verify the gateway supports non-streaming chat/completions with { "response_format": { "type": "json_object" } }.`);
|
|
1326
|
-
}
|
|
1327
|
-
tokensUsed = (tokensUsed ?? 0) + (retryResult.tokensUsed ?? 0);
|
|
1328
|
-
inputTokens = (inputTokens ?? 0) + (retryResult.inputTokens ?? 0);
|
|
1329
|
-
outputTokens = (outputTokens ?? 0) + (retryResult.outputTokens ?? 0);
|
|
1330
|
-
if (retryResult.cacheReadInputTokens !== undefined) {
|
|
1331
|
-
cacheReadInputTokens = (cacheReadInputTokens ?? 0) + retryResult.cacheReadInputTokens;
|
|
1332
|
-
}
|
|
1333
|
-
if (retryResult.cacheCreationInputTokens !== undefined) {
|
|
1334
|
-
cacheCreationInputTokens = (cacheCreationInputTokens ?? 0) + retryResult.cacheCreationInputTokens;
|
|
1335
|
-
}
|
|
1336
|
-
}
|
|
1337
|
-
}
|
|
1338
|
-
catch {
|
|
1339
|
-
// Retry failed — fall through with original wait(1000) fallback
|
|
1340
|
-
}
|
|
1341
|
-
}
|
|
1342
|
-
// Store in history
|
|
1343
|
-
this.history.push({ role: 'user', content: userContent });
|
|
1344
|
-
this.history.push({ role: 'assistant', content: raw });
|
|
1345
|
-
// Trim old history
|
|
1346
|
-
const maxMessages = this.maxHistoryTurns * 2;
|
|
1347
|
-
if (this.history.length > maxMessages) {
|
|
1348
|
-
this.history = this.history.slice(-maxMessages);
|
|
1349
|
-
}
|
|
1350
|
-
return {
|
|
1351
|
-
...parsed,
|
|
1352
|
-
raw,
|
|
1353
|
-
tokensUsed,
|
|
1354
|
-
inputTokens,
|
|
1355
|
-
outputTokens,
|
|
1356
|
-
cacheReadInputTokens,
|
|
1357
|
-
cacheCreationInputTokens,
|
|
1358
|
-
modelUsed: effectiveModel,
|
|
1359
|
-
};
|
|
221
|
+
return decideImpl(this, goal, state, extraContext, turnInfo, options);
|
|
1360
222
|
}
|
|
1361
223
|
/**
|
|
1362
224
|
* Vision-first decision path. The screenshot is the primary
|
|
@@ -1365,120 +227,7 @@ ${visibleSnapshot}`;
|
|
|
1365
227
|
* coordinate-based actions (clickAt, typeAt) in 1024×768 virtual space.
|
|
1366
228
|
*/
|
|
1367
229
|
async decideVision(goal, state, extraContext, turnInfo) {
|
|
1368
|
-
this
|
|
1369
|
-
// Adaptive observation: on same-page hybrid turns, send only changed
|
|
1370
|
-
// elements when the diff is small.
|
|
1371
|
-
const isHybrid = this.observationMode === 'hybrid';
|
|
1372
|
-
const samePageAsPrevious = this.lastDecisionUrl === state.url;
|
|
1373
|
-
const isFirstTurn = !turnInfo || turnInfo.current <= 1;
|
|
1374
|
-
const rawDiff = state.snapshotDiffRaw;
|
|
1375
|
-
const diffChanges = rawDiff ? rawDiff.added.length + rawDiff.removed.length + rawDiff.changed.length : 0;
|
|
1376
|
-
const diffTotal = rawDiff ? diffChanges + rawDiff.unchangedCount : 0;
|
|
1377
|
-
const useDiffOnly = isHybrid && samePageAsPrevious && !isFirstTurn
|
|
1378
|
-
&& rawDiff !== undefined && diffChanges > 0 && diffTotal > 0
|
|
1379
|
-
&& diffChanges / diffTotal < 0.4;
|
|
1380
|
-
let textContent = `GOAL: ${goal}
|
|
1381
|
-
|
|
1382
|
-
CURRENT PAGE:
|
|
1383
|
-
URL: ${state.url}
|
|
1384
|
-
Title: ${state.title}`;
|
|
1385
|
-
if (isHybrid && state.snapshot) {
|
|
1386
|
-
if (useDiffOnly) {
|
|
1387
|
-
// Diff-focused: only what changed since last turn
|
|
1388
|
-
const lines = [];
|
|
1389
|
-
if (rawDiff.added.length)
|
|
1390
|
-
lines.push('ADDED:', ...rawDiff.added);
|
|
1391
|
-
if (rawDiff.changed.length)
|
|
1392
|
-
lines.push('CHANGED:', ...rawDiff.changed);
|
|
1393
|
-
if (rawDiff.removed.length)
|
|
1394
|
-
lines.push('REMOVED:', ...rawDiff.removed);
|
|
1395
|
-
lines.push(`(${rawDiff.unchangedCount} elements unchanged — refs from previous turn still valid)`);
|
|
1396
|
-
textContent += `\n\nPAGE CHANGES (what changed after your last action — this is the important part):\n${lines.join('\n')}`;
|
|
1397
|
-
}
|
|
1398
|
-
else {
|
|
1399
|
-
// Progressive budget reduction: more turns on same page = less snapshot
|
|
1400
|
-
// needed (agent has already seen the full page, rely on screenshot + diff).
|
|
1401
|
-
const sameTurnCount = samePageAsPrevious ? (turnInfo?.current || 0) : 0;
|
|
1402
|
-
const snapshotBudget = samePageAsPrevious
|
|
1403
|
-
? (sameTurnCount >= 8 ? 2_500 : 4_000) // aggressive after 8+ same-page turns
|
|
1404
|
-
: 6_000;
|
|
1405
|
-
const snap = budgetSnapshot(state.snapshot, snapshotBudget);
|
|
1406
|
-
textContent += `\n\nELEMENTS:\n${snap}`;
|
|
1407
|
-
}
|
|
1408
|
-
}
|
|
1409
|
-
if (turnInfo) {
|
|
1410
|
-
const remaining = turnInfo.max - turnInfo.current;
|
|
1411
|
-
textContent += `\n\nTURN: ${turnInfo.current}/${turnInfo.max} (${remaining} remaining)`;
|
|
1412
|
-
if (remaining === 1) {
|
|
1413
|
-
textContent += ` — FINAL TURN: return a terminal action only (complete or abort)`;
|
|
1414
|
-
}
|
|
1415
|
-
else if (remaining <= 3) {
|
|
1416
|
-
textContent += ` — RUNNING LOW, prioritize completing the goal or aborting`;
|
|
1417
|
-
}
|
|
1418
|
-
}
|
|
1419
|
-
if (extraContext) {
|
|
1420
|
-
textContent += `\n\n${extraContext}`;
|
|
1421
|
-
}
|
|
1422
|
-
textContent += '\n\nLook at the screenshot. What action should you take?';
|
|
1423
|
-
// Screenshot is required for vision-first mode
|
|
1424
|
-
if (!state.screenshot) {
|
|
1425
|
-
return {
|
|
1426
|
-
action: { action: 'wait', ms: 500 },
|
|
1427
|
-
reasoning: 'No screenshot available for vision-first mode — waiting for page to render',
|
|
1428
|
-
raw: '{"action":{"action":"wait","ms":500}}',
|
|
1429
|
-
};
|
|
1430
|
-
}
|
|
1431
|
-
const userContent = [
|
|
1432
|
-
{ type: 'text', text: textContent },
|
|
1433
|
-
{ type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
|
|
1434
|
-
];
|
|
1435
|
-
// Strip old screenshots from history; the current screenshot is the only
|
|
1436
|
-
// image the model needs for this turn.
|
|
1437
|
-
const compacted = this.compactHistory().map((msg) => {
|
|
1438
|
-
if (msg.role !== 'user' || !Array.isArray(msg.content))
|
|
1439
|
-
return msg;
|
|
1440
|
-
const textOnly = msg.content
|
|
1441
|
-
.filter((part) => part.type === 'text');
|
|
1442
|
-
if (textOnly.length === msg.content.length)
|
|
1443
|
-
return msg;
|
|
1444
|
-
return { ...msg, content: textOnly };
|
|
1445
|
-
});
|
|
1446
|
-
const messages = [
|
|
1447
|
-
...compacted,
|
|
1448
|
-
{ role: 'user', content: userContent },
|
|
1449
|
-
];
|
|
1450
|
-
// Vision turns stay on the main model because smaller model routes can
|
|
1451
|
-
// count image tokens differently and exhaust the token budget.
|
|
1452
|
-
const modelOpts = { provider: this.provider, model: this.modelName };
|
|
1453
|
-
const nearingEnd = turnInfo && turnInfo.current >= turnInfo.max - 3;
|
|
1454
|
-
const maxTokens = nearingEnd ? 1200 : 600;
|
|
1455
|
-
// Hybrid mode uses the unified prompt with both action vocabularies.
|
|
1456
|
-
const systemPrompt = isHybrid ? UNIFIED_VISION_DOM_PROMPT : VISION_FIRST_PROMPT;
|
|
1457
|
-
const result = await this.generate(systemPrompt, messages, modelOpts, maxTokens);
|
|
1458
|
-
const raw = result.text;
|
|
1459
|
-
if (!raw) {
|
|
1460
|
-
throw new Error('Brain.decideVision: LLM returned empty response');
|
|
1461
|
-
}
|
|
1462
|
-
if (this.debug) {
|
|
1463
|
-
console.log('[Brain/Vision] Response:', raw.slice(0, 300));
|
|
1464
|
-
}
|
|
1465
|
-
const parsed = this.parse(raw);
|
|
1466
|
-
this.history.push({ role: 'user', content: userContent });
|
|
1467
|
-
this.history.push({ role: 'assistant', content: raw });
|
|
1468
|
-
const maxMessages = this.maxHistoryTurns * 2;
|
|
1469
|
-
if (this.history.length > maxMessages) {
|
|
1470
|
-
this.history = this.history.slice(-maxMessages);
|
|
1471
|
-
}
|
|
1472
|
-
return {
|
|
1473
|
-
...parsed,
|
|
1474
|
-
raw,
|
|
1475
|
-
tokensUsed: result.tokensUsed,
|
|
1476
|
-
inputTokens: result.inputTokens,
|
|
1477
|
-
outputTokens: result.outputTokens,
|
|
1478
|
-
cacheReadInputTokens: result.cacheReadInputTokens,
|
|
1479
|
-
cacheCreationInputTokens: result.cacheCreationInputTokens,
|
|
1480
|
-
modelUsed: this.modelName,
|
|
1481
|
-
};
|
|
230
|
+
return decideVisionImpl(this, goal, state, extraContext, turnInfo);
|
|
1482
231
|
}
|
|
1483
232
|
/**
|
|
1484
233
|
* Generate a structured plan for the entire task with one LLM call.
|
|
@@ -1494,224 +243,7 @@ Title: ${state.title}`;
|
|
|
1494
243
|
* use per-action loop".
|
|
1495
244
|
*/
|
|
1496
245
|
async plan(goal, state, options) {
|
|
1497
|
-
|
|
1498
|
-
const maxSteps = options?.maxSteps ?? 12;
|
|
1499
|
-
const extraContext = options?.extraContext;
|
|
1500
|
-
// Planner snapshots keep enough context for extraction tasks, especially
|
|
1501
|
-
// docs/spec pages with data in `<dl>`, `<code>`, and `<pre>` blocks.
|
|
1502
|
-
const snapshot = budgetSnapshot(state.snapshot, 24_000);
|
|
1503
|
-
const planSystemPrompt = `You are a planning engine for a browser automation agent.
|
|
1504
|
-
|
|
1505
|
-
Given a user goal and the current page state, your job is to generate a complete, ordered plan of actions that the agent will execute deterministically without re-entering you between steps. After each step, the runner verifies your stated post-condition. If verification fails the runner falls back to a per-action loop, so your job is to write a plan that requires the FEWEST steps and where every step's post-condition is reliably observable.
|
|
1506
|
-
|
|
1507
|
-
KEY PRINCIPLES:
|
|
1508
|
-
|
|
1509
|
-
1. PREFER BATCH VERBS. The driver supports batch \`fill\` (fill N text fields, set N selects, check N checkboxes in ONE action) and \`clickSequence\` (N sequential clicks). Use these aggressively. A 19-field form should be 2-4 fill steps, not 19 type steps.
|
|
1510
|
-
|
|
1511
|
-
CRITICAL EXCLUSION: DO NOT INCLUDE ANY SPINBUTTON OR DATE INPUT IN YOUR PLAN. AT ALL. Period. If you see \`spinbutton\` in the snapshot (year, month, day, hour, minute spinners) or any input that looks like a date/time picker, OMIT it from your plan completely. Playwright's locator.fill() and locator.click() both time out on these elements, and your plan will deviate and fall back to the per-action loop. The per-action fallback knows how to handle them. Just LEAVE THEM OUT. Your plan should silently skip those elements and continue with the rest of the task as if they don't exist. The runner WILL handle them after your plan completes — you do not need to plan a step for them.
|
|
1512
|
-
|
|
1513
|
-
2. ASSUME THE SNAPSHOT IS COMPLETE. The agent will execute your plan deterministically — you only get to see the page state ONCE (now). All @refs you emit must come from the ELEMENTS list below. Do not invent refs. If you don't see an element you'd need, do NOT plan a step for it — leave a gap and the runner will recover.
|
|
1514
|
-
|
|
1515
|
-
3. POST-CONDITIONS MUST BE OBSERVABLE. Each step's expectedEffect should describe a concrete change the runner can see in the next snapshot: a URL change, a new visible element, a status text update. Vague effects like "form is filled" are useless because verification can't check them. Use concrete strings: "Status text shows 'Account Created!'" or "URL contains /confirm/".
|
|
1516
|
-
|
|
1517
|
-
4. NAVIGATION CHANGES THE PAGE. Once you emit a \`navigate\`, \`click\` on a Next button, or any action that loads a new page, the @refs from the current snapshot are NO LONGER VALID. After such an action, your subsequent steps cannot rely on the same refs — they must use natural-language post-conditions until the runner falls back to per-action mode and observes the new page.
|
|
1518
|
-
|
|
1519
|
-
5. MAX ${maxSteps} STEPS. If the task genuinely requires more, plan the first ${maxSteps} and let the runner replan from the resulting state.
|
|
1520
|
-
|
|
1521
|
-
6. ONLY EMIT \`complete\` IF THE FINAL POST-CONDITION IS GENUINELY VERIFIABLE FROM THE PRIOR STEP'S expectedEffect. Do NOT fabricate success. If you cannot reliably know from the initial state alone whether the task succeeded (e.g. you can't predict whether a server submission will succeed, you don't know what the success message will say, or the form has multi-step navigation past your visibility), simply STOP planning at the last step you're confident about. The runner will fall through to the per-action loop after your plan exhausts and that loop will continue toward completion. It is BETTER to plan 5 confident steps and let the per-action loop finish than to plan 12 speculative steps with a fabricated complete at the end.
|
|
1522
|
-
|
|
1523
|
-
7. EXTRACTION TASKS: when the goal asks you to READ, EXTRACT, REPORT, or RETURN values from the page (numbers, text, lists, structured data), the LAST step of your plan MUST be \`runScript\`. Do NOT emit a \`complete\` step after the runScript with literal values in \`result\`, because at planning time you cannot know what runScript will return — any values you write would be fabricated. The runner has a deterministic substitution path: it will use the runScript output as the final result, OR fall through to per-action mode where the LLM can see the script output. Either way is fine. The wrong move is to put placeholder JSON like \`{"x":null,"y":null}\` or \`"<from prior step>"\` in the complete result; the runner detects and replaces those, but it's cleaner if you simply omit the complete step. RIGHT: \`[{action:runScript, script:"..."}]\`. WRONG: \`[{action:runScript,...}, {action:complete, result:"{x:null}"}]\`.
|
|
1524
|
-
|
|
1525
|
-
ACTION VERBS (same as the per-action prompt):
|
|
1526
|
-
- {"action": "click", "selector": "@REF"} — use when the element has a ref in ELEMENTS
|
|
1527
|
-
- {"action": "type", "selector": "@REF", "text": "..."} — type into a ref element
|
|
1528
|
-
- {"action": "press", "selector": "@REF", "key": "Enter"}
|
|
1529
|
-
- {"action": "select", "selector": "@REF", "value": "..."}
|
|
1530
|
-
- {"action": "clickAt", "x": 512, "y": 384} — click at pixel coordinates (use when you can see the element in the screenshot but it has no ref)
|
|
1531
|
-
- {"action": "typeAt", "x": 300, "y": 200, "text": "..."} — click at coordinates then type
|
|
1532
|
-
- {"action": "scroll", "direction": "up"|"down", "amount": 500}
|
|
1533
|
-
- {"action": "navigate", "url": "..."}
|
|
1534
|
-
- {"action": "wait", "ms": 1000}
|
|
1535
|
-
- {"action": "fill", "fields": {"@a": "v1", "@b": "v2"}, "selects": {"@c": "v3"}, "checks": ["@d", "@e"]}
|
|
1536
|
-
- {"action": "clickSequence", "refs": ["@a", "@b", "@c"]}
|
|
1537
|
-
- {"action": "runScript", "script": "document.querySelector('.x').textContent"}
|
|
1538
|
-
- {"action": "extractWithIndex", "query": "p, span, dd, code", "contains": "downloads"} — return a NUMBERED list of visible elements matching the query with their full textContent. Use this for extraction tasks where the data lives in obscurely-classed wrappers (npm download counts, MDN \`<dl>/<dt>/<dd>\` content, Python docs \`<code>\` blocks, W3C spec content) and the planner cannot guarantee a precise selector. The next step (in plan or per-action mode) reads the result and picks the right index. STRONGLY PREFER THIS OVER runScript on ANY extraction task where the snapshot doesn't already show the value verbatim.
|
|
1539
|
-
- {"action": "complete", "result": "..."}
|
|
1540
|
-
- {"action": "abort", "reason": "..."}
|
|
1541
|
-
|
|
1542
|
-
${URL_FIRST_RULES}
|
|
1543
|
-
|
|
1544
|
-
RESPONSE FORMAT — respond with ONLY this JSON:
|
|
1545
|
-
{
|
|
1546
|
-
"reasoning": "1-2 sentence strategy summary",
|
|
1547
|
-
"steps": [
|
|
1548
|
-
{
|
|
1549
|
-
"action": { "action": "fill", "fields": { "@t1": "Jordan", "@t2": "Rivera" } },
|
|
1550
|
-
"expectedEffect": "First name and last name fields are populated",
|
|
1551
|
-
"rationale": "Step 1 of the multi-step form: batch-fill all visible Personal Info text fields"
|
|
1552
|
-
},
|
|
1553
|
-
...
|
|
1554
|
-
],
|
|
1555
|
-
"finalResult": "Account creation form completed and confirmation visible"
|
|
1556
|
-
}
|
|
1557
|
-
|
|
1558
|
-
DO NOT include any prose outside the JSON. DO NOT use markdown code blocks. The runner parses your response with JSON.parse() and will fall through to the per-action loop on parse failure.`;
|
|
1559
|
-
// Replan path: when the runner re-enters plan() after a previous plan
|
|
1560
|
-
// deviated, it injects a deviation summary. The system prompt is byte-
|
|
1561
|
-
// stable so prompt cache still hits — only the user message changes.
|
|
1562
|
-
const userText = `GOAL: ${goal}
|
|
1563
|
-
|
|
1564
|
-
CURRENT PAGE:
|
|
1565
|
-
URL: ${state.url}
|
|
1566
|
-
Title: ${state.title}
|
|
1567
|
-
|
|
1568
|
-
ELEMENTS:
|
|
1569
|
-
${snapshot}
|
|
1570
|
-
${extraContext ? `\n${extraContext}\n` : ''}
|
|
1571
|
-
What is the complete plan?`;
|
|
1572
|
-
// In vision-capable modes, include the screenshot so the planner can use
|
|
1573
|
-
// visual layout when the DOM does not capture form structure.
|
|
1574
|
-
const isVisionPlanner = (this.observationMode === 'hybrid' || this.observationMode === 'vision') && !!state.screenshot;
|
|
1575
|
-
const userContent = isVisionPlanner
|
|
1576
|
-
? [
|
|
1577
|
-
{ type: 'text', text: userText },
|
|
1578
|
-
{ type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
|
|
1579
|
-
]
|
|
1580
|
-
: userText;
|
|
1581
|
-
// Planner can use its own model override.
|
|
1582
|
-
const planModelOpts = this.plannerModel
|
|
1583
|
-
? { provider: (this.plannerProvider || this.provider), model: this.plannerModel }
|
|
1584
|
-
: { provider: this.provider, model: this.modelName };
|
|
1585
|
-
const result = await this.generate(planSystemPrompt, [{ role: 'user', content: userContent }], planModelOpts,
|
|
1586
|
-
// Plans need more output tokens than decide() — a 10-step plan with
|
|
1587
|
-
// batch fills + rationale per step is comfortably over 1000 tokens.
|
|
1588
|
-
2_500).catch((err) => ({
|
|
1589
|
-
text: '',
|
|
1590
|
-
tokensUsed: undefined,
|
|
1591
|
-
inputTokens: undefined,
|
|
1592
|
-
outputTokens: undefined,
|
|
1593
|
-
cacheReadInputTokens: undefined,
|
|
1594
|
-
cacheCreationInputTokens: undefined,
|
|
1595
|
-
_error: err instanceof Error ? err.message : String(err),
|
|
1596
|
-
}));
|
|
1597
|
-
const durationMs = Date.now() - startedAt;
|
|
1598
|
-
const raw = result.text;
|
|
1599
|
-
if (!raw) {
|
|
1600
|
-
return {
|
|
1601
|
-
plan: null,
|
|
1602
|
-
raw: '',
|
|
1603
|
-
durationMs,
|
|
1604
|
-
parseError: result._error ?? 'empty response',
|
|
1605
|
-
};
|
|
1606
|
-
}
|
|
1607
|
-
// Reuse the same JSON tolerance as decide(): strip markdown fences,
|
|
1608
|
-
// then JSON.parse. On parse failure, return null and let the runner
|
|
1609
|
-
// fall through.
|
|
1610
|
-
let body = raw.trim();
|
|
1611
|
-
if (body.startsWith('```')) {
|
|
1612
|
-
body = body.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
1613
|
-
}
|
|
1614
|
-
let parsed;
|
|
1615
|
-
try {
|
|
1616
|
-
parsed = JSON.parse(body);
|
|
1617
|
-
}
|
|
1618
|
-
catch (err) {
|
|
1619
|
-
return {
|
|
1620
|
-
plan: null,
|
|
1621
|
-
raw,
|
|
1622
|
-
durationMs,
|
|
1623
|
-
tokensUsed: result.tokensUsed,
|
|
1624
|
-
inputTokens: result.inputTokens,
|
|
1625
|
-
outputTokens: result.outputTokens,
|
|
1626
|
-
cacheReadInputTokens: result.cacheReadInputTokens,
|
|
1627
|
-
cacheCreationInputTokens: result.cacheCreationInputTokens,
|
|
1628
|
-
parseError: err instanceof Error ? err.message : String(err),
|
|
1629
|
-
};
|
|
1630
|
-
}
|
|
1631
|
-
if (!Array.isArray(parsed.steps) || parsed.steps.length === 0) {
|
|
1632
|
-
return {
|
|
1633
|
-
plan: null,
|
|
1634
|
-
raw,
|
|
1635
|
-
durationMs,
|
|
1636
|
-
tokensUsed: result.tokensUsed,
|
|
1637
|
-
inputTokens: result.inputTokens,
|
|
1638
|
-
outputTokens: result.outputTokens,
|
|
1639
|
-
cacheReadInputTokens: result.cacheReadInputTokens,
|
|
1640
|
-
cacheCreationInputTokens: result.cacheCreationInputTokens,
|
|
1641
|
-
parseError: 'plan has zero steps',
|
|
1642
|
-
};
|
|
1643
|
-
}
|
|
1644
|
-
// Validate each step. Each must have a parseable action and a non-empty
|
|
1645
|
-
// expectedEffect string. We use the same validateAction helper that the
|
|
1646
|
-
// per-action parser uses, so the action shapes stay consistent.
|
|
1647
|
-
const steps = [];
|
|
1648
|
-
for (const [idx, rawStep] of parsed.steps.entries()) {
|
|
1649
|
-
if (!rawStep || typeof rawStep !== 'object') {
|
|
1650
|
-
return {
|
|
1651
|
-
plan: null,
|
|
1652
|
-
raw,
|
|
1653
|
-
durationMs,
|
|
1654
|
-
tokensUsed: result.tokensUsed,
|
|
1655
|
-
inputTokens: result.inputTokens,
|
|
1656
|
-
outputTokens: result.outputTokens,
|
|
1657
|
-
cacheReadInputTokens: result.cacheReadInputTokens,
|
|
1658
|
-
cacheCreationInputTokens: result.cacheCreationInputTokens,
|
|
1659
|
-
parseError: `step ${idx + 1}: not an object`,
|
|
1660
|
-
};
|
|
1661
|
-
}
|
|
1662
|
-
const stepObj = rawStep;
|
|
1663
|
-
const actionRaw = stepObj.action;
|
|
1664
|
-
if (!actionRaw || typeof actionRaw !== 'object') {
|
|
1665
|
-
return {
|
|
1666
|
-
plan: null,
|
|
1667
|
-
raw,
|
|
1668
|
-
durationMs,
|
|
1669
|
-
parseError: `step ${idx + 1}: missing action`,
|
|
1670
|
-
};
|
|
1671
|
-
}
|
|
1672
|
-
const actionData = actionRaw;
|
|
1673
|
-
const actionType = actionData.action;
|
|
1674
|
-
if (typeof actionType !== 'string') {
|
|
1675
|
-
return {
|
|
1676
|
-
plan: null,
|
|
1677
|
-
raw,
|
|
1678
|
-
durationMs,
|
|
1679
|
-
parseError: `step ${idx + 1}: action.action must be a string`,
|
|
1680
|
-
};
|
|
1681
|
-
}
|
|
1682
|
-
let action;
|
|
1683
|
-
try {
|
|
1684
|
-
action = validateAction(actionType, actionData);
|
|
1685
|
-
}
|
|
1686
|
-
catch (err) {
|
|
1687
|
-
return {
|
|
1688
|
-
plan: null,
|
|
1689
|
-
raw,
|
|
1690
|
-
durationMs,
|
|
1691
|
-
parseError: `step ${idx + 1}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1692
|
-
};
|
|
1693
|
-
}
|
|
1694
|
-
const expectedEffect = typeof stepObj.expectedEffect === 'string' && stepObj.expectedEffect.length > 0
|
|
1695
|
-
? stepObj.expectedEffect
|
|
1696
|
-
: 'page state advances after this action';
|
|
1697
|
-
const rationale = typeof stepObj.rationale === 'string' ? stepObj.rationale : undefined;
|
|
1698
|
-
steps.push({ action, expectedEffect, ...(rationale ? { rationale } : {}) });
|
|
1699
|
-
}
|
|
1700
|
-
const plan = {
|
|
1701
|
-
steps: steps.slice(0, maxSteps),
|
|
1702
|
-
...(typeof parsed.finalResult === 'string' ? { finalResult: parsed.finalResult } : {}),
|
|
1703
|
-
...(typeof parsed.reasoning === 'string' ? { reasoning: parsed.reasoning } : {}),
|
|
1704
|
-
};
|
|
1705
|
-
return {
|
|
1706
|
-
plan,
|
|
1707
|
-
raw,
|
|
1708
|
-
durationMs,
|
|
1709
|
-
tokensUsed: result.tokensUsed,
|
|
1710
|
-
inputTokens: result.inputTokens,
|
|
1711
|
-
outputTokens: result.outputTokens,
|
|
1712
|
-
cacheReadInputTokens: result.cacheReadInputTokens,
|
|
1713
|
-
cacheCreationInputTokens: result.cacheCreationInputTokens,
|
|
1714
|
-
};
|
|
246
|
+
return planImpl(this, goal, state, options);
|
|
1715
247
|
}
|
|
1716
248
|
/**
|
|
1717
249
|
* Evaluate quality of the current page state.
|
|
@@ -1719,104 +251,10 @@ What is the complete plan?`;
|
|
|
1719
251
|
* design, and professional polish.
|
|
1720
252
|
*/
|
|
1721
253
|
async evaluate(state, goal) {
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
CURRENT PAGE:
|
|
1725
|
-
URL: ${state.url}
|
|
1726
|
-
Title: ${state.title}
|
|
1727
|
-
|
|
1728
|
-
Please evaluate the quality of this page/application.`;
|
|
1729
|
-
const userContent = this.buildUserContent(textContent, state.screenshot, true);
|
|
1730
|
-
const result = await this.generate(EVALUATE_PROMPT, [{ role: 'user', content: userContent }], undefined, 800);
|
|
1731
|
-
const raw = result.text;
|
|
1732
|
-
const tokensUsed = result.tokensUsed;
|
|
1733
|
-
if (this.debug) {
|
|
1734
|
-
console.log('[Brain] Evaluation:', raw.slice(0, 300));
|
|
1735
|
-
}
|
|
1736
|
-
try {
|
|
1737
|
-
let text = raw.trim();
|
|
1738
|
-
if (text.startsWith('```')) {
|
|
1739
|
-
text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
1740
|
-
}
|
|
1741
|
-
const parsed = JSON.parse(text);
|
|
1742
|
-
const rawScore = typeof parsed.score === 'number' ? parsed.score : 5;
|
|
1743
|
-
return {
|
|
1744
|
-
score: Math.max(1, Math.min(10, rawScore)),
|
|
1745
|
-
assessment: parsed.assessment ?? 'No assessment provided',
|
|
1746
|
-
strengths: Array.isArray(parsed.strengths) ? parsed.strengths : [],
|
|
1747
|
-
issues: Array.isArray(parsed.issues) ? parsed.issues : [],
|
|
1748
|
-
suggestions: Array.isArray(parsed.suggestions) ? parsed.suggestions : [],
|
|
1749
|
-
raw,
|
|
1750
|
-
tokensUsed,
|
|
1751
|
-
};
|
|
1752
|
-
}
|
|
1753
|
-
catch {
|
|
1754
|
-
return {
|
|
1755
|
-
score: 5,
|
|
1756
|
-
assessment: 'Failed to parse evaluation response',
|
|
1757
|
-
strengths: [],
|
|
1758
|
-
issues: [],
|
|
1759
|
-
suggestions: [],
|
|
1760
|
-
raw,
|
|
1761
|
-
tokensUsed,
|
|
1762
|
-
};
|
|
1763
|
-
}
|
|
254
|
+
return evaluateImpl(this, state, goal);
|
|
1764
255
|
}
|
|
1765
256
|
async recommendLinkCandidate(goal, state, candidates, extraContext) {
|
|
1766
|
-
|
|
1767
|
-
// Scout only needs candidates + context, not the full snapshot (saves 2-8k tokens)
|
|
1768
|
-
const lines = [
|
|
1769
|
-
`GOAL: ${goal}`,
|
|
1770
|
-
'',
|
|
1771
|
-
`PAGE: ${state.url} — ${state.title}`,
|
|
1772
|
-
'',
|
|
1773
|
-
'CANDIDATES:',
|
|
1774
|
-
...topCandidates.map((candidate, index) => `${index + 1}. ${candidate.ref} — ${candidate.text} (score ${candidate.score})`),
|
|
1775
|
-
];
|
|
1776
|
-
if (extraContext) {
|
|
1777
|
-
lines.push('', extraContext);
|
|
1778
|
-
}
|
|
1779
|
-
lines.push('', 'Choose the single best next visible link.');
|
|
1780
|
-
const userContent = this.buildUserContent(lines.join('\n'), state.screenshot, this.scoutUseVision);
|
|
1781
|
-
const provider = this.scoutProvider || this.navProvider || this.provider;
|
|
1782
|
-
const model = this.scoutModelName || this.navModelName || this.modelName;
|
|
1783
|
-
const result = await this.generate(LINK_SCOUT_PROMPT, [{ role: 'user', content: userContent }], { provider, model }, 300);
|
|
1784
|
-
const raw = result.text;
|
|
1785
|
-
const tokensUsed = result.tokensUsed;
|
|
1786
|
-
try {
|
|
1787
|
-
let text = raw.trim();
|
|
1788
|
-
if (text.startsWith('```')) {
|
|
1789
|
-
text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
1790
|
-
}
|
|
1791
|
-
const parsed = JSON.parse(text);
|
|
1792
|
-
const selector = typeof parsed.selector === 'string' ? parsed.selector.trim() : '';
|
|
1793
|
-
const candidate = topCandidates.find((entry) => entry.ref === selector);
|
|
1794
|
-
if (!candidate) {
|
|
1795
|
-
throw new Error('invalid scout selector');
|
|
1796
|
-
}
|
|
1797
|
-
return {
|
|
1798
|
-
selector,
|
|
1799
|
-
reasoning: typeof parsed.reasoning === 'string' ? parsed.reasoning : 'No scout reasoning provided.',
|
|
1800
|
-
confidence: typeof parsed.confidence === 'number'
|
|
1801
|
-
? Math.max(0, Math.min(1, parsed.confidence))
|
|
1802
|
-
: 0.5,
|
|
1803
|
-
raw,
|
|
1804
|
-
tokensUsed,
|
|
1805
|
-
};
|
|
1806
|
-
}
|
|
1807
|
-
catch {
|
|
1808
|
-
const fallback = topCandidates[0];
|
|
1809
|
-
if (!fallback) {
|
|
1810
|
-
throw new Error('recommendLinkCandidate requires at least one candidate');
|
|
1811
|
-
}
|
|
1812
|
-
return {
|
|
1813
|
-
selector: fallback.ref,
|
|
1814
|
-
reasoning: 'Scout fallback: selected the top deterministic candidate after parse failure.',
|
|
1815
|
-
confidence: 0.5,
|
|
1816
|
-
raw,
|
|
1817
|
-
tokensUsed,
|
|
1818
|
-
};
|
|
1819
|
-
}
|
|
257
|
+
return recommendLinkCandidateImpl(this, goal, state, candidates, extraContext);
|
|
1820
258
|
}
|
|
1821
259
|
/**
|
|
1822
260
|
* Verify whether the goal was actually achieved.
|
|
@@ -1824,59 +262,7 @@ Please evaluate the quality of this page/application.`;
|
|
|
1824
262
|
* Uses a fresh LLM call (no conversation history) to avoid self-confirmation bias.
|
|
1825
263
|
*/
|
|
1826
264
|
async verifyGoalCompletion(state, goal, claimedResult) {
|
|
1827
|
-
|
|
1828
|
-
const textContent = `GOAL: ${goal}
|
|
1829
|
-
|
|
1830
|
-
AGENT'S CLAIMED RESULT: ${claimedResult}
|
|
1831
|
-
|
|
1832
|
-
CURRENT PAGE:
|
|
1833
|
-
URL: ${state.url}
|
|
1834
|
-
Title: ${state.title}
|
|
1835
|
-
|
|
1836
|
-
ELEMENTS:
|
|
1837
|
-
${budgetSnapshot(state.snapshot)}${siteBoundaryNote ? `\n\n${siteBoundaryNote}` : ''}
|
|
1838
|
-
|
|
1839
|
-
Was the goal actually achieved? Analyze the current page state carefully.`;
|
|
1840
|
-
const userContent = this.buildUserContent(textContent, state.screenshot, true);
|
|
1841
|
-
// Verifier can use its own model, then the navigation model, then main.
|
|
1842
|
-
const verifyProvider = this.verifierProvider
|
|
1843
|
-
? this.verifierProvider
|
|
1844
|
-
: (this.adaptiveModelRouting && this.navModelName ? (this.navProvider || this.provider) : undefined);
|
|
1845
|
-
const verifyModel = this.verifierModel
|
|
1846
|
-
|| (this.adaptiveModelRouting && this.navModelName ? this.navModelName : undefined);
|
|
1847
|
-
const result = await this.generate(`Verify whether the browser agent achieved its goal. Respond with ONLY JSON:
|
|
1848
|
-
{"achieved":true,"confidence":0.9,"evidence":["observation"],"missing":[]}
|
|
1849
|
-
|
|
1850
|
-
Check: page state matches goal, no errors, URL is expected, claimed result matches visible data.
|
|
1851
|
-
SUPPLEMENTAL TOOL EVIDENCE / SCRIPT RESULT in claimed results = verified DOM data, trustworthy even if page navigated away. Multi-page data collection is valid.`, [{ role: 'user', content: userContent }], verifyProvider && verifyModel ? { provider: verifyProvider, model: verifyModel } : undefined, 600);
|
|
1852
|
-
const raw = result.text;
|
|
1853
|
-
if (this.debug) {
|
|
1854
|
-
console.log('[Brain] Goal verification:', raw.slice(0, 300));
|
|
1855
|
-
}
|
|
1856
|
-
try {
|
|
1857
|
-
let text = raw.trim();
|
|
1858
|
-
if (text.startsWith('```')) {
|
|
1859
|
-
text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
1860
|
-
}
|
|
1861
|
-
const parsed = JSON.parse(text);
|
|
1862
|
-
return {
|
|
1863
|
-
achieved: parsed.achieved === true,
|
|
1864
|
-
confidence: typeof parsed.confidence === 'number'
|
|
1865
|
-
? Math.max(0, Math.min(1, parsed.confidence))
|
|
1866
|
-
: 0.5,
|
|
1867
|
-
evidence: Array.isArray(parsed.evidence) ? parsed.evidence : [],
|
|
1868
|
-
missing: Array.isArray(parsed.missing) ? parsed.missing : [],
|
|
1869
|
-
};
|
|
1870
|
-
}
|
|
1871
|
-
catch {
|
|
1872
|
-
// Parse failure — assume not verified (conservative)
|
|
1873
|
-
return {
|
|
1874
|
-
achieved: false,
|
|
1875
|
-
confidence: 0,
|
|
1876
|
-
evidence: [],
|
|
1877
|
-
missing: ['Failed to parse goal verification response'],
|
|
1878
|
-
};
|
|
1879
|
-
}
|
|
265
|
+
return verifyGoalCompletionImpl(this, state, goal, claimedResult);
|
|
1880
266
|
}
|
|
1881
267
|
/**
|
|
1882
268
|
* Audit design quality of the current page state.
|
|
@@ -1884,92 +270,7 @@ SUPPLEMENTAL TOOL EVIDENCE / SCRIPT RESULT in claimed results = verified DOM dat
|
|
|
1884
270
|
* Returns structured findings with categories and severity levels.
|
|
1885
271
|
*/
|
|
1886
272
|
async auditDesign(state, goal, checkpoints, systemPrompt) {
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
CHECKPOINTS to verify:
|
|
1890
|
-
${checkpoints.map((c, i) => `${i + 1}. ${c}`).join('\n')}
|
|
1891
|
-
|
|
1892
|
-
CURRENT PAGE:
|
|
1893
|
-
URL: ${state.url}
|
|
1894
|
-
Title: ${state.title}
|
|
1895
|
-
|
|
1896
|
-
ELEMENTS:
|
|
1897
|
-
${state.snapshot}
|
|
1898
|
-
|
|
1899
|
-
Audit this page for design quality, UX issues, and visual bugs.`;
|
|
1900
|
-
const userContent = this.buildUserContent(textContent, state.screenshot, true);
|
|
1901
|
-
const result = await this.generate(systemPrompt ?? DESIGN_AUDIT_PROMPT, [{ role: 'user', content: userContent }], undefined, 8000);
|
|
1902
|
-
const raw = result.text;
|
|
1903
|
-
const tokensUsed = result.tokensUsed;
|
|
1904
|
-
if (this.debug) {
|
|
1905
|
-
console.log('[Brain] Design audit:', raw.slice(0, 300));
|
|
1906
|
-
}
|
|
1907
|
-
try {
|
|
1908
|
-
let text = raw.trim();
|
|
1909
|
-
if (text.startsWith('```')) {
|
|
1910
|
-
text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
1911
|
-
}
|
|
1912
|
-
// Extract JSON object if surrounded by non-JSON text or truncated
|
|
1913
|
-
let parsed;
|
|
1914
|
-
try {
|
|
1915
|
-
parsed = JSON.parse(text);
|
|
1916
|
-
}
|
|
1917
|
-
catch {
|
|
1918
|
-
const start = text.indexOf('{');
|
|
1919
|
-
const end = text.lastIndexOf('}');
|
|
1920
|
-
if (start >= 0 && end > start) {
|
|
1921
|
-
parsed = JSON.parse(text.slice(start, end + 1));
|
|
1922
|
-
}
|
|
1923
|
-
else {
|
|
1924
|
-
throw new Error('No JSON object found');
|
|
1925
|
-
}
|
|
1926
|
-
}
|
|
1927
|
-
const VALID_CATEGORIES = new Set(['visual-bug', 'layout', 'contrast', 'alignment', 'spacing', 'typography', 'accessibility', 'ux']);
|
|
1928
|
-
const VALID_SEVERITIES = new Set(['critical', 'major', 'minor']);
|
|
1929
|
-
const VALID_BLAST = new Set(['page', 'section', 'component', 'system']);
|
|
1930
|
-
const clampScore = (n) => typeof n === 'number' ? Math.max(1, Math.min(10, n)) : undefined;
|
|
1931
|
-
const findings = Array.isArray(parsed.findings)
|
|
1932
|
-
? parsed.findings.map((f) => ({
|
|
1933
|
-
category: (VALID_CATEGORIES.has(f.category) ? f.category : 'ux'),
|
|
1934
|
-
severity: (VALID_SEVERITIES.has(f.severity) ? f.severity : 'minor'),
|
|
1935
|
-
description: String(f.description ?? ''),
|
|
1936
|
-
location: String(f.location ?? ''),
|
|
1937
|
-
suggestion: String(f.suggestion ?? ''),
|
|
1938
|
-
...(f.cssSelector ? { cssSelector: String(f.cssSelector) } : {}),
|
|
1939
|
-
...(f.cssFix ? { cssFix: String(f.cssFix) } : {}),
|
|
1940
|
-
// Optional ROI fields.
|
|
1941
|
-
...(clampScore(f.impact) !== undefined ? { impact: clampScore(f.impact) } : {}),
|
|
1942
|
-
...(clampScore(f.effort) !== undefined ? { effort: clampScore(f.effort) } : {}),
|
|
1943
|
-
...(VALID_BLAST.has(f.blast)
|
|
1944
|
-
? { blast: f.blast }
|
|
1945
|
-
: {}),
|
|
1946
|
-
// Layer 2 — preserve raw patches array (untyped passthrough). The
|
|
1947
|
-
// parsePatches/validatePatch pipeline in build-result.ts converts
|
|
1948
|
-
// these into typed, validated Patch objects.
|
|
1949
|
-
...(Array.isArray(f.patches) ? { rawPatches: f.patches } : {}),
|
|
1950
|
-
}))
|
|
1951
|
-
: [];
|
|
1952
|
-
const designSystemScore = parsed.designSystemScore && typeof parsed.designSystemScore === 'object'
|
|
1953
|
-
? parsed.designSystemScore
|
|
1954
|
-
: undefined;
|
|
1955
|
-
const rawScore = typeof parsed.score === 'number' ? parsed.score : 5;
|
|
1956
|
-
return {
|
|
1957
|
-
score: Math.max(1, Math.min(10, rawScore)),
|
|
1958
|
-
findings,
|
|
1959
|
-
raw,
|
|
1960
|
-
tokensUsed,
|
|
1961
|
-
designSystemScore,
|
|
1962
|
-
};
|
|
1963
|
-
}
|
|
1964
|
-
catch (err) {
|
|
1965
|
-
return {
|
|
1966
|
-
score: 5,
|
|
1967
|
-
findings: [],
|
|
1968
|
-
raw,
|
|
1969
|
-
tokensUsed,
|
|
1970
|
-
parseError: err instanceof Error ? err.message : String(err),
|
|
1971
|
-
};
|
|
1972
|
-
}
|
|
273
|
+
return auditDesignImpl(this, state, goal, checkpoints, systemPrompt);
|
|
1973
274
|
}
|
|
1974
275
|
/**
|
|
1975
276
|
* Extract reusable knowledge from a completed trajectory.
|
|
@@ -1977,49 +278,7 @@ Audit this page for design quality, UX issues, and visual bugs.`;
|
|
|
1977
278
|
* and app quirks from a successful run.
|
|
1978
279
|
*/
|
|
1979
280
|
async extractKnowledge(trajectoryText, domain) {
|
|
1980
|
-
|
|
1981
|
-
Extract facts that would help an agent complete similar tasks faster next time.
|
|
1982
|
-
|
|
1983
|
-
Respond with ONLY a JSON array of facts:
|
|
1984
|
-
[
|
|
1985
|
-
{"type": "timing", "key": "page-load", "value": "wait 3000ms after navigation for content to hydrate"},
|
|
1986
|
-
{"type": "selector", "key": "send-button", "value": "[data-testid='chat-send-button'] is the reliable send button selector"},
|
|
1987
|
-
{"type": "pattern", "key": "auth-flow", "value": "Click sign-in → fill email → fill password → click submit → wait for redirect"},
|
|
1988
|
-
{"type": "quirk", "key": "lazy-loading", "value": "File tree loads asynchronously — wait for entries before asserting"}
|
|
1989
|
-
]
|
|
1990
|
-
|
|
1991
|
-
Types:
|
|
1992
|
-
- timing: wait durations, delays that are necessary
|
|
1993
|
-
- selector: reliable selectors for important elements
|
|
1994
|
-
- pattern: multi-step interaction sequences
|
|
1995
|
-
- quirk: app-specific behaviors or gotchas
|
|
1996
|
-
|
|
1997
|
-
Only include facts that are genuinely useful. Quality over quantity. Max 10 facts.`, [{
|
|
1998
|
-
role: 'user',
|
|
1999
|
-
content: `Domain: ${domain}\n\nTrajectory:\n${trajectoryText}`,
|
|
2000
|
-
}], undefined, 800);
|
|
2001
|
-
try {
|
|
2002
|
-
let text = result.text.trim();
|
|
2003
|
-
if (text.startsWith('```')) {
|
|
2004
|
-
text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
|
|
2005
|
-
}
|
|
2006
|
-
const parsed = JSON.parse(text);
|
|
2007
|
-
if (!Array.isArray(parsed))
|
|
2008
|
-
return [];
|
|
2009
|
-
const VALID_TYPES = new Set(['timing', 'selector', 'pattern', 'quirk']);
|
|
2010
|
-
return parsed
|
|
2011
|
-
.filter((f) => VALID_TYPES.has(f.type) &&
|
|
2012
|
-
typeof f.key === 'string' &&
|
|
2013
|
-
typeof f.value === 'string')
|
|
2014
|
-
.map((f) => ({
|
|
2015
|
-
type: f.type,
|
|
2016
|
-
key: f.key,
|
|
2017
|
-
value: f.value,
|
|
2018
|
-
}));
|
|
2019
|
-
}
|
|
2020
|
-
catch {
|
|
2021
|
-
return [];
|
|
2022
|
-
}
|
|
281
|
+
return extractKnowledgeImpl(this, trajectoryText, domain);
|
|
2023
282
|
}
|
|
2024
283
|
/**
|
|
2025
284
|
* Generic text-completion entry point for non-agent uses (GEPA reflective
|
|
@@ -2030,490 +289,25 @@ Only include facts that are genuinely useful. Quality over quantity. Max 10 fact
|
|
|
2030
289
|
* the configured provider/model.
|
|
2031
290
|
*/
|
|
2032
291
|
async complete(system, user, options = {}) {
|
|
2033
|
-
|
|
2034
|
-
return { text: result.text, tokensUsed: result.tokensUsed };
|
|
292
|
+
return completeImpl(this, system, user, options);
|
|
2035
293
|
}
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
// Macro dispatch. The driver validates the macro name at execute time.
|
|
2051
|
-
'macro',
|
|
2052
|
-
// Parallel fan-out. Runner handles dispatch; validator checks shape.
|
|
2053
|
-
'fanOut',
|
|
2054
|
-
]);
|
|
2055
|
-
// Parse strategy: exact → first-{/last-} extraction.
|
|
2056
|
-
// Some OpenAI-compat gateways (router.tangle.tools, LiteLLM proxies, etc.)
|
|
2057
|
-
// wrap model output in prose preambles ("Here's your response:\n{...}")
|
|
2058
|
-
// that markdown-fence stripping doesn't catch. Fall back to extracting the
|
|
2059
|
-
// outermost object literal before giving up.
|
|
2060
|
-
let parsed = null;
|
|
2061
|
-
let parseError = '';
|
|
2062
|
-
try {
|
|
2063
|
-
parsed = JSON.parse(text);
|
|
2064
|
-
}
|
|
2065
|
-
catch (err) {
|
|
2066
|
-
parseError = err instanceof Error ? err.message : String(err);
|
|
2067
|
-
const firstBrace = text.indexOf('{');
|
|
2068
|
-
const lastBrace = text.lastIndexOf('}');
|
|
2069
|
-
if (firstBrace >= 0 && lastBrace > firstBrace) {
|
|
2070
|
-
try {
|
|
2071
|
-
parsed = JSON.parse(text.slice(firstBrace, lastBrace + 1));
|
|
2072
|
-
parseError = '';
|
|
2073
|
-
}
|
|
2074
|
-
catch { /* fall through to retry fallback */ }
|
|
2075
|
-
}
|
|
2076
|
-
}
|
|
2077
|
-
if (!parsed) {
|
|
2078
|
-
return {
|
|
2079
|
-
// Do not hard-abort the scenario on transient JSON formatting issues.
|
|
2080
|
-
// Waiting one turn lets the loop continue and recover on the next model call.
|
|
2081
|
-
action: { action: 'wait', ms: 1000 },
|
|
2082
|
-
reasoning: `Malformed LLM JSON response (${parseError}). Retrying next turn.`,
|
|
2083
|
-
};
|
|
2084
|
-
}
|
|
2085
|
-
try {
|
|
2086
|
-
const actionObj = parsed.action && typeof parsed.action === 'object' ? parsed.action : parsed;
|
|
2087
|
-
const actionType = typeof parsed.action === 'string' ? parsed.action : actionObj?.action;
|
|
2088
|
-
if (!actionType) {
|
|
2089
|
-
throw new Error('Missing action field');
|
|
2090
|
-
}
|
|
2091
|
-
if (!VALID_ACTIONS.has(actionType)) {
|
|
2092
|
-
throw new Error(`Unknown action "${actionType}". Valid: ${[...VALID_ACTIONS].join(', ')}`);
|
|
2093
|
-
}
|
|
2094
|
-
const actionData = parsed.action && typeof parsed.action === 'object'
|
|
2095
|
-
? parsed.action
|
|
2096
|
-
: parsed;
|
|
2097
|
-
const action = validateAction(actionType, actionData);
|
|
2098
|
-
return {
|
|
2099
|
-
action,
|
|
2100
|
-
nextActions: parseNextActions(parsed, VALID_ACTIONS),
|
|
2101
|
-
reasoning: (parsed.reasoning || parsed.thought || parsed.thinking),
|
|
2102
|
-
plan: Array.isArray(parsed.plan) ? parsed.plan : undefined,
|
|
2103
|
-
currentStep: typeof parsed.currentStep === 'number' ? parsed.currentStep : undefined,
|
|
2104
|
-
expectedEffect: (parsed.expectedEffect || parsed.expected_effect),
|
|
2105
|
-
};
|
|
2106
|
-
}
|
|
2107
|
-
catch (err) {
|
|
2108
|
-
const validationError = err instanceof Error ? err.message : String(err);
|
|
2109
|
-
return {
|
|
2110
|
-
action: { action: 'wait', ms: 1000 },
|
|
2111
|
-
reasoning: `Malformed LLM JSON response (${validationError}). Retrying next turn.`,
|
|
2112
|
-
};
|
|
2113
|
-
}
|
|
2114
|
-
}
|
|
2115
|
-
}
|
|
2116
|
-
/**
|
|
2117
|
-
* Collapse consecutive runs of similar elements (same indent + role, names
|
|
2118
|
-
* differing only by a trailing number/short suffix) into a single representative
|
|
2119
|
-
* line with a count. Reduces token cost on pages with long pagination, nav
|
|
2120
|
-
* lists, or repeated product cards.
|
|
2121
|
-
*
|
|
2122
|
-
* Skips dialog/alertdialog (agent must see each one) and groups < 3 items.
|
|
2123
|
-
*/
|
|
2124
|
-
function deduplicateSnapshot(snapshot) {
|
|
2125
|
-
const lines = snapshot.split('\n');
|
|
2126
|
-
const out = [];
|
|
2127
|
-
// Extract (indent, role) from a snapshot line. Returns null for non-element lines.
|
|
2128
|
-
const parseLine = (line) => {
|
|
2129
|
-
const m = line.match(/^(\s*-\s+)(\w+)\s+"([^"]*)"\s*\[ref=(\w+)\]/);
|
|
2130
|
-
if (!m)
|
|
2131
|
-
return null;
|
|
2132
|
-
return { indent: m[1], role: m[2], name: m[3], ref: m[4], full: line };
|
|
2133
|
-
};
|
|
2134
|
-
// Strip trailing numbers/ordinals to get a "name stem" for grouping.
|
|
2135
|
-
// "Page 1" and "Page 20" → "Page ", "Item #3" and "Item #42" → "Item #"
|
|
2136
|
-
const nameStem = (name) => name.replace(/\d+/g, '#');
|
|
2137
|
-
// Structural fingerprint for a block of lines (element + its children).
|
|
2138
|
-
// Used for card-level dedup: two hotel cards have different names but the
|
|
2139
|
-
// same structure (listitem > link + img + text + text + button).
|
|
2140
|
-
const structuralFingerprint = (startIdx, baseIndent) => {
|
|
2141
|
-
const roles = [];
|
|
2142
|
-
let j = startIdx;
|
|
2143
|
-
while (j < lines.length) {
|
|
2144
|
-
const p = parseLine(lines[j]);
|
|
2145
|
-
if (!p) {
|
|
2146
|
-
j++;
|
|
2147
|
-
continue;
|
|
2148
|
-
}
|
|
2149
|
-
// Stop when we hit an element at the same or shallower indent (sibling or parent)
|
|
2150
|
-
if (j > startIdx && p.indent.length <= baseIndent.length)
|
|
2151
|
-
break;
|
|
2152
|
-
roles.push(p.role);
|
|
2153
|
-
j++;
|
|
2154
|
-
}
|
|
2155
|
-
return { fp: roles.join(','), endIdx: j };
|
|
2156
|
-
};
|
|
2157
|
-
let i = 0;
|
|
2158
|
-
while (i < lines.length) {
|
|
2159
|
-
const parsed = parseLine(lines[i]);
|
|
2160
|
-
// Non-element line or dialog/alertdialog — emit as-is
|
|
2161
|
-
if (!parsed || /\b(?:dialog|alertdialog)\b/i.test(parsed.role)) {
|
|
2162
|
-
out.push(lines[i]);
|
|
2163
|
-
i++;
|
|
2164
|
-
continue;
|
|
2165
|
-
}
|
|
2166
|
-
// Try block-level dedup first: look for consecutive sibling blocks
|
|
2167
|
-
// with the same structural fingerprint (same child-role sequence).
|
|
2168
|
-
// This catches card patterns like Booking hotel results, Allrecipes cards.
|
|
2169
|
-
if (/\b(?:listitem|article|group|region)\b/i.test(parsed.role)) {
|
|
2170
|
-
const { fp: firstFp, endIdx: firstEnd } = structuralFingerprint(i, parsed.indent);
|
|
2171
|
-
if (firstEnd > i + 2 && firstFp.includes(',')) { // non-trivial block
|
|
2172
|
-
const blocks = [
|
|
2173
|
-
{ start: i, end: firstEnd, firstLine: lines[i] },
|
|
2174
|
-
];
|
|
2175
|
-
let scanIdx = firstEnd;
|
|
2176
|
-
while (scanIdx < lines.length) {
|
|
2177
|
-
const nextParsed = parseLine(lines[scanIdx]);
|
|
2178
|
-
if (!nextParsed || nextParsed.indent !== parsed.indent || nextParsed.role !== parsed.role)
|
|
2179
|
-
break;
|
|
2180
|
-
const { fp: nextFp, endIdx: nextEnd } = structuralFingerprint(scanIdx, nextParsed.indent);
|
|
2181
|
-
if (nextFp !== firstFp)
|
|
2182
|
-
break;
|
|
2183
|
-
blocks.push({ start: scanIdx, end: nextEnd, firstLine: lines[scanIdx] });
|
|
2184
|
-
scanIdx = nextEnd;
|
|
2185
|
-
}
|
|
2186
|
-
if (blocks.length >= 3) {
|
|
2187
|
-
// Emit first 2 blocks fully, summarize the rest
|
|
2188
|
-
for (let b = 0; b < Math.min(2, blocks.length); b++) {
|
|
2189
|
-
for (let k = blocks[b].start; k < blocks[b].end; k++) {
|
|
2190
|
-
out.push(lines[k]);
|
|
2191
|
-
}
|
|
2192
|
-
}
|
|
2193
|
-
const remaining = blocks.length - 2;
|
|
2194
|
-
out.push(`${parsed.indent}... [${remaining} more similar ${parsed.role} blocks with same structure]`);
|
|
2195
|
-
i = blocks[blocks.length - 1].end;
|
|
2196
|
-
continue;
|
|
2197
|
-
}
|
|
2198
|
-
}
|
|
2199
|
-
}
|
|
2200
|
-
// Line-level dedup: consecutive runs of same (indent, role, name stem)
|
|
2201
|
-
const group = [parsed];
|
|
2202
|
-
const stem = nameStem(parsed.name);
|
|
2203
|
-
let j = i + 1;
|
|
2204
|
-
while (j < lines.length) {
|
|
2205
|
-
const next = parseLine(lines[j]);
|
|
2206
|
-
if (!next ||
|
|
2207
|
-
next.indent !== parsed.indent ||
|
|
2208
|
-
next.role !== parsed.role ||
|
|
2209
|
-
nameStem(next.name) !== stem)
|
|
2210
|
-
break;
|
|
2211
|
-
group.push(next);
|
|
2212
|
-
j++;
|
|
2213
|
-
}
|
|
2214
|
-
if (group.length < 3) {
|
|
2215
|
-
// Not enough to dedup — emit originals
|
|
2216
|
-
for (const g of group)
|
|
2217
|
-
out.push(g.full);
|
|
2218
|
-
}
|
|
2219
|
-
else {
|
|
2220
|
-
// Emit first element with a summary of the rest
|
|
2221
|
-
const last = group[group.length - 1];
|
|
2222
|
-
out.push(`${parsed.full} (+${group.length - 1} similar: "${group[1].name}"\u2026"${last.name}")`);
|
|
2223
|
-
}
|
|
2224
|
-
i = j;
|
|
2225
|
-
}
|
|
2226
|
-
return out.join('\n');
|
|
2227
|
-
}
|
|
2228
|
-
/**
|
|
2229
|
-
* Cap snapshot size to control token cost on large pages.
|
|
2230
|
-
* Keeps the full snapshot when it fits within budget; otherwise preserves:
|
|
2231
|
-
* 1. Interactive elements with refs (buttons, inputs, links — for action targets)
|
|
2232
|
-
* 2. Content lines: term/definition/code/pre/paragraph (for extraction tasks
|
|
2233
|
-
* like MDN, Python docs, W3C spec where the value the agent needs lives in
|
|
2234
|
-
* a `<dl>/<code>/<pre>` block, not in an interactive element)
|
|
2235
|
-
*
|
|
2236
|
-
* New-page callers use a larger default budget; same-page callers can pass a
|
|
2237
|
-
* tighter budget after the model has already seen the full snapshot once.
|
|
2238
|
-
*/
|
|
2239
|
-
export function budgetSnapshot(snapshot, maxChars = 24_000) {
|
|
2240
|
-
// Skip dedup on small snapshots — not enough repetition to justify the O(n) scan
|
|
2241
|
-
if (snapshot.length > 6_000) {
|
|
2242
|
-
snapshot = deduplicateSnapshot(snapshot);
|
|
2243
|
-
}
|
|
2244
|
-
if (snapshot.length <= maxChars)
|
|
2245
|
-
return snapshot;
|
|
2246
|
-
// First pass: separate keep-set (interactive + content lines) from decorative.
|
|
2247
|
-
// Content roles (term, definition, code, pre, paragraph) carry text the LLM
|
|
2248
|
-
// needs for extraction tasks. They have no [ref=] but the text is the data.
|
|
2249
|
-
const lines = snapshot.split('\n');
|
|
2250
|
-
const interactive = [];
|
|
2251
|
-
const content = [];
|
|
2252
|
-
const decorative = [];
|
|
2253
|
-
for (const line of lines) {
|
|
2254
|
-
if (/\b(?:button|link|textbox|combobox|menuitem|checkbox|radio|select|heading|dialog|alertdialog)\b/i.test(line) && /\[ref=/.test(line)) {
|
|
2255
|
-
interactive.push(line);
|
|
2256
|
-
}
|
|
2257
|
-
else if (/^\s*-\s+(?:term|definition|code|pre|paragraph)\b/i.test(line)) {
|
|
2258
|
-
content.push(line);
|
|
2259
|
-
}
|
|
2260
|
-
else {
|
|
2261
|
-
decorative.push(line);
|
|
2262
|
-
}
|
|
2263
|
-
}
|
|
2264
|
-
// If interactive + content fits, use both with a truncation note
|
|
2265
|
-
const keepSet = interactive.concat(content);
|
|
2266
|
-
const keepText = keepSet.join('\n');
|
|
2267
|
-
if (keepText.length <= maxChars) {
|
|
2268
|
-
return keepText + `\n... [${decorative.length} decorative elements omitted for brevity]`;
|
|
2269
|
-
}
|
|
2270
|
-
// Second pass: when interactive + content still exceed budget, prioritize:
|
|
2271
|
-
// 1. inputs (searchbox/textbox/combobox) — essential for form tasks
|
|
2272
|
-
// 2. headings + dialogs — structural navigation
|
|
2273
|
-
// 3. content lines (term/definition/code/pre) — extraction data
|
|
2274
|
-
// 4. bulk links/buttons — main content
|
|
2275
|
-
const priority = [];
|
|
2276
|
-
const bulk = [];
|
|
2277
|
-
for (const line of interactive) {
|
|
2278
|
-
if (/\b(?:searchbox|textbox|combobox|heading|dialog|alertdialog)\b/i.test(line)) {
|
|
2279
|
-
priority.push(line);
|
|
2280
|
-
}
|
|
2281
|
-
else {
|
|
2282
|
-
bulk.push(line);
|
|
2283
|
-
}
|
|
2284
|
-
}
|
|
2285
|
-
// Content lines come right after priority interactive (they're the extraction
|
|
2286
|
-
// data) and before bulk links/buttons.
|
|
2287
|
-
const priorityWithContent = priority.concat(content);
|
|
2288
|
-
const priorityText = priorityWithContent.join('\n');
|
|
2289
|
-
const remaining = maxChars - priorityText.length - 80; // reserve space for note
|
|
2290
|
-
if (remaining > 0) {
|
|
2291
|
-
const bulkText = bulk.join('\n');
|
|
2292
|
-
const trimmedBulk = bulkText.slice(0, remaining);
|
|
2293
|
-
const bulkKept = trimmedBulk.lastIndexOf('\n') > 0
|
|
2294
|
-
? trimmedBulk.slice(0, trimmedBulk.lastIndexOf('\n'))
|
|
2295
|
-
: trimmedBulk;
|
|
2296
|
-
return priorityText + '\n' + bulkKept +
|
|
2297
|
-
`\n... [${interactive.length - priority.length - bulkKept.split('\n').length} interactive + ${decorative.length} decorative elements omitted]`;
|
|
2298
|
-
}
|
|
2299
|
-
// Hard cap: take the first maxChars of the full snapshot
|
|
2300
|
-
return snapshot.slice(0, maxChars) + '\n... [snapshot truncated — large page]';
|
|
2301
|
-
}
|
|
2302
|
-
function compactFirstTurnSnapshot(snapshot) {
|
|
2303
|
-
const compact = AriaSnapshotHelper.formatCompact(snapshot);
|
|
2304
|
-
const basis = compact.length > 0 ? compact : snapshot;
|
|
2305
|
-
const maxChars = 4000;
|
|
2306
|
-
if (basis.length <= maxChars)
|
|
2307
|
-
return basis;
|
|
2308
|
-
return `${basis.slice(0, maxChars)}\n... [snapshot truncated for first-turn fast path]`;
|
|
2309
|
-
}
|
|
2310
|
-
function parseNextActions(parsed, validActions) {
|
|
2311
|
-
if (!Array.isArray(parsed.nextActions)) {
|
|
2312
|
-
return undefined;
|
|
2313
|
-
}
|
|
2314
|
-
const nextActions = [];
|
|
2315
|
-
for (const entry of parsed.nextActions.slice(0, 3)) {
|
|
2316
|
-
if (!entry || typeof entry !== 'object')
|
|
2317
|
-
continue;
|
|
2318
|
-
const rawEntry = entry;
|
|
2319
|
-
const actionType = typeof rawEntry.action === 'string' ? rawEntry.action : undefined;
|
|
2320
|
-
if (!actionType || !validActions.has(actionType))
|
|
2321
|
-
continue;
|
|
2322
|
-
try {
|
|
2323
|
-
nextActions.push(validateAction(actionType, rawEntry));
|
|
2324
|
-
}
|
|
2325
|
-
catch {
|
|
2326
|
-
// Best effort: ignore malformed follow-up action.
|
|
2327
|
-
}
|
|
2328
|
-
}
|
|
2329
|
-
return nextActions.length > 0 ? nextActions : undefined;
|
|
2330
|
-
}
|
|
2331
|
-
/**
|
|
2332
|
-
* Runtime validation of LLM-parsed action objects.
|
|
2333
|
-
* Ensures required fields are present and correctly typed per action variant.
|
|
2334
|
-
* Throws on missing/invalid fields so the caller can abort gracefully.
|
|
2335
|
-
*/
|
|
2336
|
-
function validateAction(actionType, data) {
|
|
2337
|
-
const requireStr = (field) => {
|
|
2338
|
-
const v = data[field];
|
|
2339
|
-
if (typeof v !== 'string' || !v)
|
|
2340
|
-
throw new Error(`${actionType} action requires "${field}" (string)`);
|
|
2341
|
-
return v;
|
|
2342
|
-
};
|
|
2343
|
-
const optStr = (field) => {
|
|
2344
|
-
const v = data[field];
|
|
2345
|
-
return typeof v === 'string' ? v : '';
|
|
2346
|
-
};
|
|
2347
|
-
const num = (v, fallback) => (typeof v === 'number' ? v : fallback);
|
|
2348
|
-
switch (actionType) {
|
|
2349
|
-
case 'click':
|
|
2350
|
-
return { action: 'click', selector: requireStr('selector') };
|
|
2351
|
-
case 'type':
|
|
2352
|
-
return { action: 'type', selector: requireStr('selector'), text: optStr('text') };
|
|
2353
|
-
case 'press':
|
|
2354
|
-
return { action: 'press', selector: requireStr('selector'), key: requireStr('key') };
|
|
2355
|
-
case 'hover':
|
|
2356
|
-
return { action: 'hover', selector: requireStr('selector') };
|
|
2357
|
-
case 'select':
|
|
2358
|
-
return { action: 'select', selector: requireStr('selector'), value: optStr('value') };
|
|
2359
|
-
case 'scroll':
|
|
2360
|
-
return {
|
|
2361
|
-
action: 'scroll',
|
|
2362
|
-
direction: data.direction === 'up' ? 'up' : 'down',
|
|
2363
|
-
...(data.amount != null ? { amount: num(data.amount, 500) } : {}),
|
|
2364
|
-
};
|
|
2365
|
-
case 'navigate':
|
|
2366
|
-
return { action: 'navigate', url: requireStr('url') };
|
|
2367
|
-
case 'wait':
|
|
2368
|
-
return { action: 'wait', ms: num(data.ms, 1000) };
|
|
2369
|
-
case 'evaluate':
|
|
2370
|
-
return { action: 'evaluate', criteria: optStr('criteria') };
|
|
2371
|
-
case 'runScript':
|
|
2372
|
-
return { action: 'runScript', script: requireStr('script') };
|
|
2373
|
-
case 'extractWithIndex': {
|
|
2374
|
-
const query = requireStr('query');
|
|
2375
|
-
const contains = typeof data.contains === 'string' ? data.contains : '';
|
|
2376
|
-
return {
|
|
2377
|
-
action: 'extractWithIndex',
|
|
2378
|
-
query,
|
|
2379
|
-
...(contains ? { contains } : {}),
|
|
2380
|
-
};
|
|
2381
|
-
}
|
|
2382
|
-
case 'verifyPreview':
|
|
2383
|
-
return { action: 'verifyPreview' };
|
|
2384
|
-
case 'complete':
|
|
2385
|
-
return { action: 'complete', result: optStr('result') };
|
|
2386
|
-
case 'abort':
|
|
2387
|
-
return { action: 'abort', reason: optStr('reason') || 'No reason provided' };
|
|
2388
|
-
case 'fill': {
|
|
2389
|
-
// Multi-field batch fill — at least one of fields/selects/checks must be non-empty
|
|
2390
|
-
const fields = isStringRecord(data.fields) ? data.fields : undefined;
|
|
2391
|
-
const selects = isStringRecord(data.selects) ? data.selects : undefined;
|
|
2392
|
-
const checks = Array.isArray(data.checks) && data.checks.every((c) => typeof c === 'string')
|
|
2393
|
-
? data.checks
|
|
2394
|
-
: undefined;
|
|
2395
|
-
const fieldCount = (fields ? Object.keys(fields).length : 0)
|
|
2396
|
-
+ (selects ? Object.keys(selects).length : 0)
|
|
2397
|
-
+ (checks ? checks.length : 0);
|
|
2398
|
-
if (fieldCount === 0) {
|
|
2399
|
-
throw new Error('fill action requires at least one of "fields" (object), "selects" (object), or "checks" (string[])');
|
|
2400
|
-
}
|
|
2401
|
-
return {
|
|
2402
|
-
action: 'fill',
|
|
2403
|
-
...(fields ? { fields } : {}),
|
|
2404
|
-
...(selects ? { selects } : {}),
|
|
2405
|
-
...(checks ? { checks } : {}),
|
|
2406
|
-
};
|
|
2407
|
-
}
|
|
2408
|
-
case 'clickSequence': {
|
|
2409
|
-
const refs = Array.isArray(data.refs) && data.refs.every((r) => typeof r === 'string')
|
|
2410
|
-
? data.refs
|
|
2411
|
-
: null;
|
|
2412
|
-
if (!refs || refs.length === 0) {
|
|
2413
|
-
throw new Error('clickSequence action requires "refs" (string[]) with at least one entry');
|
|
2414
|
-
}
|
|
2415
|
-
return {
|
|
2416
|
-
action: 'clickSequence',
|
|
2417
|
-
refs,
|
|
2418
|
-
...(typeof data.intervalMs === 'number' ? { intervalMs: data.intervalMs } : {}),
|
|
2419
|
-
};
|
|
2420
|
-
}
|
|
2421
|
-
// Vision-first coordinate actions.
|
|
2422
|
-
case 'clickAt':
|
|
2423
|
-
return { action: 'clickAt', x: num(data.x, 0), y: num(data.y, 0) };
|
|
2424
|
-
case 'typeAt':
|
|
2425
|
-
return { action: 'typeAt', x: num(data.x, 0), y: num(data.y, 0), text: optStr('text') };
|
|
2426
|
-
// Set-of-Marks label-based actions.
|
|
2427
|
-
case 'clickLabel':
|
|
2428
|
-
return { action: 'clickLabel', label: num(data.label, 0) };
|
|
2429
|
-
case 'typeLabel':
|
|
2430
|
-
return { action: 'typeLabel', label: num(data.label, 0), text: optStr('text') };
|
|
2431
|
-
// Macro invocation. The driver validates the name and required args at
|
|
2432
|
-
// execute time; macros without params may omit args.
|
|
2433
|
-
case 'macro': {
|
|
2434
|
-
const args = {};
|
|
2435
|
-
if (data.args && typeof data.args === 'object' && !Array.isArray(data.args)) {
|
|
2436
|
-
for (const [k, v] of Object.entries(data.args)) {
|
|
2437
|
-
if (typeof v === 'string')
|
|
2438
|
-
args[k] = v;
|
|
2439
|
-
}
|
|
2440
|
-
}
|
|
2441
|
-
return {
|
|
2442
|
-
action: 'macro',
|
|
2443
|
-
name: requireStr('name'),
|
|
2444
|
-
...(Object.keys(args).length > 0 ? { args } : {}),
|
|
2445
|
-
};
|
|
2446
|
-
}
|
|
2447
|
-
case 'fanOut': {
|
|
2448
|
-
// Accept either explicit subGoals[] or the shorthand
|
|
2449
|
-
// baseUrl+goalTemplate+items trio. resolveSubGoals() in
|
|
2450
|
-
// runner/fan-out.ts expands shorthand → subGoals at execute time.
|
|
2451
|
-
const subGoals = Array.isArray(data.subGoals)
|
|
2452
|
-
? data.subGoals
|
|
2453
|
-
.filter((s) => s && typeof s === 'object')
|
|
2454
|
-
.map((s) => ({
|
|
2455
|
-
url: typeof s.url === 'string' ? s.url : '',
|
|
2456
|
-
goal: typeof s.goal === 'string' ? s.goal : '',
|
|
2457
|
-
...(typeof s.label === 'string' ? { label: s.label } : {}),
|
|
2458
|
-
...(typeof s.maxTurns === 'number' ? { maxTurns: s.maxTurns } : {}),
|
|
2459
|
-
}))
|
|
2460
|
-
.filter((s) => s.url && s.goal)
|
|
2461
|
-
: undefined;
|
|
2462
|
-
const baseUrl = typeof data.baseUrl === 'string' ? data.baseUrl : undefined;
|
|
2463
|
-
const goalTemplate = typeof data.goalTemplate === 'string' ? data.goalTemplate : undefined;
|
|
2464
|
-
const items = Array.isArray(data.items) && data.items.every((i) => typeof i === 'string')
|
|
2465
|
-
? data.items
|
|
2466
|
-
: undefined;
|
|
2467
|
-
const hasExplicit = subGoals && subGoals.length > 0;
|
|
2468
|
-
const hasShorthand = baseUrl && goalTemplate && items && items.length > 0;
|
|
2469
|
-
if (!hasExplicit && !hasShorthand) {
|
|
2470
|
-
throw new Error('fanOut action requires either "subGoals" (array of {url,goal}) or the shorthand trio "baseUrl" + "goalTemplate" + "items" (non-empty string[])');
|
|
2471
|
-
}
|
|
2472
|
-
return {
|
|
2473
|
-
action: 'fanOut',
|
|
2474
|
-
...(hasExplicit ? { subGoals } : {}),
|
|
2475
|
-
...(hasShorthand ? { baseUrl, goalTemplate, items } : {}),
|
|
2476
|
-
...(typeof data.summarize === 'string' ? { summarize: data.summarize } : {}),
|
|
2477
|
-
};
|
|
2478
|
-
}
|
|
2479
|
-
default:
|
|
2480
|
-
throw new Error(`Unknown action type: ${actionType}`);
|
|
294
|
+
/**
|
|
295
|
+
* Multimodal sibling of {@link complete}: a single round-trip with a system
|
|
296
|
+
* prompt, a user prompt, and one-or-more already-encoded images. The narrow
|
|
297
|
+
* vision seam the reference-grounded taste judge binds to (one Brain per
|
|
298
|
+
* `{ provider, model }` ref), built on the same private `generate` so it goes
|
|
299
|
+
* through the existing provider abstraction — any vision-capable backend works.
|
|
300
|
+
*
|
|
301
|
+
* Images arrive ENCODED (`{ image, mediaType }`, base64/data); the disk read +
|
|
302
|
+
* mediaType inference live in the judge's `createBrainVisionModel` adapter, so
|
|
303
|
+
* this method does no IO. It is NOT `auditDesign` — the page-audit seam stays
|
|
304
|
+
* off-limits to taste comparison by contract.
|
|
305
|
+
*/
|
|
306
|
+
async completeVision(system, user, images, options = {}) {
|
|
307
|
+
return completeVisionImpl(this, system, user, images, options);
|
|
2481
308
|
}
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
function isStringRecord(value) {
|
|
2485
|
-
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
2486
|
-
return false;
|
|
2487
|
-
for (const v of Object.values(value)) {
|
|
2488
|
-
if (typeof v !== 'string')
|
|
2489
|
-
return false;
|
|
309
|
+
parse(raw) {
|
|
310
|
+
return parseDecision(raw);
|
|
2490
311
|
}
|
|
2491
|
-
return true;
|
|
2492
|
-
}
|
|
2493
|
-
/**
|
|
2494
|
-
* Build a fetch replacement that forces `"stream": false` on chat completions
|
|
2495
|
-
* bodies for OpenAI-compatible gateways that default to SSE streaming.
|
|
2496
|
-
*/
|
|
2497
|
-
function createForceNonStreamingFetch() {
|
|
2498
|
-
return async (input, init) => {
|
|
2499
|
-
if (init?.body && typeof init.body === 'string') {
|
|
2500
|
-
const body = init.body;
|
|
2501
|
-
// Cheap content-sniff so we only rewrite chat-completions shaped bodies,
|
|
2502
|
-
// not arbitrary POSTs the caller might make (embeddings, etc.).
|
|
2503
|
-
if (body.includes('"messages"') && body.includes('"model"')) {
|
|
2504
|
-
try {
|
|
2505
|
-
const parsed = JSON.parse(body);
|
|
2506
|
-
if (parsed.stream === undefined || parsed.stream === true) {
|
|
2507
|
-
parsed.stream = false;
|
|
2508
|
-
init = { ...init, body: JSON.stringify(parsed) };
|
|
2509
|
-
}
|
|
2510
|
-
}
|
|
2511
|
-
catch {
|
|
2512
|
-
// Non-JSON body — pass through unchanged.
|
|
2513
|
-
}
|
|
2514
|
-
}
|
|
2515
|
-
}
|
|
2516
|
-
return fetch(input, init);
|
|
2517
|
-
};
|
|
2518
312
|
}
|
|
2519
313
|
//# sourceMappingURL=index.js.map
|