@tangle-network/browser-agent-driver 0.33.3 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (485) hide show
  1. package/README.md +42 -1
  2. package/discovery/llms.txt +27 -0
  3. package/discovery/tangle-agent.json +36 -0
  4. package/dist/brain/action-parse.d.ts +16 -0
  5. package/dist/brain/action-parse.d.ts.map +1 -0
  6. package/dist/brain/action-parse.js +191 -0
  7. package/dist/brain/action-parse.js.map +1 -0
  8. package/dist/brain/decide.d.ts +58 -0
  9. package/dist/brain/decide.d.ts.map +1 -0
  10. package/dist/brain/decide.js +418 -0
  11. package/dist/brain/decide.js.map +1 -0
  12. package/dist/brain/history-compact.d.ts +19 -0
  13. package/dist/brain/history-compact.d.ts.map +1 -0
  14. package/dist/brain/history-compact.js +153 -0
  15. package/dist/brain/history-compact.js.map +1 -0
  16. package/dist/brain/index.d.ts +78 -152
  17. package/dist/brain/index.d.ts.map +1 -1
  18. package/dist/brain/index.js +62 -2268
  19. package/dist/brain/index.js.map +1 -1
  20. package/dist/brain/model-client.d.ts +101 -0
  21. package/dist/brain/model-client.d.ts.map +1 -0
  22. package/dist/brain/model-client.js +391 -0
  23. package/dist/brain/model-client.js.map +1 -0
  24. package/dist/brain/plan.d.ts +57 -0
  25. package/dist/brain/plan.d.ts.map +1 -0
  26. package/dist/brain/plan.js +194 -0
  27. package/dist/brain/plan.js.map +1 -0
  28. package/dist/brain/prompts.d.ts +37 -0
  29. package/dist/brain/prompts.d.ts.map +1 -0
  30. package/dist/brain/prompts.js +455 -0
  31. package/dist/brain/prompts.js.map +1 -0
  32. package/dist/brain/provider-fetch.d.ts +26 -0
  33. package/dist/brain/provider-fetch.d.ts.map +1 -0
  34. package/dist/brain/provider-fetch.js +45 -0
  35. package/dist/brain/provider-fetch.js.map +1 -0
  36. package/dist/brain/snapshot-budget.d.ts +20 -0
  37. package/dist/brain/snapshot-budget.d.ts.map +1 -0
  38. package/dist/brain/snapshot-budget.js +202 -0
  39. package/dist/brain/snapshot-budget.js.map +1 -0
  40. package/dist/brain/system-prompt.d.ts +67 -0
  41. package/dist/brain/system-prompt.d.ts.map +1 -0
  42. package/dist/brain/system-prompt.js +128 -0
  43. package/dist/brain/system-prompt.js.map +1 -0
  44. package/dist/brain/tasks/design-audit.d.ts +33 -0
  45. package/dist/brain/tasks/design-audit.d.ts.map +1 -0
  46. package/dist/brain/tasks/design-audit.js +100 -0
  47. package/dist/brain/tasks/design-audit.js.map +1 -0
  48. package/dist/brain/tasks/evaluate.d.ts +25 -0
  49. package/dist/brain/tasks/evaluate.d.ts.map +1 -0
  50. package/dist/brain/tasks/evaluate.js +55 -0
  51. package/dist/brain/tasks/evaluate.js.map +1 -0
  52. package/dist/brain/tasks/goal-verification.d.ts +32 -0
  53. package/dist/brain/tasks/goal-verification.d.ts.map +1 -0
  54. package/dist/brain/tasks/goal-verification.js +68 -0
  55. package/dist/brain/tasks/goal-verification.js.map +1 -0
  56. package/dist/brain/tasks/knowledge.d.ts +26 -0
  57. package/dist/brain/tasks/knowledge.d.ts.map +1 -0
  58. package/dist/brain/tasks/knowledge.js +56 -0
  59. package/dist/brain/tasks/knowledge.js.map +1 -0
  60. package/dist/brain/tasks/link-scout.d.ts +37 -0
  61. package/dist/brain/tasks/link-scout.d.ts.map +1 -0
  62. package/dist/brain/tasks/link-scout.js +69 -0
  63. package/dist/brain/tasks/link-scout.js.map +1 -0
  64. package/dist/brain/types.d.ts +55 -0
  65. package/dist/brain/types.d.ts.map +1 -0
  66. package/dist/brain/types.js +7 -0
  67. package/dist/brain/types.js.map +1 -0
  68. package/dist/browser/stealth-init-script.d.ts +9 -0
  69. package/dist/browser/stealth-init-script.d.ts.map +1 -0
  70. package/dist/browser/stealth-init-script.js +128 -0
  71. package/dist/browser/stealth-init-script.js.map +1 -0
  72. package/dist/browser/storage-state.d.ts +29 -0
  73. package/dist/browser/storage-state.d.ts.map +1 -0
  74. package/dist/browser/storage-state.js +47 -0
  75. package/dist/browser/storage-state.js.map +1 -0
  76. package/dist/cli/args.d.ts +141 -0
  77. package/dist/cli/args.d.ts.map +1 -0
  78. package/dist/cli/args.js +167 -0
  79. package/dist/cli/args.js.map +1 -0
  80. package/dist/cli/benchmark-sync.d.ts +9 -0
  81. package/dist/cli/benchmark-sync.d.ts.map +1 -0
  82. package/dist/cli/benchmark-sync.js +43 -0
  83. package/dist/cli/benchmark-sync.js.map +1 -0
  84. package/dist/cli/build-config.d.ts +11 -0
  85. package/dist/cli/build-config.d.ts.map +1 -0
  86. package/dist/cli/build-config.js +355 -0
  87. package/dist/cli/build-config.js.map +1 -0
  88. package/dist/cli/commands/auth.d.ts +3 -0
  89. package/dist/cli/commands/auth.d.ts.map +1 -0
  90. package/dist/cli/commands/auth.js +36 -0
  91. package/dist/cli/commands/auth.js.map +1 -0
  92. package/dist/cli/commands/chrome-debug.d.ts +7 -0
  93. package/dist/cli/commands/chrome-debug.d.ts.map +1 -0
  94. package/dist/cli/commands/chrome-debug.js +11 -0
  95. package/dist/cli/commands/chrome-debug.js.map +1 -0
  96. package/dist/cli/commands/design-audit.d.ts +3 -0
  97. package/dist/cli/commands/design-audit.d.ts.map +1 -0
  98. package/dist/cli/commands/design-audit.js +68 -0
  99. package/dist/cli/commands/design-audit.js.map +1 -0
  100. package/dist/cli/commands/preview.d.ts +14 -0
  101. package/dist/cli/commands/preview.d.ts.map +1 -0
  102. package/dist/cli/commands/preview.js +31 -0
  103. package/dist/cli/commands/preview.js.map +1 -0
  104. package/dist/cli/commands/run.d.ts +3 -0
  105. package/dist/cli/commands/run.d.ts.map +1 -0
  106. package/dist/cli/commands/run.js +728 -0
  107. package/dist/cli/commands/run.js.map +1 -0
  108. package/dist/cli/commands/runs.d.ts +8 -0
  109. package/dist/cli/commands/runs.d.ts.map +1 -0
  110. package/dist/cli/commands/runs.js +35 -0
  111. package/dist/cli/commands/runs.js.map +1 -0
  112. package/dist/cli/commands/share.d.ts +10 -0
  113. package/dist/cli/commands/share.d.ts.map +1 -0
  114. package/dist/cli/commands/share.js +32 -0
  115. package/dist/cli/commands/share.js.map +1 -0
  116. package/dist/cli/commands/showcase.d.ts +17 -0
  117. package/dist/cli/commands/showcase.d.ts.map +1 -0
  118. package/dist/cli/commands/showcase.js +20 -0
  119. package/dist/cli/commands/showcase.js.map +1 -0
  120. package/dist/cli/commands/snapshot.d.ts +12 -0
  121. package/dist/cli/commands/snapshot.d.ts.map +1 -0
  122. package/dist/cli/commands/snapshot.js +24 -0
  123. package/dist/cli/commands/snapshot.js.map +1 -0
  124. package/dist/cli/commands/view.d.ts +7 -0
  125. package/dist/cli/commands/view.d.ts.map +1 -0
  126. package/dist/cli/commands/view.js +23 -0
  127. package/dist/cli/commands/view.js.map +1 -0
  128. package/dist/cli/constants.d.ts +6 -0
  129. package/dist/cli/constants.d.ts.map +1 -0
  130. package/dist/cli/constants.js +3 -0
  131. package/dist/cli/constants.js.map +1 -0
  132. package/dist/cli/version.d.ts +7 -0
  133. package/dist/cli/version.d.ts.map +1 -0
  134. package/dist/cli/version.js +16 -0
  135. package/dist/cli/version.js.map +1 -0
  136. package/dist/cli-design-audit.d.ts +44 -36
  137. package/dist/cli-design-audit.d.ts.map +1 -1
  138. package/dist/cli-design-audit.js +137 -1646
  139. package/dist/cli-design-audit.js.map +1 -1
  140. package/dist/cli-jobs.js +2 -1
  141. package/dist/cli-jobs.js.map +1 -1
  142. package/dist/cli.js +66 -1697
  143. package/dist/cli.js.map +1 -1
  144. package/dist/config.d.ts +5 -0
  145. package/dist/config.d.ts.map +1 -1
  146. package/dist/config.js +8 -0
  147. package/dist/config.js.map +1 -1
  148. package/dist/design/audit/evolve/agent.d.ts +23 -0
  149. package/dist/design/audit/evolve/agent.d.ts.map +1 -0
  150. package/dist/design/audit/evolve/agent.js +272 -0
  151. package/dist/design/audit/evolve/agent.js.map +1 -0
  152. package/dist/design/audit/evolve/css.d.ts +10 -0
  153. package/dist/design/audit/evolve/css.d.ts.map +1 -0
  154. package/dist/design/audit/evolve/css.js +206 -0
  155. package/dist/design/audit/evolve/css.js.map +1 -0
  156. package/dist/design/audit/evolve/index.d.ts +13 -0
  157. package/dist/design/audit/evolve/index.d.ts.map +1 -0
  158. package/dist/design/audit/evolve/index.js +12 -0
  159. package/dist/design/audit/evolve/index.js.map +1 -0
  160. package/dist/design/audit/evolve/report.d.ts +3 -0
  161. package/dist/design/audit/evolve/report.d.ts.map +1 -0
  162. package/dist/design/audit/evolve/report.js +45 -0
  163. package/dist/design/audit/evolve/report.js.map +1 -0
  164. package/dist/design/audit/evolve/types.d.ts +14 -0
  165. package/dist/design/audit/evolve/types.d.ts.map +1 -0
  166. package/dist/design/audit/evolve/types.js +2 -0
  167. package/dist/design/audit/evolve/types.js.map +1 -0
  168. package/dist/design/audit/pipeline.d.ts +12 -0
  169. package/dist/design/audit/pipeline.d.ts.map +1 -1
  170. package/dist/design/audit/pipeline.js +43 -27
  171. package/dist/design/audit/pipeline.js.map +1 -1
  172. package/dist/design/audit/reference/artifact/build.d.ts +33 -0
  173. package/dist/design/audit/reference/artifact/build.d.ts.map +1 -0
  174. package/dist/design/audit/reference/artifact/build.js +64 -0
  175. package/dist/design/audit/reference/artifact/build.js.map +1 -0
  176. package/dist/design/audit/reference/artifact/render.d.ts +39 -0
  177. package/dist/design/audit/reference/artifact/render.d.ts.map +1 -0
  178. package/dist/design/audit/reference/artifact/render.js +213 -0
  179. package/dist/design/audit/reference/artifact/render.js.map +1 -0
  180. package/dist/design/audit/reference/artifact/text.d.ts +11 -0
  181. package/dist/design/audit/reference/artifact/text.d.ts.map +1 -0
  182. package/dist/design/audit/reference/artifact/text.js +20 -0
  183. package/dist/design/audit/reference/artifact/text.js.map +1 -0
  184. package/dist/design/audit/reference/artifact/to-findings.d.ts +57 -0
  185. package/dist/design/audit/reference/artifact/to-findings.d.ts.map +1 -0
  186. package/dist/design/audit/reference/artifact/to-findings.js +314 -0
  187. package/dist/design/audit/reference/artifact/to-findings.js.map +1 -0
  188. package/dist/design/audit/reference/config.d.ts +50 -0
  189. package/dist/design/audit/reference/config.d.ts.map +1 -0
  190. package/dist/design/audit/reference/config.js +108 -0
  191. package/dist/design/audit/reference/config.js.map +1 -0
  192. package/dist/design/audit/reference/contracts.d.ts +1030 -0
  193. package/dist/design/audit/reference/contracts.d.ts.map +1 -0
  194. package/dist/design/audit/reference/contracts.js +28 -0
  195. package/dist/design/audit/reference/contracts.js.map +1 -0
  196. package/dist/design/audit/reference/corpus/build.d.ts +115 -0
  197. package/dist/design/audit/reference/corpus/build.d.ts.map +1 -0
  198. package/dist/design/audit/reference/corpus/build.js +152 -0
  199. package/dist/design/audit/reference/corpus/build.js.map +1 -0
  200. package/dist/design/audit/reference/corpus/schema.d.ts +33 -0
  201. package/dist/design/audit/reference/corpus/schema.d.ts.map +1 -0
  202. package/dist/design/audit/reference/corpus/schema.js +249 -0
  203. package/dist/design/audit/reference/corpus/schema.js.map +1 -0
  204. package/dist/design/audit/reference/corpus/store.d.ts +26 -0
  205. package/dist/design/audit/reference/corpus/store.d.ts.map +1 -0
  206. package/dist/design/audit/reference/corpus/store.js +104 -0
  207. package/dist/design/audit/reference/corpus/store.js.map +1 -0
  208. package/dist/design/audit/reference/dna/delta.d.ts +21 -0
  209. package/dist/design/audit/reference/dna/delta.d.ts.map +1 -0
  210. package/dist/design/audit/reference/dna/delta.js +126 -0
  211. package/dist/design/audit/reference/dna/delta.js.map +1 -0
  212. package/dist/design/audit/reference/dna/derive.d.ts +31 -0
  213. package/dist/design/audit/reference/dna/derive.d.ts.map +1 -0
  214. package/dist/design/audit/reference/dna/derive.js +381 -0
  215. package/dist/design/audit/reference/dna/derive.js.map +1 -0
  216. package/dist/design/audit/reference/dna/descriptor.d.ts +36 -0
  217. package/dist/design/audit/reference/dna/descriptor.d.ts.map +1 -0
  218. package/dist/design/audit/reference/dna/descriptor.js +209 -0
  219. package/dist/design/audit/reference/dna/descriptor.js.map +1 -0
  220. package/dist/design/audit/reference/dna/page-adapter.d.ts +17 -0
  221. package/dist/design/audit/reference/dna/page-adapter.d.ts.map +1 -0
  222. package/dist/design/audit/reference/dna/page-adapter.js +35 -0
  223. package/dist/design/audit/reference/dna/page-adapter.js.map +1 -0
  224. package/dist/design/audit/reference/dna/scroll-capture.d.ts +93 -0
  225. package/dist/design/audit/reference/dna/scroll-capture.d.ts.map +1 -0
  226. package/dist/design/audit/reference/dna/scroll-capture.js +374 -0
  227. package/dist/design/audit/reference/dna/scroll-capture.js.map +1 -0
  228. package/dist/design/audit/reference/engine/budget.d.ts +63 -0
  229. package/dist/design/audit/reference/engine/budget.d.ts.map +1 -0
  230. package/dist/design/audit/reference/engine/budget.js +110 -0
  231. package/dist/design/audit/reference/engine/budget.js.map +1 -0
  232. package/dist/design/audit/reference/engine/core.d.ts +32 -0
  233. package/dist/design/audit/reference/engine/core.d.ts.map +1 -0
  234. package/dist/design/audit/reference/engine/core.js +261 -0
  235. package/dist/design/audit/reference/engine/core.js.map +1 -0
  236. package/dist/design/audit/reference/engine/guard.d.ts +43 -0
  237. package/dist/design/audit/reference/engine/guard.d.ts.map +1 -0
  238. package/dist/design/audit/reference/engine/guard.js +43 -0
  239. package/dist/design/audit/reference/engine/guard.js.map +1 -0
  240. package/dist/design/audit/reference/engine/score-core.d.ts +48 -0
  241. package/dist/design/audit/reference/engine/score-core.d.ts.map +1 -0
  242. package/dist/design/audit/reference/engine/score-core.js +134 -0
  243. package/dist/design/audit/reference/engine/score-core.js.map +1 -0
  244. package/dist/design/audit/reference/engine/wiring.d.ts +35 -0
  245. package/dist/design/audit/reference/engine/wiring.d.ts.map +1 -0
  246. package/dist/design/audit/reference/engine/wiring.js +80 -0
  247. package/dist/design/audit/reference/engine/wiring.js.map +1 -0
  248. package/dist/design/audit/reference/eval/taste-core.d.ts +77 -0
  249. package/dist/design/audit/reference/eval/taste-core.d.ts.map +1 -0
  250. package/dist/design/audit/reference/eval/taste-core.js +95 -0
  251. package/dist/design/audit/reference/eval/taste-core.js.map +1 -0
  252. package/dist/design/audit/reference/generate/generator.d.ts +49 -0
  253. package/dist/design/audit/reference/generate/generator.d.ts.map +1 -0
  254. package/dist/design/audit/reference/generate/generator.js +94 -0
  255. package/dist/design/audit/reference/generate/generator.js.map +1 -0
  256. package/dist/design/audit/reference/generate/parse.d.ts +28 -0
  257. package/dist/design/audit/reference/generate/parse.d.ts.map +1 -0
  258. package/dist/design/audit/reference/generate/parse.js +180 -0
  259. package/dist/design/audit/reference/generate/parse.js.map +1 -0
  260. package/dist/design/audit/reference/generate/prompt.d.ts +36 -0
  261. package/dist/design/audit/reference/generate/prompt.d.ts.map +1 -0
  262. package/dist/design/audit/reference/generate/prompt.js +124 -0
  263. package/dist/design/audit/reference/generate/prompt.js.map +1 -0
  264. package/dist/design/audit/reference/index.d.ts +26 -0
  265. package/dist/design/audit/reference/index.d.ts.map +1 -0
  266. package/dist/design/audit/reference/index.js +31 -0
  267. package/dist/design/audit/reference/index.js.map +1 -0
  268. package/dist/design/audit/reference/judge/image-clamp.d.ts +34 -0
  269. package/dist/design/audit/reference/judge/image-clamp.d.ts.map +1 -0
  270. package/dist/design/audit/reference/judge/image-clamp.js +88 -0
  271. package/dist/design/audit/reference/judge/image-clamp.js.map +1 -0
  272. package/dist/design/audit/reference/judge/pairwise.d.ts +34 -0
  273. package/dist/design/audit/reference/judge/pairwise.d.ts.map +1 -0
  274. package/dist/design/audit/reference/judge/pairwise.js +105 -0
  275. package/dist/design/audit/reference/judge/pairwise.js.map +1 -0
  276. package/dist/design/audit/reference/judge/parse.d.ts +31 -0
  277. package/dist/design/audit/reference/judge/parse.d.ts.map +1 -0
  278. package/dist/design/audit/reference/judge/parse.js +113 -0
  279. package/dist/design/audit/reference/judge/parse.js.map +1 -0
  280. package/dist/design/audit/reference/judge/prompt.d.ts +38 -0
  281. package/dist/design/audit/reference/judge/prompt.d.ts.map +1 -0
  282. package/dist/design/audit/reference/judge/prompt.js +111 -0
  283. package/dist/design/audit/reference/judge/prompt.js.map +1 -0
  284. package/dist/design/audit/reference/judge/quality.d.ts +28 -0
  285. package/dist/design/audit/reference/judge/quality.d.ts.map +1 -0
  286. package/dist/design/audit/reference/judge/quality.js +62 -0
  287. package/dist/design/audit/reference/judge/quality.js.map +1 -0
  288. package/dist/design/audit/reference/judge/rank.d.ts +41 -0
  289. package/dist/design/audit/reference/judge/rank.d.ts.map +1 -0
  290. package/dist/design/audit/reference/judge/rank.js +196 -0
  291. package/dist/design/audit/reference/judge/rank.js.map +1 -0
  292. package/dist/design/audit/reference/judge/text-judge.d.ts +31 -0
  293. package/dist/design/audit/reference/judge/text-judge.d.ts.map +1 -0
  294. package/dist/design/audit/reference/judge/text-judge.js +43 -0
  295. package/dist/design/audit/reference/judge/text-judge.js.map +1 -0
  296. package/dist/design/audit/reference/judge/vision-judge.d.ts +59 -0
  297. package/dist/design/audit/reference/judge/vision-judge.d.ts.map +1 -0
  298. package/dist/design/audit/reference/judge/vision-judge.js +150 -0
  299. package/dist/design/audit/reference/judge/vision-judge.js.map +1 -0
  300. package/dist/design/audit/reference/judge/vision-model.d.ts +65 -0
  301. package/dist/design/audit/reference/judge/vision-model.d.ts.map +1 -0
  302. package/dist/design/audit/reference/judge/vision-model.js +110 -0
  303. package/dist/design/audit/reference/judge/vision-model.js.map +1 -0
  304. package/dist/design/audit/reference/pipeline/evaluate-reference.d.ts +68 -0
  305. package/dist/design/audit/reference/pipeline/evaluate-reference.d.ts.map +1 -0
  306. package/dist/design/audit/reference/pipeline/evaluate-reference.js +52 -0
  307. package/dist/design/audit/reference/pipeline/evaluate-reference.js.map +1 -0
  308. package/dist/design/audit/reference/reference-context.d.ts +31 -0
  309. package/dist/design/audit/reference/reference-context.d.ts.map +1 -0
  310. package/dist/design/audit/reference/reference-context.js +73 -0
  311. package/dist/design/audit/reference/reference-context.js.map +1 -0
  312. package/dist/design/audit/reference/retrieval/embedding-hash.d.ts +30 -0
  313. package/dist/design/audit/reference/retrieval/embedding-hash.d.ts.map +1 -0
  314. package/dist/design/audit/reference/retrieval/embedding-hash.js +87 -0
  315. package/dist/design/audit/reference/retrieval/embedding-hash.js.map +1 -0
  316. package/dist/design/audit/reference/retrieval/embedding-openai.d.ts +38 -0
  317. package/dist/design/audit/reference/retrieval/embedding-openai.d.ts.map +1 -0
  318. package/dist/design/audit/reference/retrieval/embedding-openai.js +73 -0
  319. package/dist/design/audit/reference/retrieval/embedding-openai.js.map +1 -0
  320. package/dist/design/audit/reference/retrieval/matcher.d.ts +34 -0
  321. package/dist/design/audit/reference/retrieval/matcher.d.ts.map +1 -0
  322. package/dist/design/audit/reference/retrieval/matcher.js +107 -0
  323. package/dist/design/audit/reference/retrieval/matcher.js.map +1 -0
  324. package/dist/design/audit/reference/run.d.ts +59 -0
  325. package/dist/design/audit/reference/run.d.ts.map +1 -0
  326. package/dist/design/audit/reference/run.js +99 -0
  327. package/dist/design/audit/reference/run.js.map +1 -0
  328. package/dist/design/audit/report.d.ts +35 -0
  329. package/dist/design/audit/report.d.ts.map +1 -0
  330. package/dist/design/audit/report.js +195 -0
  331. package/dist/design/audit/report.js.map +1 -0
  332. package/dist/design/audit/tokens/extract.d.ts +71 -0
  333. package/dist/design/audit/tokens/extract.d.ts.map +1 -0
  334. package/dist/design/audit/tokens/extract.js +1034 -0
  335. package/dist/design/audit/tokens/extract.js.map +1 -0
  336. package/dist/design/audit/types.d.ts +8 -0
  337. package/dist/design/audit/types.d.ts.map +1 -1
  338. package/dist/design/compare.d.ts.map +1 -1
  339. package/dist/design/compare.js +2 -6
  340. package/dist/design/compare.js.map +1 -1
  341. package/dist/design/cookie-consent.d.ts +3 -0
  342. package/dist/design/cookie-consent.d.ts.map +1 -0
  343. package/dist/design/cookie-consent.js +26 -0
  344. package/dist/design/cookie-consent.js.map +1 -0
  345. package/dist/design/viewports.d.ts +18 -0
  346. package/dist/design/viewports.d.ts.map +1 -0
  347. package/dist/design/viewports.js +10 -0
  348. package/dist/design/viewports.js.map +1 -0
  349. package/dist/index.d.ts +2 -2
  350. package/dist/index.d.ts.map +1 -1
  351. package/dist/index.js +1 -1
  352. package/dist/index.js.map +1 -1
  353. package/dist/memory/store.d.ts +19 -1
  354. package/dist/memory/store.d.ts.map +1 -1
  355. package/dist/memory/store.js +47 -0
  356. package/dist/memory/store.js.map +1 -1
  357. package/dist/provider-defaults.d.ts +18 -0
  358. package/dist/provider-defaults.d.ts.map +1 -1
  359. package/dist/provider-defaults.js +33 -0
  360. package/dist/provider-defaults.js.map +1 -1
  361. package/dist/runner/allowed-domains.d.ts +11 -0
  362. package/dist/runner/allowed-domains.d.ts.map +1 -0
  363. package/dist/runner/allowed-domains.js +14 -0
  364. package/dist/runner/allowed-domains.js.map +1 -0
  365. package/dist/runner/batch-fill.d.ts +28 -0
  366. package/dist/runner/batch-fill.d.ts.map +1 -0
  367. package/dist/runner/batch-fill.js +80 -0
  368. package/dist/runner/batch-fill.js.map +1 -0
  369. package/dist/runner/completion-policy.d.ts +26 -0
  370. package/dist/runner/completion-policy.d.ts.map +1 -0
  371. package/dist/runner/completion-policy.js +76 -0
  372. package/dist/runner/completion-policy.js.map +1 -0
  373. package/dist/runner/constants.d.ts +14 -0
  374. package/dist/runner/constants.d.ts.map +1 -0
  375. package/dist/runner/constants.js +23 -0
  376. package/dist/runner/constants.js.map +1 -0
  377. package/dist/runner/decision-screenshot.d.ts +24 -0
  378. package/dist/runner/decision-screenshot.d.ts.map +1 -0
  379. package/dist/runner/decision-screenshot.js +23 -0
  380. package/dist/runner/decision-screenshot.js.map +1 -0
  381. package/dist/runner/domain-boundary.d.ts +36 -0
  382. package/dist/runner/domain-boundary.d.ts.map +1 -0
  383. package/dist/runner/domain-boundary.js +81 -0
  384. package/dist/runner/domain-boundary.js.map +1 -0
  385. package/dist/runner/effect-verify.d.ts +28 -0
  386. package/dist/runner/effect-verify.d.ts.map +1 -0
  387. package/dist/runner/effect-verify.js +42 -0
  388. package/dist/runner/effect-verify.js.map +1 -0
  389. package/dist/runner/execute-plan.d.ts +58 -0
  390. package/dist/runner/execute-plan.d.ts.map +1 -0
  391. package/dist/runner/execute-plan.js +426 -0
  392. package/dist/runner/execute-plan.js.map +1 -0
  393. package/dist/runner/max-turns-extension.d.ts +19 -0
  394. package/dist/runner/max-turns-extension.d.ts.map +1 -0
  395. package/dist/runner/max-turns-extension.js +25 -0
  396. package/dist/runner/max-turns-extension.js.map +1 -0
  397. package/dist/runner/micro-plan.d.ts +11 -0
  398. package/dist/runner/micro-plan.d.ts.map +1 -0
  399. package/dist/runner/micro-plan.js +29 -0
  400. package/dist/runner/micro-plan.js.map +1 -0
  401. package/dist/runner/prompt-snippets.d.ts +5 -0
  402. package/dist/runner/prompt-snippets.d.ts.map +1 -0
  403. package/dist/runner/prompt-snippets.js +13 -0
  404. package/dist/runner/prompt-snippets.js.map +1 -0
  405. package/dist/runner/replay/contracts.d.ts +300 -0
  406. package/dist/runner/replay/contracts.d.ts.map +1 -0
  407. package/dist/runner/replay/contracts.js +42 -0
  408. package/dist/runner/replay/contracts.js.map +1 -0
  409. package/dist/runner/replay/controller.d.ts +27 -0
  410. package/dist/runner/replay/controller.d.ts.map +1 -0
  411. package/dist/runner/replay/controller.js +197 -0
  412. package/dist/runner/replay/controller.js.map +1 -0
  413. package/dist/runner/replay/guard.d.ts +28 -0
  414. package/dist/runner/replay/guard.d.ts.map +1 -0
  415. package/dist/runner/replay/guard.js +100 -0
  416. package/dist/runner/replay/guard.js.map +1 -0
  417. package/dist/runner/runner.d.ts +44 -51
  418. package/dist/runner/runner.d.ts.map +1 -1
  419. package/dist/runner/runner.js +155 -851
  420. package/dist/runner/runner.js.map +1 -1
  421. package/dist/runner/scout-feedback.d.ts +51 -0
  422. package/dist/runner/scout-feedback.d.ts.map +1 -0
  423. package/dist/runner/scout-feedback.js +149 -0
  424. package/dist/runner/scout-feedback.js.map +1 -0
  425. package/dist/skills/macro-loader.d.ts +1 -1
  426. package/dist/test-runner.d.ts.map +1 -1
  427. package/dist/test-runner.js +25 -0
  428. package/dist/test-runner.js.map +1 -1
  429. package/dist/types/actions.d.ts +241 -0
  430. package/dist/types/actions.d.ts.map +1 -0
  431. package/dist/types/actions.js +5 -0
  432. package/dist/types/actions.js.map +1 -0
  433. package/dist/types/config.d.ts +211 -0
  434. package/dist/types/config.d.ts.map +1 -0
  435. package/dist/types/config.js +2 -0
  436. package/dist/types/config.js.map +1 -0
  437. package/dist/types/design-audit.d.ts +92 -0
  438. package/dist/types/design-audit.d.ts.map +1 -0
  439. package/dist/types/design-audit.js +5 -0
  440. package/dist/types/design-audit.js.map +1 -0
  441. package/dist/types/design-tokens.d.ts +138 -0
  442. package/dist/types/design-tokens.d.ts.map +1 -0
  443. package/dist/types/design-tokens.js +5 -0
  444. package/dist/types/design-tokens.js.map +1 -0
  445. package/dist/types/page.d.ts +16 -0
  446. package/dist/types/page.d.ts.map +1 -0
  447. package/dist/types/page.js +2 -0
  448. package/dist/types/page.js.map +1 -0
  449. package/dist/types/plan.d.ts +50 -0
  450. package/dist/types/plan.d.ts.map +1 -0
  451. package/dist/types/plan.js +2 -0
  452. package/dist/types/plan.js.map +1 -0
  453. package/dist/types/preview.d.ts +9 -0
  454. package/dist/types/preview.d.ts.map +1 -0
  455. package/dist/types/preview.js +5 -0
  456. package/dist/types/preview.js.map +1 -0
  457. package/dist/types/result.d.ts +62 -0
  458. package/dist/types/result.d.ts.map +1 -0
  459. package/dist/types/result.js +2 -0
  460. package/dist/types/result.js.map +1 -0
  461. package/dist/types/scenario.d.ts +20 -0
  462. package/dist/types/scenario.d.ts.map +1 -0
  463. package/dist/types/scenario.js +5 -0
  464. package/dist/types/scenario.js.map +1 -0
  465. package/dist/types/test-runner.d.ts +145 -0
  466. package/dist/types/test-runner.d.ts.map +1 -0
  467. package/dist/types/test-runner.js +10 -0
  468. package/dist/types/test-runner.js.map +1 -0
  469. package/dist/types/trajectory.d.ts +38 -0
  470. package/dist/types/trajectory.d.ts.map +1 -0
  471. package/dist/types/trajectory.js +2 -0
  472. package/dist/types/trajectory.js.map +1 -0
  473. package/dist/types/turn.d.ts +50 -0
  474. package/dist/types/turn.d.ts.map +1 -0
  475. package/dist/types/turn.js +2 -0
  476. package/dist/types/turn.js.map +1 -0
  477. package/dist/types.d.ts +16 -1017
  478. package/dist/types.d.ts.map +1 -1
  479. package/dist/types.js +15 -8
  480. package/dist/types.js.map +1 -1
  481. package/dist/wallet/rpc-interception.d.ts +15 -0
  482. package/dist/wallet/rpc-interception.d.ts.map +1 -0
  483. package/dist/wallet/rpc-interception.js +95 -0
  484. package/dist/wallet/rpc-interception.js.map +1 -0
  485. package/package.json +6 -2
@@ -4,411 +4,28 @@
4
4
  *
5
5
  * Uses Vercel AI SDK for multi-provider support (OpenAI, Anthropic, Google, Codex CLI, Claude Code).
6
6
  */
7
- import { generateText, streamText } from 'ai';
8
- import { AriaSnapshotHelper } from '../drivers/snapshot.js';
9
- import { resolveProviderApiKey, resolveProviderModelName, isClaudeCodeRoutedModel, shouldSendTemperature, ZAI_OPENAI_BASE_URL, ZAI_ANTHROPIC_BASE_URL, } from '../provider-defaults.js';
10
- import { buildFirstPartyBoundaryNote } from '../domain-policy.js';
11
- import { generateWithSandboxBackend } from '../providers/sandbox-backend.js';
12
- /** Core system prompt: preamble, actions, format, and rules 1-14 (always sent) */
13
- const CORE_RULES = `You are a senior staff engineer operating a browser via Playwright automation.
14
-
15
- You can SEE the page (via screenshot) and READ the page structure (via accessibility tree with @ref IDs).
16
- Use BOTH inputs together the screenshot shows layout/design/visual state, the a11y tree shows interactive elements with refs.
17
-
18
- ACTIONS:
19
- - {"action": "click", "selector": "@REF"}
20
- - {"action": "type", "selector": "@REF", "text": "text to type"}
21
- - {"action": "press", "selector": "@REF", "key": "Enter"} (or Tab, Escape, ArrowDown, etc.)
22
- - {"action": "hover", "selector": "@REF"}
23
- - {"action": "select", "selector": "@REF", "value": "option-value"}
24
- - {"action": "scroll", "direction": "up" | "down", "amount": 500} — add "selector": "@REF" to scroll a specific container
25
- - {"action": "navigate", "url": "https://..."}
26
- - {"action": "wait", "ms": 1000}
27
- - {"action": "evaluate", "criteria": "Is the layout professional? Are colors consistent?"}
28
- - {"action": "runScript", "script": "document.querySelector('.count').textContent"} — run JS in page context and get the result. Use for reading content not in the a11y tree (canvas, computed styles, hidden state).
29
- - {"action": "extractWithIndex", "query": "p, span, dd, code", "contains": "downloads"} — return a NUMBERED list of every visible element matching \`query\`, with each element's tag, full textContent, key attributes, and a stable selector. PREFER THIS OVER runScript when you need to find data inside the page but don't know the exact selector. The wide query (e.g. \`'p, span, strong'\`) finds candidates and the response shows the actual text so you can pick by content match. Optional \`contains\` filters matches to those whose text contains a substring (case-insensitive). After this action, your next turn can complete with the picked element's text or click its selector.
30
- - {"action": "verifyPreview"} — after the app builds, inspect the preview iframe. Returns URL, title, a11y tree, and errors. Use this AFTER you see a preview iframe on the page.
31
- - {"action": "fill", "fields": {"@t1": "Jordan", "@t2": "Rivera"}, "selects": {"@s1": "WA"}, "checks": ["@c1", "@c2"]} — BATCH fill multiple form fields, dropdowns, and checkboxes in ONE turn. Use this whenever you can see 2+ form fields you need to fill — it's dramatically faster than per-field type/click. fields/selects/checks are all optional but at least one must be non-empty.
32
- - {"action": "clickSequence", "refs": ["@r1", "@r2", "@r3"]} — click a known sequence of refs in order. Use for multi-step UI navigation chains where the click order is obvious from the page structure.
33
- - {"action": "complete", "result": "description of what was accomplished"}
34
- - {"action": "abort", "reason": "why you cannot continue"}
35
-
36
- SELECTOR FORMAT:
37
- - CRITICAL: Replace @REF with an actual ref from the ELEMENTS list below (e.g., @b3cee, @t1f2a)
38
- - NEVER invent or guess ref IDs — only use refs that appear as [ref=XXX] in the ELEMENTS list
39
- - Refs are deterministic — same element keeps the same ref across observations
40
- - Fallback: [data-testid="..."], [aria-label="..."], text="...", role=button[name="..."]
41
-
42
- RESPONSE FORMAT — respond with ONLY a JSON object:
43
- {
44
- "plan": ["step 1", "step 2", ...],
45
- "currentStep": 0,
46
- "action": { "action": "click", "selector": "@REF_FROM_ELEMENTS" },
47
- "nextActions": [{ "action": "type", "selector": "@REF_FROM_ELEMENTS", "text": "..." }],
48
- "reasoning": "Why I chose this action based on what I see",
49
- "expectedEffect": "What should change (e.g., 'URL should contain /chat/', 'modal should close')"
50
- }
51
-
52
- RULES:
53
- 1. Respond with ONLY valid JSON, no markdown or extra text
54
- 2. Use @ref selectors from the ELEMENTS list — they are stable across turns
55
- 3. Include plan, currentStep, reasoning, and expectedEffect in every response
56
- 4. Primary action must be in "action". Optional "nextActions" can contain up to 2 safe follow-ups (click/type/press/hover/select/scroll/wait) only when deterministic
57
- 5. When the goal is achieved, use "complete" with a detailed result description
58
- 6. If stuck after multiple attempts, use "abort" — don't loop forever
59
- 7. LOOK at the screenshot — it shows visual state the a11y tree may miss
60
- 8. If an action failed, try a DIFFERENT approach (different selector, different strategy)
61
- 9. For complex goals, break them into clear plan steps and track progress
62
- 10. Use "evaluate" when you need to assess visual quality, layout, or design
63
- 11. After the app builds and a preview is visible, use "verifyPreview" to check for errors before completing
64
- 12. BLOCKER-FIRST POLICY: if a modal, limit, quota, permission, or error dialog blocks progress, resolve THAT first before continuing the main goal
65
- 13. For quota/limit blockers, use an unblock ladder: open manage path -> clean up old test resources if needed -> retry the original action
66
- 14. If the same action triggers the same blocker twice, switch strategy immediately (different button/path), do not repeat blind retries
67
- 15. BATCH FILL FOR MULTI-FIELD FORMS: when you can see 2+ form fields that need to be filled, ALWAYS use a single "fill" action with all the fields at once instead of multiple type/click turns. A 5-field form takes 1 turn with fill, not 10 turns with type. Same for dropdowns (use selects map) and checkboxes (use checks array). The page rarely cares which order fields are filled — batch them.
68
- - CRITICAL: every key in fields/selects/checks MUST be an @ref taken VERBATIM from the ELEMENTS list (e.g., "@t1f2a"), or a simple [data-testid="..."] selector copied from the DATA-TESTID SELECTORS section. NEVER invent CSS combinators like "[data-testid=\"x\"] input" or "@refXXX child". If a target doesn't appear in the snapshot, use single-step type/click for it instead.
69
- - Date inputs (type="date") and spinbuttons (year/month/day) typically need single-step "type" actions, NOT batch fill. They have non-text input behavior that confuses Playwright's fill(). Skip them in your batch and handle them with type after.
70
- - If a batch fill fails, do NOT retry the same batch on the next turn. The error message will tell you which target failed — switch to single-step type/click for that target and shrink your next batch to just the targets that work.`;
71
- /** Search-related rules (15-17): injected when page has search elements or /search URL */
72
- const SEARCH_RULES = `
73
- 15. SEARCH FORMS: Always interact with the form (type in search box, then click Search or press Enter). Do NOT navigate to a URL with search query parameters — many sites require form submission to trigger filtering. If a search yields no results, try the page's own search box rather than the site-wide search
74
- 16. CONTENT DISCOVERY: If the ELEMENTS list doesn't show the link/content you need (e.g., the page has many links but the a11y tree is truncated), use runScript to find it: document.querySelectorAll('a[href]') filtered by keyword. Navigate to the discovered URL directly instead of clicking blindly through menus
75
- 17. EXTERNAL SEARCH REDIRECTS: If a site's search form redirects to an external search engine (e.g., search.usa.gov for .gov sites), the results still link back to the original site. Click a relevant search result link — it will take you to the target domain. Do NOT abandon search results to navigate the target site manually`;
76
- /** Data extraction rules (18, 21-23, 25): injected when goal involves extracting data */
77
- const DATA_EXTRACTION_RULES = `
78
- 18. DATA EXTRACTION: When the goal asks for specific data (prices, ratings, counts, names) from a list or search results page, prefer extractWithIndex with a wide query (e.g. \`'p, span, dd, code, strong'\`) over runScript when you don't already see the value in the snapshot. extractWithIndex returns the actual textContent of every match so you can pick the right one by content. Use runScript only when you need a transformation the LLM can't do from text alone.
79
- 21. EFFICIENT COMPLETION: When you have enough data to answer the goal, complete immediately. Do not navigate to additional pages for "confirmation" if the data was already extracted or is visible in the current a11y tree. Include all extracted data in the completion result
80
- 22. EXTRACT BEFORE NAVIGATING: On search results, directory listings, or any page showing multiple items, ALWAYS extract ALL needed data BEFORE clicking into individual items. Use extractWithIndex with a wide query for unknown structure, or runScript with document.querySelectorAll('.result-card') if the structure is well-known. Many sites use anti-bot protection on detail pages but leave listing pages accessible. If you can answer the goal from list-level data, do so without navigating deeper.
81
- 23. FILTER vs SEARCH: When a goal asks to filter results (e.g., "under $50", "4+ stars"), look for filter controls (sliders, dropdowns, checkboxes in a sidebar or toolbar) rather than typing filter values into the search box. Search boxes are for keyword queries, not numeric filters. After applying a filter: (1) wait 2-3 seconds for results to update, (2) verify the filter took effect by checking the updated results, (3) extract the filtered data. Do NOT keep searching for more filter controls after one is applied — extract and complete
82
- 25. EXTRACTWITHINDEX RECOVERY: If a previous runScript returned null/empty/{x:null} on an extraction task, the selector was wrong. DO NOT retry the same runScript or guess a similar selector — the LLM cannot guess CSS class names that aren't visible in the snapshot. Switch to extractWithIndex with a WIDE query: \`'p, span, dd, code, strong, em'\` plus a \`contains\` filter naming the expected text fragment (e.g. contains: "downloads" for npm download counts, contains: "callbackFn" for MDN method signatures). The response shows you the actual text per element so you can pick by content match. Pick-by-content beats pick-by-selector on pages where the planner couldn't see the data at plan time.`;
83
- /** Heavy page rules (19-20, 24): injected when snapshot is large or turn count is high */
84
- const HEAVY_PAGE_RULES = `
85
- 19. FORM FIELD TARGETING: Before typing, verify you are targeting the correct input field using its @ref from the ELEMENTS list. If multiple inputs are visible (e.g., search box + price filter), ensure you select the right one by checking its label or placeholder text in the a11y tree. Never assume focus — always specify the exact @ref
86
- 20. SECTION NAVIGATION: When you need to find a specific section (e.g., rugby, sports, travel) and the nav links aren't in the truncated a11y tree, use runScript to discover navigation: JSON.stringify(Array.from(document.querySelectorAll('nav a, header a, [role="navigation"] a, .nav a')).slice(0, 30).map(a => ({text: a.textContent.trim(), href: a.href}))). Then navigate directly to the matching section URL
87
- 24. HEAVY PAGE RECOVERY: If a page takes very long to load or seems stuck, do NOT wait — use runScript to check document.readyState and extract whatever content is already in the DOM. Partial data is better than a timeout. If the page is completely blank, try navigating to a simpler version (mobile site, search page) instead of waiting`;
88
- /** URL-first navigation rules for complex forms and search pages.
89
- * Teaches the agent to construct search/results URLs from goal text
90
- * instead of fighting form UIs. Works on any site with URL parameters. */
91
- const URL_FIRST_RULES = `
92
- URL-FIRST NAVIGATION: When a search form is complex (date pickers, multi-step dropdowns, dynamic widgets), try constructing a results URL directly instead of interacting with the form.
93
-
94
- STRATEGY:
95
- 1. Look at the current URL structure. Most search sites encode parameters: ?q=query, ?checkin=date, ?dest=city, etc.
96
- 2. Construct a URL with the goal's parameters filled in. Use the site's own URL pattern.
97
- 3. Navigate directly to that URL — skip the form entirely.
98
- 4. If the URL doesn't work (wrong page, error), fall back to form interaction.
99
-
100
- HOW TO DISCOVER URL PATTERNS:
101
- - If you're on a search results page, the URL already shows the pattern. Modify the parameters for your goal.
102
- - Most sites accept ?q= or ?search= for keyword queries.
103
- - Travel sites typically use: checkin/checkout dates, destination/origin, adults count.
104
- - Use runScript to read window.location.href if the URL isn't visible in the snapshot.
105
- - ENCODED PARAMETERS: If a URL contains encoded parameters (base64, protobuf), you may be able to replicate them from a previous successful URL, but do NOT invent new encodings. If a navigate with encoded parameters lands on the wrong page, do not retry with a different encoding — it will waste turns.
106
-
107
- IMPORTANT EXCEPTIONS — some sites BLOCK direct URL navigation:
108
- - If a direct URL navigate lands on the homepage or an error page instead of results, the site blocks URL manipulation. STOP trying URLs and use the site's form/search UI instead.
109
- - After ONE failed URL attempt, switch to form interaction immediately. Do NOT retry different URL patterns — you will waste turns.
110
-
111
- FORM RESET DETECTION: Some sites (especially SPAs) silently reset form fields after filling. After batch-filling a form:
112
- 1. Use runScript to verify values stuck: document.querySelector('[aria-label="From"]')?.value or similar.
113
- 2. If fields reset to defaults (wrong city, blank dates), do NOT re-fill with the same approach — it will reset again.
114
- 3. Instead, switch to keyboard-only interaction: click the field, type the value character by character, wait for autocomplete dropdown, press Enter to confirm. Then Tab to the next field.
115
-
116
- DATE PICKER STRATEGY: Calendar widgets often ignore programmatic fill/type. When a date field opens a calendar popup that blocks further input:
117
- 1. Try typing the date directly into the field in the site's format (e.g., "Jan 25, 2026" or "01/25/2026"). Press Escape first if the calendar covers the input.
118
- 2. If typing doesn't stick, use runScript to find clickable date elements: document.querySelectorAll('[data-iso],[aria-label*="January"],[aria-label*="25"]') and click the matching element.
119
- 3. NEVER spend more than 4 turns on a single date field.
120
-
121
- WHY: Complex forms with date pickers, calendar widgets, and multi-step dropdowns consume many turns and often time out. A single "navigate" action replaces 5-10 form interaction turns. But only use this on sites that support it.`;
122
- /** Reasoning framework and examples (always appended after rules) */
123
- const REASONING_SUFFIX = `
124
-
125
- REASONING FRAMEWORK — before choosing an action:
126
- 1. What is the current state vs. the goal state? What is missing?
127
- 2. What is the smallest action that makes progress toward the goal?
128
- 3. If multiple elements could match, prefer the one closest to the user-visible label
129
- 4. If an action just failed, identify WHY it failed before trying again
130
- 5. Ask: "Is there a blocker preventing progress right now?" If yes, clear blocker first, then continue goal plan
131
-
132
- EXAMPLE 1 — Multi-step form fill (use actual refs from ELEMENTS, not these placeholders):
133
- {"plan":["Navigate to signup page","Fill email field","Fill password field","Click submit","Verify success"],"currentStep":1,"action":{"action":"type","selector":"@REF","text":"user@example.com"},"reasoning":"I see the signup form with email input [ref=...] and password input [ref=...]. Starting with email since it is the first required field.","expectedEffect":"Email field should show 'user@example.com'"}
134
-
135
- EXAMPLE 2 — Recovery after failure:
136
- {"plan":["Click the send button","Wait for response"],"currentStep":0,"action":{"action":"scroll","direction":"down","amount":300},"reasoning":"My last click failed because the element was not visible in the viewport. I can see from the screenshot that the send button is below the fold. Scrolling down to bring it into view before retrying.","expectedEffect":"The send button should become visible in the viewport"}
137
-
138
- EXAMPLE 3 — Batch fill a multi-field form (one turn instead of ten):
139
- {"plan":["Fill all visible Personal Info fields","Click Next","Fill Contact step","Submit"],"currentStep":0,"action":{"action":"fill","fields":{"@firstname":"Jordan","@lastname":"Rivera","@dob":"1990-04-15"},"selects":{"@gender":"other"}},"reasoning":"Step 1 of the form has 3 text fields and 1 select all visible at once. Filling them in a single batch action saves 7 turns vs typing each individually.","expectedEffect":"All four Step 1 fields populated with the supplied values"}`;
140
- /** Full static prompt (all rules) — used as default when config.systemPrompt is not set */
141
- const SYSTEM_PROMPT = CORE_RULES + SEARCH_RULES + DATA_EXTRACTION_RULES + HEAVY_PAGE_RULES + REASONING_SUFFIX;
142
- // Vision-first system prompt for pure coordinate actions.
143
- const VISION_FIRST_PROMPT = `You are a browser automation agent. You operate by looking at screenshots and clicking on elements using pixel coordinates.
144
-
145
- The screenshot shows the current page state at 1024×768 resolution. You identify elements visually and specify where to click using (x, y) coordinates in this coordinate space.
146
-
147
- ACTIONS:
148
- - {"action": "clickAt", "x": 512, "y": 384} — click at pixel coordinates (x, y) in 1024×768 space
149
- - {"action": "typeAt", "x": 300, "y": 200, "text": "search query"} — click at coordinates then type text
150
- - {"action": "scroll", "direction": "up" | "down", "amount": 500}
151
- - {"action": "navigate", "url": "https://..."}
152
- - {"action": "wait", "ms": 1000}
153
- - {"action": "complete", "result": "description of what was accomplished"}
154
- - {"action": "abort", "reason": "why you cannot continue"}
155
- - {"action": "runScript", "script": "document.querySelector('.count').textContent"} — run JS in page context for data the screenshot can't show
156
- - {"action": "extractWithIndex", "query": "p, span, dd", "contains": "keyword"} — find text in the DOM by content match
157
- - {"action": "fanOut", "subGoals": [...]} — spawn up to 8 parallel sub-agents in separate tabs (same session/cookies). See FAN-OUT section below for full rules + worked example.
158
-
159
- FAN-OUT — PARALLEL INVESTIGATION:
160
- fanOut is the way to investigate N independent candidates in PARALLEL instead of serially. When the current page shows a list of candidates or you have a queue of independent sub-tasks, emit ONE fanOut action instead of processing them one at a time. The system spawns N sub-agents in fresh tabs of the same session; they run concurrently; their results are merged and returned to you as structured JSON in the NEXT turn's feedback.
161
-
162
- STRONGLY PREFER THE SHORTHAND FORM when every branch shares a URL + instruction template. It emits tiny JSON that can't malform:
163
-
164
- {"action":"fanOut","baseUrl":"https://site.example/","goalTemplate":"Investigate {item} — report outcome","items":["X","Y","Z"]}
165
-
166
- The string {item} in goalTemplate is replaced with each array entry. Labels default to the item. This is the RIGHT shape for N>=3 branches.
167
-
168
- USE fanOut WHEN:
169
- - A search returned multiple results and each needs investigation (click in / extract / return verdict per row).
170
- - A batch job has ≥3 independent sub-tasks (screen multiple customers, check multiple products, compare multiple pages).
171
- - The task is obviously parallelizable and sequential execution would 3x+ the wall-clock time.
172
-
173
- DO NOT use fanOut for:
174
- - A truly sequential task where step 2 depends on step 1's outcome.
175
- - A single-target investigation (use regular click/type).
176
- - Cases where fewer than 3 branches would run (overhead not worth it).
177
-
178
- SHAPE:
179
- {
180
- "action": "fanOut",
181
- "subGoals": [
182
- {
183
- "url": "https://target.site/",
184
- "goal": "Full natural-language instruction for this branch. Must be self-contained because the sub-agent starts from the url with no other context. End with 'Complete with a structured verdict of {schema}.'",
185
- "label": "SHORT-LABEL-FOR-OVERLAY",
186
- "maxTurns": 8
187
- },
188
- ... (1-8 entries)
189
- ]
190
- }
191
-
192
- WORKED EXAMPLE (OFAC batch screening, after C-001 is done via regular actions):
193
- {
194
- "action": "fanOut",
195
- "subGoals": [
196
- {"url":"https://sanctionssearch.ofac.treas.gov/","goal":"Click the Reset button, then type 'SMITH' into Last Name and 'JOHN' into First Name. Leave score at 95. Click Search. If 0 matches, complete with result 'CLEARED'. If 1+ exact matches with score 95-100, click the top row, read SDN program + list + DOB, complete with result 'POSITIVE MATCH: <program>/<list>'. Else complete with 'NEEDS REVIEW'.","label":"C-002 SMITH"},
197
- {"url":"https://sanctionssearch.ofac.treas.gov/","goal":"Click Reset, type 'MADURO' into Last Name and 'NICOLAS' into First Name. Leave score at 95. Click Search. [same disposition rules]","label":"C-003 MADURO"},
198
- ... (up to 8 per fanOut)
199
- ]
200
- }
201
-
202
- AFTER fanOut RETURNS, you receive FAN-OUT RESULTS as feedback — a JSON payload with {label, success, verdict, turnsUsed} per branch. Update your progress ledger with each branch's verdict, then either fire another fanOut for the next batch OR complete() if all sub-tasks are done.
203
-
204
- EFFICIENCY: one fanOut with 8 branches running in parallel finishes in ~the time of ONE sequential customer (not 8×). For a 10-customer batch, prefer: C-001 sequential (learn the form) → fanOut C-002..C-009 (8 parallel) → C-010 sequential or 2nd fanOut. Target: ~12 parent turns instead of ~50.
205
-
206
- COORDINATE SYSTEM:
207
- - (0, 0) is the top-left corner of the viewport
208
- - (1024, 768) is the bottom-right corner
209
- - Click the CENTER of the target element, not its edge
210
- - For text inputs, click the middle of the input field
211
- - For buttons, click the center of the button text or icon
212
-
213
- RESPONSE FORMAT — respond with ONLY a JSON object:
214
- {
215
- "plan": ["step 1", "step 2", ...],
216
- "currentStep": 0,
217
- "action": { "action": "clickAt", "x": 512, "y": 384 },
218
- "reasoning": "I see [element description] at approximately (x, y). Clicking it to [purpose].",
219
- "expectedEffect": "What should change after this action"
220
- }
221
-
222
- RULES:
223
- 1. Respond with ONLY valid JSON, no markdown or extra text
224
- 2. LOOK at the screenshot carefully — it is your primary information source
225
- 3. Include plan, currentStep, reasoning, and expectedEffect in every response
226
- 4. When the goal is achieved, use "complete" with a detailed result description
227
- 5. If stuck after multiple attempts, use "abort" — don't loop forever
228
- 6. If an action failed, try a DIFFERENT approach (different location, different strategy)
229
- 7. For search: click the search box, type your query, then press Enter
230
- 8. For navigation: click visible links or use the "navigate" action for direct URLs
231
- 9. BLOCKER-FIRST: if a modal, cookie banner, or error dialog blocks progress, dismiss it first
232
- 10. Use runScript or extractWithIndex when you need to extract data that isn't clearly visible in the screenshot
233
- 11. BATCH: when filling forms, you can type in one field, then immediately use clickAt on the next field. Plan multiple actions per turn when they are sequential and obvious.
234
- 12. VERIFY BEFORE COMPLETING: Before using "complete", re-read the GOAL and check: does your result ACTUALLY answer what was asked? If the goal asks for "5 beauty salons with ratings > 4.8" and you only found 3, do NOT complete — keep searching. If the goal asks for a specific date/price/name and your result doesn't contain it, do NOT complete. Premature completion with wrong data is worse than using another turn.
235
- 13. DATE PICKER BYPASS: If you encounter a complex date picker widget (calendar popup, date spinner) that is hard to interact with, DO NOT spend multiple turns clicking through calendar months. Instead, use "navigate" to construct a URL with the date parameters encoded. For Google Flights: navigate to google.com/travel/flights with search params. For Booking: navigate to booking.com/searchresults with checkin/checkout params. URL-based date setting is faster and more reliable than fighting date picker UIs.
236
-
237
- REASONING FRAMEWORK:
238
- 1. What do I see in the screenshot? Describe the visual layout.
239
- 2. Where is the element I need to interact with? Estimate its (x, y) coordinates.
240
- 3. What is the smallest action that makes progress toward the goal?
241
- 4. If my last action failed, WHY did it fail? Try a different location or strategy.`;
242
- // Unified vision+DOM prompt. The model sees BOTH the screenshot AND
243
- // the ARIA snapshot with @refs. It can use EITHER coordinate actions (clickAt/
244
- // typeAt for visual targets) OR ref actions (click/type/fill for DOM elements).
245
- // This lets it pick the best tool per interaction: vision for visual layout,
246
- // DOM for precise form interaction.
247
- const UNIFIED_VISION_DOM_PROMPT = `You are a browser automation agent with TWO input modalities: a screenshot showing the visual page state, and a structured ELEMENTS list with interactive element refs.
248
-
249
- Use BOTH together:
250
- - The screenshot shows layout, visual state, images, icons — what a human sees
251
- - The ELEMENTS list shows interactive elements with @ref IDs for precise targeting
252
-
253
- ACTIONS — pick the best tool for each interaction:
254
-
255
- LABEL ACTIONS (PREFERRED — use the [N] numbered labels visible on the screenshot):
256
- - {"action": "clickLabel", "label": 3} — click element labeled [3] in the screenshot
257
- - {"action": "typeLabel", "label": 5, "text": "query"} — click [5] then type text
258
- The screenshot has numbered red badges on interactive elements. Use these labels — they're MORE ACCURATE than coordinate guessing.
259
-
260
- REF ACTIONS (use for form fields, buttons, links with clear @refs from ELEMENTS):
261
- - {"action": "click", "selector": "@REF"}
262
- - {"action": "type", "selector": "@REF", "text": "text"}
263
- - {"action": "press", "selector": "@REF", "key": "Enter"}
264
- - {"action": "select", "selector": "@REF", "value": "option"}
265
- - {"action": "fill", "fields": {"@REF1": "val1", "@REF2": "val2"}} — batch fill multiple form fields
266
-
267
- COORDINATE ACTIONS (fallback when no label or ref is available):
268
- - {"action": "clickAt", "x": 512, "y": 384} — click at pixel (x, y) in 1024×768 space
269
- - {"action": "typeAt", "x": 300, "y": 200, "text": "query"} — click + type
270
-
271
- SHARED ACTIONS:
272
- - {"action": "scroll", "direction": "up" | "down", "amount": 500}
273
- - {"action": "navigate", "url": "https://..."}
274
- - {"action": "wait", "ms": 1000}
275
- - {"action": "runScript", "script": "..."} — run JS in page context
276
- - {"action": "extractWithIndex", "query": "p, span", "contains": "keyword"}
277
- - {"action": "complete", "result": "description"}
278
- - {"action": "abort", "reason": "why"}
279
-
280
- WHEN TO USE WHICH (priority order):
281
- 1. Element has a [N] label in the screenshot → use clickLabel/typeLabel (most accurate)
282
- 2. Element has an @ref in ELEMENTS → use click/type/fill (fast and precise)
283
- 3. Element is visible but has no label or ref → use clickAt/typeAt (coordinate fallback)
284
- - Date pickers, dropdown items rendered dynamically → use clickLabel if labeled, else clickAt
285
-
286
- RESPONSE FORMAT — respond with ONLY a JSON object:
287
- {
288
- "plan": ["step 1", "step 2", ...],
289
- "currentStep": 0,
290
- "action": { "action": "click", "selector": "@REF" },
291
- "nextActions": [{ "action": "type", "selector": "@REF2", "text": "query" }],
292
- "reasoning": "Why I chose this action",
293
- "expectedEffect": "What should change"
294
- }
295
-
296
- NOTE: "nextActions" is optional — include up to 3 safe follow-up actions (click, type, press, clickAt, typeAt, scroll) that are DETERMINISTIC given the current state. For example: click a search box THEN type a query. This saves turns.
297
-
298
- RULES:
299
- 1. Respond with ONLY valid JSON
300
- 2. Use @ref selectors from ELEMENTS when available — they are stable and precise
301
- 3. Fall back to clickAt coordinates when the target has no ref or is visual-only
302
- 4. LOOK at the screenshot — it shows visual state the ELEMENTS list may miss
303
- 5. When the goal is achieved, use "complete" with a detailed result
304
- 6. BLOCKER-FIRST: dismiss modals, cookie banners, login walls before continuing
305
- 7. BATCH FILL: when 2+ form fields are visible with refs, use a single "fill" action
306
- 8. If stuck after multiple attempts, use "abort"
307
- 9. VERIFY BEFORE COMPLETING: Before using "complete", re-read the GOAL and check: does your result ACTUALLY answer what was asked? If the goal asks for specific data (prices, names, ratings, counts) and your result doesn't contain ALL of them, keep going. Premature completion with partial data is worse than using another turn.
308
- 10. FORM RESET DETECTION: After batch-filling a form, verify values stuck via runScript. If fields reset to defaults, switch to keyboard-only: click field → type value → wait for autocomplete → press Enter → Tab to next. Do NOT re-fill with the same approach if it reset once.
309
- 11. DATE PICKER STRATEGY: When a calendar popup opens over a date field:
310
- a. Press Escape to dismiss, then type the date directly (e.g., "Jan 25, 2026").
311
- b. If typing doesn't stick, use runScript to find clickable dates: document.querySelectorAll('[data-iso],[aria-label*="25"]').
312
- c. NEVER spend more than 4 turns on a single date field.`;
313
- /** Pattern for detecting data-extraction keywords in goal text */
314
- const DATA_EXTRACTION_PATTERN = /\b(extract|list|find|data|price|pric|names?|rating|cost|count)\b/i;
315
- /** Pattern for detecting search-related roles in snapshot text */
316
- const SEARCH_SNAPSHOT_PATTERN = /^\s*-\s+(?:searchbox|combobox)\s/m;
317
- const FIRST_TURN_COMPACT_PROMPT = `You are a browser agent choosing the fastest safe next action.
318
-
319
- Return ONLY valid JSON with:
320
- {
321
- "plan": ["step 1", "step 2"],
322
- "currentStep": 0,
323
- "action": { "action": "click", "selector": "@REF" },
324
- "nextActions": [],
325
- "reasoning": "brief reason",
326
- "expectedEffect": "what should change"
327
- }
328
-
329
- Rules:
330
- 1. Use exact @ref selectors from ELEMENTS. Never invent refs.
331
- 2. Prefer the smallest high-signal action.
332
- 3. On landing pages, prefer site search, primary navigation, or an obvious goal-matching link.
333
- 4. If a blocker is visible, resolve it first.
334
- 5. Do not over-explore on the first turn.
335
- 6. Respond with JSON only.`;
336
- const LINK_SCOUT_PROMPT = `Pick the best link from CANDIDATES to advance the GOAL. Respond with ONLY JSON:
337
- {"selector":"@ref","reasoning":"brief reason","confidence":0.82}
338
- Rules: use exact candidate ref, pick one, confidence 0-1, prefer first-party and text-matching links.`;
339
- const DESIGN_AUDIT_PROMPT = `You are a senior product designer and UX engineer auditing a web application.
340
-
341
- Analyze the screenshot and accessibility tree for design quality, UX issues, and visual bugs.
342
-
343
- CHECK FOR:
344
- - Layout: misaligned elements, broken grids, inconsistent spacing, overflow/clipping
345
- - Typography: inconsistent font sizes, poor hierarchy, text overflow, unreadable text
346
- - Colors: poor contrast (WCAG AA requires 4.5:1 for text), inconsistent color palette
347
- - Spacing: inconsistent padding/margins, crowded elements, excessive whitespace
348
- - Alignment: elements not vertically/horizontally aligned with their siblings
349
- - Accessibility: missing labels, unclear focus indicators, keyboard traps
350
- - UX: confusing navigation, hidden actions, missing feedback states, dead-end flows
351
- - Visual bugs: z-index issues, overlapping elements, broken images, rendering artifacts
352
-
353
- You will also receive CHECKPOINTS — specific conditions to verify. Include a finding for each checkpoint that fails.
354
-
355
- For each issue found, categorize it and rate its severity:
356
- - critical: blocks user flow or causes data loss
357
- - major: significantly impacts usability or looks unprofessional
358
- - minor: cosmetic issue, polish improvement
359
-
360
- RESPOND WITH ONLY a JSON object:
361
- {
362
- "score": 7,
363
- "findings": [
364
- {
365
- "category": "layout",
366
- "severity": "major",
367
- "description": "Navigation sidebar overlaps main content on narrower viewports",
368
- "location": "Left sidebar, main content area",
369
- "suggestion": "Add responsive breakpoint or collapse sidebar below 1024px"
370
- }
371
- ]
372
- }
373
-
374
- Categories: visual-bug, layout, contrast, alignment, spacing, typography, accessibility, ux
375
- Score: 1-3 = poor, 4-5 = needs work, 6-7 = acceptable, 8-9 = good, 10 = excellent`;
376
- const EVALUATE_PROMPT = `You are evaluating the quality of a web page or application output.
377
-
378
- Look at the screenshot and assess:
379
- 1. Visual design quality (layout, spacing, colors, typography)
380
- 2. Functionality completeness (does it match the intended goal?)
381
- 3. Professional polish (would this be acceptable in production?)
382
- 4. Accessibility (readable text, good contrast, clear labels)
383
- 5. Responsiveness indicators (proper scaling, no overflow)
384
-
385
- Respond with ONLY a JSON object:
386
- {
387
- "score": 8,
388
- "assessment": "Brief overall assessment",
389
- "strengths": ["strength 1", "strength 2"],
390
- "issues": ["issue 1", "issue 2"],
391
- "suggestions": ["improvement 1", "improvement 2"]
392
- }
393
-
394
- Score: 1-3 = poor, 4-5 = needs work, 6-7 = acceptable, 8-9 = good, 10 = excellent`;
395
- const JSON_TEXT_OUTPUT = {
396
- name: 'json-text',
397
- responseFormat: Promise.resolve({ type: 'json' }),
398
- async parseCompleteOutput({ text }) {
399
- return text;
400
- },
401
- async parsePartialOutput({ text }) {
402
- return { partial: text };
403
- },
404
- createElementStreamTransform() {
405
- return undefined;
406
- },
407
- };
7
+ import { generateText } from 'ai';
8
+ import { SYSTEM_PROMPT } from './prompts.js';
9
+ import { decideImpl, decideVisionImpl, parseDecision } from './decide.js';
10
+ import { planImpl } from './plan.js';
11
+ import { getModelImpl, getLanguageModelImpl, generateImpl, generationOptionsImpl, completeImpl, completeVisionImpl, } from './model-client.js';
12
+ import { composeSystemPromptPartsImpl, buildSystemForDecideImpl, } from './system-prompt.js';
13
+ import { evaluateImpl } from './tasks/evaluate.js';
14
+ import { recommendLinkCandidateImpl } from './tasks/link-scout.js';
15
+ import { verifyGoalCompletionImpl } from './tasks/goal-verification.js';
16
+ import { auditDesignImpl } from './tasks/design-audit.js';
17
+ import { extractKnowledgeImpl } from './tasks/knowledge.js';
18
+ export { budgetSnapshot } from './snapshot-budget.js';
408
19
  export class Brain {
20
+ // Public fields below satisfy the extracted host interfaces (BrainModelHost,
21
+ // BrainSystemPromptHost). The free functions in model-client.ts and
22
+ // system-prompt.ts read them through those interfaces; `implements` makes
23
+ // tsc prove the surface is complete.
409
24
  modelCache = new Map();
410
25
  provider;
411
26
  modelName;
27
+ // Public fields below additionally satisfy the extracted task host interfaces
28
+ // (BrainLinkScoutHost, BrainGoalVerificationHost).
412
29
  adaptiveModelRouting;
413
30
  navModelName;
414
31
  navProvider;
@@ -420,6 +37,10 @@ export class Brain {
420
37
  visionEnabled;
421
38
  visionStrategy;
422
39
  observationMode;
40
+ // Force streaming input on claude-code so completeVision images are not dropped
41
+ // (the Claude Code SDK omits image parts without it). Off by default; set only
42
+ // by the vision-judge wiring. See AgentConfig.claudeCodeStreamingInput.
43
+ claudeCodeStreamingInput;
423
44
  llmTimeoutMs;
424
45
  compactFirstTurn;
425
46
  lastDecisionUrl;
@@ -459,6 +80,7 @@ export class Brain {
459
80
  this.visionEnabled = config.vision !== false;
460
81
  this.visionStrategy = config.visionStrategy ?? (this.visionEnabled ? 'always' : 'never');
461
82
  this.observationMode = config.observationMode ?? 'dom';
83
+ this.claudeCodeStreamingInput = config.claudeCodeStreamingInput === true;
462
84
  this.compactFirstTurn = config.compactFirstTurn === true;
463
85
  this.sandboxBackendType = config.sandboxBackendType;
464
86
  this.sandboxBackendProfile = config.sandboxBackendProfile;
@@ -480,449 +102,21 @@ export class Brain {
480
102
  this.adaptiveModelRouting = true; // enable routing when executor model is set
481
103
  }
482
104
  }
483
- resolveModelName(provider, requestedModel) {
484
- return resolveProviderModelName(provider, requestedModel, {
485
- sandboxBackendType: provider === 'sandbox-backend' ? this.sandboxBackendType : undefined,
486
- });
487
- }
488
- shouldSendTemperature(modelName = this.modelName) {
489
- // Reasoning models reject an explicit temperature — see the shared
490
- // capability check (GPT-5, o-series, Opus 4.8+, Kimi K2.6+, DeepSeek
491
- // reasoner).
492
- return shouldSendTemperature(modelName);
493
- }
494
105
  generationOptions(maxOutputTokens, selection) {
495
- const providerName = selection?.provider || this.provider;
496
- const modelName = this.resolveModelName(providerName, selection?.model || this.modelName);
497
- // CLI-spawning providers (codex-cli, claude-code, and zai-coding-plan
498
- // when routed through claude-code) don't accept maxOutputTokens through
499
- // the AI SDK shape — the subprocess controls its own output.
500
- const isCliSpawning = providerName === 'codex-cli' ||
501
- providerName === 'claude-code' ||
502
- providerName === 'sandbox-backend' ||
503
- (providerName === 'zai-coding-plan' && isClaudeCodeRoutedModel(modelName));
504
- const omitsLegacyMaxTokens = providerName === 'cli-bridge' || /(^|\/)gpt-5(?:[.-]|$)/i.test(modelName);
505
- return {
506
- ...(this.shouldSendTemperature(modelName) ? { temperature: 0 } : {}),
507
- ...(isCliSpawning || omitsLegacyMaxTokens ? {} : { maxOutputTokens }),
508
- // forceReasoning routes the AI SDK to OpenAI's Responses API
509
- // (`/v1/responses`). Most third-party OpenAI-compatible proxies
510
- // (router.tangle.tools, LiteLLM, Together, vLLM, etc.) only implement
511
- // /v1/chat/completions — Responses API requests come back 503 / 4xx
512
- // and the SDK throws "Invalid JSON response". Disable on proxied
513
- // openai routes; only OpenAI direct supports the Responses API today.
514
- ...(providerName === 'openai'
515
- && /(^|\/)gpt-5(?:[.-]|$)/i.test(modelName)
516
- && !this.isProxiedOpenAI(providerName)
517
- ? {
518
- providerOptions: {
519
- openai: {
520
- forceReasoning: true,
521
- maxCompletionTokens: maxOutputTokens,
522
- },
523
- },
524
- }
525
- : {}),
526
- };
527
- }
528
- /**
529
- * True iff we're hitting an OpenAI-compatible *proxy* (router.tangle.tools,
530
- * LiteLLM, etc.) rather than OpenAI direct. Used to downshift to the lowest
531
- * common denominator API surface — chat-completions, no Responses API,
532
- * non-streaming — that every OpenAI-compatible proxy is guaranteed to serve.
533
- *
534
- * The single source of truth for "we're talking to a proxy, be conservative
535
- * about API features"; both `createForceNonStreamingFetch()` and the
536
- * `forceReasoning` gate route through this predicate.
537
- */
538
- isProxiedOpenAI(providerName) {
539
- return providerName === 'openai' && Boolean(this.baseUrl);
106
+ return generationOptionsImpl(this, maxOutputTokens, selection);
540
107
  }
541
108
  /** Get a LLM model instance, optionally with provider/model override (e.g. for CAPTCHA fallback) */
542
109
  async getLanguageModel(selection) {
543
- return this.getModel(selection);
110
+ return getLanguageModelImpl(this, selection);
544
111
  }
545
112
  /** Lazily create the LLM model instance based on provider config */
546
- async getModel(selection) {
547
- const providerName = selection?.provider || this.provider;
548
- const modelName = this.resolveModelName(providerName, selection?.model || this.modelName);
549
- const apiKey = resolveProviderApiKey(providerName, this.explicitApiKey);
550
- const cacheKey = `${providerName}:${modelName}`;
551
- const cached = this.modelCache.get(cacheKey);
552
- if (cached)
553
- return cached;
554
- let model;
555
- switch (providerName) {
556
- case 'anthropic': {
557
- const { createAnthropic } = await import('@ai-sdk/anthropic');
558
- const provider = createAnthropic({
559
- apiKey,
560
- ...(this.baseUrl ? { baseURL: this.baseUrl } : {}),
561
- });
562
- model = provider(modelName);
563
- break;
564
- }
565
- case 'google': {
566
- const { createGoogleGenerativeAI } = await import('@ai-sdk/google');
567
- const provider = createGoogleGenerativeAI({
568
- apiKey,
569
- ...(this.baseUrl ? { baseURL: this.baseUrl } : {}),
570
- });
571
- model = provider(modelName);
572
- break;
573
- }
574
- case 'cli-bridge': {
575
- const { createOpenAI } = await import('@ai-sdk/openai');
576
- const rawUrl = this.baseUrl || process.env.CLI_BRIDGE_URL;
577
- if (!rawUrl) {
578
- throw new Error('cli-bridge provider requires CLI_BRIDGE_URL or --base-url');
579
- }
580
- const baseURL = rawUrl.endsWith('/v1') ? rawUrl : `${rawUrl.replace(/\/+$/, '')}/v1`;
581
- const provider = createOpenAI({
582
- apiKey: apiKey || '',
583
- baseURL,
584
- });
585
- model = provider.chat(modelName);
586
- break;
587
- }
588
- case 'codex-cli': {
589
- const { codexExec } = await import('ai-sdk-provider-codex-cli');
590
- const env = {};
591
- if (apiKey)
592
- env.OPENAI_API_KEY = apiKey;
593
- model = codexExec(modelName, {
594
- allowNpx: process.env.CODEX_ALLOW_NPX !== '0',
595
- skipGitRepoCheck: true,
596
- ...(process.env.CODEX_CLI_PATH ? { codexPath: process.env.CODEX_CLI_PATH } : {}),
597
- ...(Object.keys(env).length > 0 ? { env } : {}),
598
- });
599
- break;
600
- }
601
- case 'claude-code': {
602
- const { createClaudeCode } = await import('ai-sdk-provider-claude-code');
603
- const env = {};
604
- if (apiKey)
605
- env.ANTHROPIC_API_KEY = apiKey;
606
- const provider = createClaudeCode({
607
- defaultSettings: {
608
- ...(process.env.CLAUDE_CODE_CLI_PATH ? { pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_CLI_PATH } : {}),
609
- permissionMode: 'default',
610
- allowDangerouslySkipPermissions: false,
611
- ...(this.debug ? { verbose: true } : {}),
612
- ...(this.debug
613
- ? {
614
- stderr: (chunk) => {
615
- const line = chunk.trim();
616
- if (line)
617
- console.error(`[claude-code] ${line}`);
618
- },
619
- }
620
- : {}),
621
- ...(Object.keys(env).length > 0 ? { env } : {}),
622
- },
623
- });
624
- model = provider(modelName);
625
- break;
626
- }
627
- case 'zai-coding-plan': {
628
- // Z.ai coding plan: GLM models (glm-4.6, glm-4.5-air) at a fraction
629
- // of Anthropic prices. Two backends:
630
- // 1. Default — OpenAI-compatible endpoint, native GLM models
631
- // 2. claude-code routed — spawn the Claude Code CLI subprocess
632
- // with ANTHROPIC_BASE_URL/AUTH_TOKEN env vars pointed at Z.ai's
633
- // Anthropic-compatible endpoint. The user gets the Claude Code
634
- // agent loop with Z.ai pricing.
635
- if (!apiKey) {
636
- throw new Error('zai-coding-plan: API key required (set ZAI_API_KEY env var or pass --api-key)');
637
- }
638
- if (isClaudeCodeRoutedModel(modelName)) {
639
- const { createClaudeCode } = await import('ai-sdk-provider-claude-code');
640
- const env = {
641
- // Override Claude Code's API base + auth at the env-var level.
642
- // The CLI subprocess inherits these and routes all Anthropic
643
- // API calls to Z.ai's Anthropic-compatible endpoint.
644
- ANTHROPIC_BASE_URL: ZAI_ANTHROPIC_BASE_URL,
645
- ANTHROPIC_AUTH_TOKEN: apiKey,
646
- // Some Claude Code versions still read ANTHROPIC_API_KEY — set
647
- // both so we don't depend on which env var name wins.
648
- ANTHROPIC_API_KEY: apiKey,
649
- };
650
- const provider = createClaudeCode({
651
- defaultSettings: {
652
- ...(process.env.CLAUDE_CODE_CLI_PATH ? { pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_CLI_PATH } : {}),
653
- permissionMode: 'default',
654
- allowDangerouslySkipPermissions: false,
655
- ...(this.debug ? { verbose: true } : {}),
656
- ...(this.debug
657
- ? {
658
- stderr: (chunk) => {
659
- const line = chunk.trim();
660
- if (line)
661
- console.error(`[zai-claude-code] ${line}`);
662
- },
663
- }
664
- : {}),
665
- env,
666
- },
667
- });
668
- // The Claude Code SDK takes a model alias (sonnet/haiku/opus); when
669
- // the user passes a glm-* name we just hand it through — Claude
670
- // Code passes it as the model param to the upstream API, which is
671
- // Z.ai's Anthropic-compat endpoint that maps it to the right glm.
672
- // For 'claude-code'/'cc' we default to 'sonnet' which Z.ai aliases
673
- // to its strongest glm model under the coding plan.
674
- const ccModel = modelName === 'claude-code' || modelName === 'cc' ? 'sonnet' : modelName;
675
- model = provider(ccModel);
676
- break;
677
- }
678
- // Default path — direct OpenAI-compatible API to Z.ai. Z.ai's
679
- // paas/v4 endpoint only implements `/chat/completions` (no
680
- // `/responses`), so we MUST use `provider.chat(model)` rather than
681
- // the bare `provider(model)` call which now defaults to the new
682
- // OpenAI Responses API in @ai-sdk/openai v3+.
683
- const { createOpenAI } = await import('@ai-sdk/openai');
684
- const provider = createOpenAI({
685
- apiKey,
686
- baseURL: ZAI_OPENAI_BASE_URL,
687
- });
688
- model = provider.chat(modelName);
689
- break;
690
- }
691
- default: {
692
- // 'openai' or any OpenAI-compatible API (LiteLLM, Together, etc.).
693
- // Same reason as the zai-coding-plan branch above: most third-party
694
- // OpenAI-compatible proxies (LiteLLM, vLLM, Together) implement
695
- // `/chat/completions` but not the new `/responses` API. Use the
696
- // chat factory explicitly so we hit a path the upstream actually
697
- // serves.
698
- const { createOpenAI } = await import('@ai-sdk/openai');
699
- const usingProxy = this.isProxiedOpenAI(providerName);
700
- const provider = createOpenAI({
701
- apiKey: apiKey || '',
702
- ...(this.baseUrl ? { baseURL: this.baseUrl } : {}),
703
- // Proxy downshift — see isProxiedOpenAI(). Routes like
704
- // router.tangle.tools default chat-completions to SSE when the
705
- // client omits `stream`, and the AI SDK's generateText errors with
706
- // "Invalid JSON response" on SSE. Force stream: false on every
707
- // chat-completions body. Paired with the forceReasoning gate in
708
- // generationOptions() so all "talk to a proxy" downshifts share
709
- // one predicate.
710
- ...(usingProxy ? { fetch: createForceNonStreamingFetch() } : {}),
711
- });
712
- model = provider.chat(modelName);
713
- break;
714
- }
715
- }
716
- this.modelCache.set(cacheKey, model);
717
- return model;
718
- }
719
- async generate(system, messages, selection, maxOutputTokens = 800) {
720
- const providerName = selection?.provider || this.provider;
721
- const modelName = this.resolveModelName(providerName, selection?.model || this.modelName);
722
- // Sandbox backend doesn't accept structured system messages — flatten.
723
- const systemForSandbox = typeof system === 'string'
724
- ? system
725
- : system.map(m => m.content).join('\n\n');
726
- if (providerName === 'sandbox-backend') {
727
- const result = await generateWithSandboxBackend({
728
- system: systemForSandbox,
729
- messages,
730
- model: modelName,
731
- timeoutMs: this.llmTimeoutMs,
732
- debug: this.debug,
733
- backendType: this.sandboxBackendType,
734
- backendProfile: this.sandboxBackendProfile,
735
- backendModelProvider: this.sandboxBackendProvider,
736
- });
737
- return { text: result.text };
738
- }
739
- const model = await this.getModel({
740
- provider: providerName,
741
- model: modelName,
742
- });
743
- // Anthropic supports structured system messages with cache_control. For
744
- // every other provider, flatten back to a string — they ignore the array
745
- // form's per-message provider options anyway, and some (claude-code,
746
- // codex-cli) wrap their own subprocess CLI which only takes plain text.
747
- const systemForRequest = providerName === 'anthropic' || typeof system === 'string'
748
- ? system
749
- : systemForSandbox;
750
- const generationSettings = {
751
- model,
752
- system: systemForRequest,
753
- messages,
754
- ...(providerName === 'cli-bridge' ? { output: JSON_TEXT_OUTPUT } : {}),
755
- ...this.generationOptions(maxOutputTokens, { provider: providerName, model: modelName }),
756
- abortSignal: AbortSignal.timeout(this.llmTimeoutMs),
757
- };
758
- const result = providerName === 'cli-bridge'
759
- ? await (async () => {
760
- const streamed = streamText(generationSettings);
761
- const [text, usage, providerMetadata] = await Promise.all([
762
- streamed.text,
763
- streamed.totalUsage,
764
- streamed.providerMetadata,
765
- ]);
766
- return { text, usage, providerMetadata };
767
- })()
768
- : await generateText(generationSettings);
769
- // Extract prompt-cache stats from the AI SDK's PROVIDER-AGNOSTIC fields:
770
- // result.usage.inputTokenDetails.{cacheReadTokens, cacheWriteTokens}
771
- //
772
- // These flow uniformly from every provider that supports prompt caching:
773
- // - OpenAI (gpt-5.4, gpt-4.1-mini): AUTOMATIC server-side caching for
774
- // >1024 token prefixes, no markers needed. Returns cached_tokens.
775
- // - Anthropic (claude-*): EXPLICIT cache_control markers required (we
776
- // set them in buildSystemForDecide). Returns cache_read_input_tokens.
777
- // - ZAI / GLM (zai-coding-plan via OpenAI-compatible endpoint): AUTOMATIC
778
- // server-side caching, returns cached_tokens in prompt_tokens_details.
779
- // - Google Gemini: explicit `cachedContent` ID-based caching (different
780
- // paradigm — not currently exercised).
781
- //
782
- // The unified inputTokenDetails fields mean we get cache observability
783
- // for free across providers without per-provider extraction code.
784
- const inputDetails = result.usage?.inputTokenDetails;
785
- let cacheReadInputTokens = typeof inputDetails?.cacheReadTokens === 'number'
786
- ? inputDetails.cacheReadTokens
787
- : undefined;
788
- let cacheCreationInputTokens = typeof inputDetails?.cacheWriteTokens === 'number'
789
- ? inputDetails.cacheWriteTokens
790
- : undefined;
791
- // Fallback: some providers expose cache stats only via providerMetadata.
792
- // Anthropic uses cacheReadInputTokens / cacheCreationInputTokens directly,
793
- // OpenAI sometimes lands cached_tokens under providerMetadata.openai.
794
- if (cacheReadInputTokens === undefined || cacheCreationInputTokens === undefined) {
795
- const meta = result.providerMetadata;
796
- const anthropicMeta = meta?.anthropic;
797
- const openaiMeta = meta?.openai;
798
- if (cacheReadInputTokens === undefined) {
799
- const fromAnthropic = typeof anthropicMeta?.cacheReadInputTokens === 'number'
800
- ? anthropicMeta.cacheReadInputTokens
801
- : undefined;
802
- const fromOpenAI = typeof openaiMeta?.cachedPromptTokens === 'number'
803
- ? openaiMeta.cachedPromptTokens
804
- : undefined;
805
- cacheReadInputTokens = fromAnthropic ?? fromOpenAI;
806
- }
807
- if (cacheCreationInputTokens === undefined) {
808
- const fromAnthropic = typeof anthropicMeta?.cacheCreationInputTokens === 'number'
809
- ? anthropicMeta.cacheCreationInputTokens
810
- : undefined;
811
- cacheCreationInputTokens = fromAnthropic;
812
- }
813
- }
814
- return {
815
- text: result.text,
816
- tokensUsed: result.usage?.totalTokens,
817
- inputTokens: result.usage?.inputTokens ?? undefined,
818
- outputTokens: result.usage?.outputTokens ?? undefined,
819
- cacheReadInputTokens,
820
- cacheCreationInputTokens,
821
- };
822
- }
823
- /**
824
- * Classify whether this turn should use the nav (cheap) model for decide().
825
- *
826
- * Empirically tested: routing early navigation turns to gpt-4.1-mini causes
827
- * worse decisions that cascade into longer runs (more turns = more total cost).
828
- * gpt-5.4 without routing is cheaper overall because it navigates more efficiently.
829
- *
830
- * Current strategy: nav model is ONLY used for verification (see verifyGoalCompletion),
831
- * not for decide(). The flag is kept for future experiments with better routing signals.
832
- */
833
- shouldUseNavigationModel(state, extraContext, turnInfo) {
834
- if (!this.adaptiveModelRouting || !this.navModelName)
835
- return false;
836
- // Use the navigation model for DOM-only same-page turns. Keep the primary
837
- // model for first turns, new pages, and error recovery.
838
- const isFirstTurn = !turnInfo || turnInfo.current <= 1;
839
- const samePageAsPrevious = this.lastDecisionUrl === state.url;
840
- const hasError = extraContext?.includes('REJECTED') || extraContext?.includes('ERROR');
841
- if (isFirstTurn || !samePageAsPrevious || hasError)
842
- return false;
843
- return true;
113
+ getModel(selection) {
114
+ return getModelImpl(this, selection);
844
115
  }
845
- /**
846
- * Build the system prompt dynamically, injecting conditional rule groups
847
- * based on goal text, page snapshot content, and turn number.
848
- * Saves ~800 tokens per turn on simple navigation tasks.
849
- */
850
- buildSystemPrompt(goal, state, turn) {
851
- return this.composeSystemPromptParts(goal, state, turn).join('');
852
- }
853
- /**
854
- * Same as buildSystemPrompt but returns the parts so the caller can decide
855
- * how to send them. For Anthropic, decide() ships them as a SystemModelMessage[]
856
- * with cache_control on the stable CORE_RULES prefix; other providers join.
857
- *
858
- * The first slot is ALWAYS CORE_RULES (or the user's custom override) so the
859
- * cache breakpoint placement is deterministic. Extension-supplied rules are
860
- * appended AFTER REASONING_SUFFIX so the cached prefix stays byte-stable
861
- * across turns.
862
- */
863
- composeSystemPromptParts(goal, state, turn) {
864
- if (this.systemPrompt !== SYSTEM_PROMPT)
865
- return [this.systemPrompt];
866
- const parts = [CORE_RULES];
867
- const snapshotSample = state.snapshot.length > 4000 ? state.snapshot.slice(0, 4000) : state.snapshot;
868
- if (SEARCH_SNAPSHOT_PATTERN.test(snapshotSample) || /\/search\b/i.test(state.url)) {
869
- parts.push(SEARCH_RULES);
870
- if (this.extensionRules?.search) {
871
- parts.push(`\n\nUSER RULES (search):\n${this.extensionRules.search}`);
872
- }
873
- }
874
- if (DATA_EXTRACTION_PATTERN.test(goal)) {
875
- parts.push(DATA_EXTRACTION_RULES);
876
- if (this.extensionRules?.dataExtraction) {
877
- parts.push(`\n\nUSER RULES (data extraction):\n${this.extensionRules.dataExtraction}`);
878
- }
879
- }
880
- if (state.snapshot.length > 10_000 || turn > 10) {
881
- parts.push(HEAVY_PAGE_RULES);
882
- if (this.extensionRules?.heavy) {
883
- parts.push(`\n\nUSER RULES (heavy page):\n${this.extensionRules.heavy}`);
884
- }
885
- }
886
- parts.push(REASONING_SUFFIX);
887
- // Global user rules + matching per-domain rules. Both are appended AFTER
888
- // REASONING_SUFFIX so they don't pollute the byte-stable cached prefix.
889
- if (this.extensionRules?.global) {
890
- parts.push(`\n\nUSER RULES (global):\n${this.extensionRules.global}`);
891
- }
892
- if (this.extensionDomainRules) {
893
- const domainRules = this.matchDomainRules(state.url);
894
- if (domainRules) {
895
- parts.push(`\n\nUSER RULES (domain match):\n${domainRules}`);
896
- }
897
- }
898
- // Macros live AFTER the cached prefix so registering new macros
899
- // doesn't bust the Anthropic cache.
900
- if (this.macroPromptBlock) {
901
- parts.push(`\n\n${this.macroPromptBlock}`);
902
- }
903
- return parts;
904
- }
905
- /**
906
- * Find the per-domain extra rules whose domain key matches the URL host.
907
- * Multiple matches are concatenated in registration order.
908
- */
909
- matchDomainRules(url) {
910
- if (!this.extensionDomainRules)
911
- return undefined;
912
- let host;
913
- try {
914
- host = new URL(url).hostname;
915
- }
916
- catch {
917
- return undefined;
918
- }
919
- const matches = [];
920
- for (const [domain, rules] of Object.entries(this.extensionDomainRules)) {
921
- if (host.includes(domain) && rules.extraRules) {
922
- matches.push(rules.extraRules);
923
- }
924
- }
925
- return matches.length > 0 ? matches.join('\n\n') : undefined;
116
+ // Public so the extracted task impls (src/brain/tasks/*) can reach it through
117
+ // their host interfaces. The class still owns the single transport funnel.
118
+ generate(system, messages, selection, maxOutputTokens = 800) {
119
+ return generateImpl(this, system, messages, selection, maxOutputTokens);
926
120
  }
927
121
  /**
928
122
  * Inject extension-supplied rules. Called by the runner after loading
@@ -937,6 +131,9 @@ export class Brain {
937
131
  setMacroPromptBlock(block) {
938
132
  this.macroPromptBlock = block ?? '';
939
133
  }
134
+ composeSystemPromptParts(goal, state, turn) {
135
+ return composeSystemPromptPartsImpl(this, goal, state, turn);
136
+ }
940
137
  /**
941
138
  * Build the system prompt for `decide()` in the form best suited to the
942
139
  * active provider:
@@ -949,25 +146,7 @@ export class Brain {
949
146
  * is opt-in via the default prompt path only.
950
147
  */
951
148
  buildSystemForDecide(goal, state, turn, providerName) {
952
- const parts = this.composeSystemPromptParts(goal, state, turn);
953
- if (providerName !== 'anthropic' || this.systemPrompt !== SYSTEM_PROMPT || parts.length === 0) {
954
- return parts.join('');
955
- }
956
- // Anthropic path: first slot is CORE_RULES (cached), remaining parts ship
957
- // as a separate uncached system message so the prefix stays byte-stable
958
- // across turns and the cache hits.
959
- const corePart = {
960
- role: 'system',
961
- content: parts[0],
962
- providerOptions: {
963
- anthropic: {
964
- cacheControl: { type: 'ephemeral' },
965
- },
966
- },
967
- };
968
- if (parts.length === 1)
969
- return [corePart];
970
- return [corePart, { role: 'system', content: parts.slice(1).join('') }];
149
+ return buildSystemForDecideImpl(this, goal, state, turn, providerName);
971
150
  }
972
151
  /** Reset conversation history (call between scenarios) */
973
152
  reset() {
@@ -1021,6 +200,8 @@ export class Brain {
1021
200
  * Build the user message content parts — text + optional screenshot.
1022
201
  * Multimodal when vision is enabled and screenshot is available.
1023
202
  */
203
+ // Public so the extracted task impls (src/brain/tasks/*) can reach it through
204
+ // their host interfaces.
1024
205
  buildUserContent(text, screenshot, forceVision = false) {
1025
206
  const shouldUseVision = !!screenshot && (this.visionStrategy === 'always'
1026
207
  || (this.visionStrategy === 'auto' && forceVision));
@@ -1036,327 +217,8 @@ export class Brain {
1036
217
  },
1037
218
  ];
1038
219
  }
1039
- /**
1040
- * Compact conversation history: strip ELEMENTS blocks and screenshots
1041
- * from older observations, keeping the last 2 user messages intact.
1042
- *
1043
- * For older turns, replaces the full ELEMENTS block with a one-line
1044
- * summary showing element count and the selectors the agent actually
1045
- * used, extracted from the paired assistant response.
1046
- */
1047
- compactHistory() {
1048
- if (this.history.length === 0)
1049
- return [];
1050
- // Find indices of the last 2 user messages to keep intact
1051
- const userIndices = [];
1052
- for (let i = this.history.length - 1; i >= 0; i--) {
1053
- if (this.history[i].role === 'user') {
1054
- userIndices.push(i);
1055
- if (userIndices.length === 2)
1056
- break;
1057
- }
1058
- }
1059
- const keepIntactFrom = userIndices.length > 0
1060
- ? userIndices[userIndices.length - 1]
1061
- : this.history.length;
1062
- // Three-tier compression:
1063
- // Zone 1 (intact): last 2 turns — full content
1064
- // Zone 2 (standard): turns 3-5 back — ELEMENTS stripped from user msgs
1065
- // Zone 3 (deep compact): turns 6+ back — both user and assistant ultra-compacted
1066
- // Keep enough older history for long multi-step travel workflows.
1067
- const deepCompactBefore = Math.max(0, this.history.length - 10);
1068
- return this.history.map((msg, idx) => {
1069
- // Zone 1: keep recent turns intact
1070
- if (idx >= keepIntactFrom)
1071
- return msg;
1072
- // Zone 3: ultra-compact for very old messages (user + assistant)
1073
- if (idx < deepCompactBefore) {
1074
- if (msg.role === 'assistant') {
1075
- const raw = typeof msg.content === 'string' ? msg.content : '';
1076
- return { ...msg, content: this.deepCompactAssistant(raw) };
1077
- }
1078
- if (msg.role === 'user') {
1079
- return { ...msg, content: this.deepCompactUser(msg) };
1080
- }
1081
- return msg;
1082
- }
1083
- // Zone 2: standard compact — strip ELEMENTS from user messages only
1084
- if (msg.role !== 'user')
1085
- return msg;
1086
- const assistantMsg = idx + 1 < this.history.length ? this.history[idx + 1] : undefined;
1087
- const selectors = assistantMsg?.role === 'assistant'
1088
- ? this.extractSelectorsFromResponse(typeof assistantMsg.content === 'string' ? assistantMsg.content : '')
1089
- : [];
1090
- // Handle multimodal content (array of parts)
1091
- if (Array.isArray(msg.content)) {
1092
- const compacted = msg.content
1093
- .filter((part) => part.type === 'text')
1094
- .map((part) => ({
1095
- ...part,
1096
- text: this.summarizeElements(part.text, selectors),
1097
- }));
1098
- return { ...msg, content: compacted };
1099
- }
1100
- // Handle string content
1101
- if (typeof msg.content === 'string') {
1102
- return { ...msg, content: this.summarizeElements(msg.content, selectors) };
1103
- }
1104
- return msg;
1105
- });
1106
- }
1107
- deepCompactUser(msg) {
1108
- const text = typeof msg.content === 'string'
1109
- ? msg.content
1110
- : Array.isArray(msg.content)
1111
- ? msg.content
1112
- .filter((p) => p.type === 'text')
1113
- .map((p) => p.text)
1114
- .join('\n')
1115
- : '';
1116
- const urlMatch = text.match(/URL:\s*(\S+)/);
1117
- const titleMatch = text.match(/Title:\s*(.+?)(?:\n|$)/);
1118
- const url = urlMatch?.[1] ?? 'unknown';
1119
- const title = titleMatch?.[1]?.slice(0, 80) ?? '';
1120
- return `[Prior turn — URL: ${url}${title ? ` | ${title}` : ''}]`;
1121
- }
1122
- deepCompactAssistant(raw) {
1123
- try {
1124
- let text = raw.trim();
1125
- if (text.startsWith('```')) {
1126
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
1127
- }
1128
- const parsed = JSON.parse(text);
1129
- const action = parsed.action?.action ?? 'unknown';
1130
- const selector = parsed.action?.selector ?? '';
1131
- const parts = [action];
1132
- if (selector)
1133
- parts.push(selector);
1134
- if (parsed.action?.url)
1135
- parts.push(parsed.action.url.slice(0, 120));
1136
- return `[${parts.join(' → ')}]`;
1137
- }
1138
- catch {
1139
- return raw.slice(0, 100) + (raw.length > 100 ? '…' : '');
1140
- }
1141
- }
1142
- /**
1143
- * Extract @ref selectors from an assistant JSON response.
1144
- */
1145
- extractSelectorsFromResponse(raw) {
1146
- const selectors = [];
1147
- try {
1148
- let text = raw.trim();
1149
- if (text.startsWith('```')) {
1150
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
1151
- }
1152
- const parsed = JSON.parse(text);
1153
- if (parsed.action?.selector)
1154
- selectors.push(parsed.action.selector);
1155
- if (Array.isArray(parsed.nextActions)) {
1156
- for (const na of parsed.nextActions) {
1157
- if (na?.selector)
1158
- selectors.push(na.selector);
1159
- }
1160
- }
1161
- }
1162
- catch {
1163
- // Best effort
1164
- }
1165
- return selectors;
1166
- }
1167
- /**
1168
- * Replace the ELEMENTS block with a one-line action-only summary.
1169
- */
1170
- summarizeElements(text, selectors) {
1171
- return text.replace(/ELEMENTS[^:\n]*:\n[\s\S]*?(?=\n\n|What action should you take\?|$)/, (match) => {
1172
- const snapshotStart = match.indexOf('\n');
1173
- if (snapshotStart === -1)
1174
- return 'ELEMENTS:\n[previous snapshot]';
1175
- const snapshotText = match.slice(snapshotStart + 1);
1176
- const elementCount = (snapshotText.match(/\[ref=\w+\]/g) || []).length;
1177
- const selectorList = selectors.length > 0
1178
- ? selectors.join(', ')
1179
- : 'none';
1180
- return `ELEMENTS:\n[Page snapshot: ${elementCount} elements | agent used: ${selectorList}]`;
1181
- });
1182
- }
1183
220
  async decide(goal, state, extraContext, turnInfo, options) {
1184
- // Vision-first and hybrid modes delegate to the vision path.
1185
- if (this.observationMode === 'vision' || this.observationMode === 'hybrid') {
1186
- return this.decideVision(goal, state, extraContext, turnInfo);
1187
- }
1188
- const useCompactFirstTurn = this.compactFirstTurn && turnInfo?.current === 1;
1189
- const samePageAsPrevious = this.lastDecisionUrl === state.url;
1190
- const isFirstTurn = !turnInfo || turnInfo.current <= 1;
1191
- // Diff-only mode: on same-page turns with small diffs, send only changed
1192
- // elements instead of the full snapshot. Saves 40-80% of input tokens on
1193
- // form-fill / interaction-heavy pages where the page structure is stable.
1194
- const rawDiff = state.snapshotDiffRaw;
1195
- const diffChanges = rawDiff ? rawDiff.added.length + rawDiff.removed.length + rawDiff.changed.length : 0;
1196
- const diffTotal = rawDiff ? diffChanges + rawDiff.unchangedCount : 0;
1197
- const useDiffOnly = samePageAsPrevious
1198
- && !isFirstTurn
1199
- && rawDiff !== undefined
1200
- && diffChanges > 0
1201
- && diffTotal > 0
1202
- && diffChanges / diffTotal < 0.3;
1203
- // Tighter snapshot budget on same-page turns; new pages keep enough
1204
- // content for extraction from docs/spec pages.
1205
- const snapshotBudget = samePageAsPrevious ? 8_000 : 24_000;
1206
- let visibleSnapshot;
1207
- let elementsHeader;
1208
- if (useDiffOnly) {
1209
- // Build compact diff-only view: changed/added elements with refs
1210
- const lines = [];
1211
- if (rawDiff.added.length)
1212
- lines.push('ADDED:', ...rawDiff.added);
1213
- if (rawDiff.changed.length)
1214
- lines.push('CHANGED:', ...rawDiff.changed);
1215
- if (rawDiff.removed.length)
1216
- lines.push('REMOVED:', ...rawDiff.removed);
1217
- lines.push(`(${rawDiff.unchangedCount} elements unchanged — refs from previous turn still valid)`);
1218
- visibleSnapshot = lines.join('\n');
1219
- elementsHeader = 'ELEMENTS (diff-only, previous refs still valid)';
1220
- }
1221
- else {
1222
- visibleSnapshot = useCompactFirstTurn
1223
- ? compactFirstTurnSnapshot(state.snapshot)
1224
- : budgetSnapshot(state.snapshot, snapshotBudget);
1225
- elementsHeader = 'ELEMENTS';
1226
- }
1227
- this.lastDecisionUrl = state.url;
1228
- // Build user message with stable prefix (GOAL) for prompt caching,
1229
- // then dynamic per-turn content (turn budget, page state, elements).
1230
- let textContent = `GOAL: ${goal}
1231
-
1232
- CURRENT PAGE:
1233
- URL: ${state.url}
1234
- Title: ${state.title}
1235
-
1236
- ${elementsHeader}:
1237
- ${visibleSnapshot}`;
1238
- if (turnInfo) {
1239
- const remaining = turnInfo.max - turnInfo.current;
1240
- const budgetUsed = turnInfo.current / turnInfo.max;
1241
- textContent += `\n\nTURN: ${turnInfo.current}/${turnInfo.max} (${remaining} remaining)`;
1242
- if (remaining === 1) {
1243
- textContent += ` — FINAL TURN: return a terminal action only (complete or abort)`;
1244
- }
1245
- else if (remaining <= 3) {
1246
- textContent += ` — RUNNING LOW, avoid exploratory navigation; prioritize completing the goal or aborting with a clear blocker reason`;
1247
- }
1248
- else if (budgetUsed >= 0.5) {
1249
- textContent += ` — HALF BUDGET USED. If you have extracted useful data, try completing now. Do not navigate away from pages with relevant content without attempting completion first`;
1250
- }
1251
- }
1252
- // Append snapshot diff only when NOT using diff-only mode (avoid redundant info)
1253
- if (!useDiffOnly && state.snapshotDiff && state.snapshotDiff.length < state.snapshot.length * 0.3) {
1254
- textContent += `\n\nSNAPSHOT CHANGES (since last turn):\n${state.snapshotDiff}`;
1255
- }
1256
- if (extraContext) {
1257
- textContent += `\n\n${extraContext}`;
1258
- }
1259
- textContent += '\n\nWhat action should you take?';
1260
- const userContent = this.buildUserContent(textContent, state.screenshot, options?.forceVision === true);
1261
- const useNavModel = this.shouldUseNavigationModel(state, extraContext, turnInfo);
1262
- const effectiveProvider = useNavModel ? (this.navProvider || this.provider) : this.provider;
1263
- const effectiveModel = useNavModel ? (this.navModelName || this.modelName) : this.modelName;
1264
- if (this.debug) {
1265
- const turnNum = Math.floor(this.history.length / 2) + 1;
1266
- const usingVision = !!state.screenshot && (this.visionStrategy === 'always' || (this.visionStrategy === 'auto' && options?.forceVision === true));
1267
- console.log(`[Brain] Turn ${turnNum} | URL: ${state.url} | Vision: ${usingVision}`);
1268
- if (this.adaptiveModelRouting) {
1269
- const mode = useNavModel ? 'nav-model' : 'primary-model';
1270
- console.log(`[Brain] Model route: ${mode} (${effectiveProvider}/${effectiveModel}) turn=${turnInfo?.current}/${turnInfo?.max}`);
1271
- }
1272
- }
1273
- const messages = [
1274
- ...this.compactHistory(),
1275
- { role: 'user', content: userContent },
1276
- ];
1277
- const dynamicSystemPrompt = useCompactFirstTurn
1278
- ? FIRST_TURN_COMPACT_PROMPT
1279
- : this.buildSystemForDecide(goal, state, turnInfo?.current ?? 1, effectiveProvider);
1280
- const modelOpts = { provider: effectiveProvider, model: effectiveModel };
1281
- // Bump output budget near max turns so data-heavy completions don't truncate
1282
- const nearingEnd = turnInfo && turnInfo.current >= turnInfo.max - 3;
1283
- const maxTokens = useCompactFirstTurn ? 500 : nearingEnd ? 1200 : 600;
1284
- const result = await this.generate(dynamicSystemPrompt, messages, modelOpts, maxTokens);
1285
- let raw = result.text;
1286
- let tokensUsed = result.tokensUsed;
1287
- let inputTokens = result.inputTokens;
1288
- let outputTokens = result.outputTokens;
1289
- let cacheReadInputTokens = result.cacheReadInputTokens;
1290
- let cacheCreationInputTokens = result.cacheCreationInputTokens;
1291
- if (!raw) {
1292
- throw new Error('Brain.decide: LLM returned empty response — possible rate limit or model error');
1293
- }
1294
- if (this.debug) {
1295
- console.log('[Brain] Response:', raw.slice(0, 300));
1296
- }
1297
- let parsed = this.parse(raw);
1298
- // On malformed JSON, retry with minimal context (current page + correction
1299
- // hint) instead of burning a full turn. Costs ~7K tokens vs ~25K for a
1300
- // full-history retry on the next turn.
1301
- if (parsed.reasoning?.startsWith('Malformed LLM JSON response') && !useCompactFirstTurn) {
1302
- if (this.debug) {
1303
- console.log('[Brain] Malformed JSON — retrying with format hint');
1304
- }
1305
- const retryMessages = [
1306
- { role: 'user', content: userContent },
1307
- { role: 'assistant', content: raw },
1308
- { role: 'user', content: 'Your previous response was not valid JSON. Respond with ONLY a valid JSON object matching the required schema.' },
1309
- ];
1310
- try {
1311
- const retryResult = await this.generate(dynamicSystemPrompt, retryMessages, modelOpts, maxTokens);
1312
- if (retryResult.text) {
1313
- const retryParsed = this.parse(retryResult.text);
1314
- if (!retryParsed.reasoning?.startsWith('Malformed LLM JSON response')) {
1315
- raw = retryResult.text;
1316
- parsed = retryParsed;
1317
- }
1318
- else if (this.baseUrl) {
1319
- // Both the initial parse and the format-hint retry failed while a
1320
- // custom LLM_BASE_URL is set. Strong signal the gateway is
1321
- // returning a shape the scout can't consume (e.g. SSE streams,
1322
- // non-JSON wrappers). Surface the likely cause instead of
1323
- // burning silent retries turn after turn.
1324
- console.error(`[Brain] scout_json_parse_failed: LLM_BASE_URL=${this.baseUrl} returned a response the scout could not parse even after a format-hint retry. ` +
1325
- `Suggestion: switch to an Anthropic-native endpoint, or verify the gateway supports non-streaming chat/completions with { "response_format": { "type": "json_object" } }.`);
1326
- }
1327
- tokensUsed = (tokensUsed ?? 0) + (retryResult.tokensUsed ?? 0);
1328
- inputTokens = (inputTokens ?? 0) + (retryResult.inputTokens ?? 0);
1329
- outputTokens = (outputTokens ?? 0) + (retryResult.outputTokens ?? 0);
1330
- if (retryResult.cacheReadInputTokens !== undefined) {
1331
- cacheReadInputTokens = (cacheReadInputTokens ?? 0) + retryResult.cacheReadInputTokens;
1332
- }
1333
- if (retryResult.cacheCreationInputTokens !== undefined) {
1334
- cacheCreationInputTokens = (cacheCreationInputTokens ?? 0) + retryResult.cacheCreationInputTokens;
1335
- }
1336
- }
1337
- }
1338
- catch {
1339
- // Retry failed — fall through with original wait(1000) fallback
1340
- }
1341
- }
1342
- // Store in history
1343
- this.history.push({ role: 'user', content: userContent });
1344
- this.history.push({ role: 'assistant', content: raw });
1345
- // Trim old history
1346
- const maxMessages = this.maxHistoryTurns * 2;
1347
- if (this.history.length > maxMessages) {
1348
- this.history = this.history.slice(-maxMessages);
1349
- }
1350
- return {
1351
- ...parsed,
1352
- raw,
1353
- tokensUsed,
1354
- inputTokens,
1355
- outputTokens,
1356
- cacheReadInputTokens,
1357
- cacheCreationInputTokens,
1358
- modelUsed: effectiveModel,
1359
- };
221
+ return decideImpl(this, goal, state, extraContext, turnInfo, options);
1360
222
  }
1361
223
  /**
1362
224
  * Vision-first decision path. The screenshot is the primary
@@ -1365,120 +227,7 @@ ${visibleSnapshot}`;
1365
227
  * coordinate-based actions (clickAt, typeAt) in 1024×768 virtual space.
1366
228
  */
1367
229
  async decideVision(goal, state, extraContext, turnInfo) {
1368
- this.lastDecisionUrl = state.url;
1369
- // Adaptive observation: on same-page hybrid turns, send only changed
1370
- // elements when the diff is small.
1371
- const isHybrid = this.observationMode === 'hybrid';
1372
- const samePageAsPrevious = this.lastDecisionUrl === state.url;
1373
- const isFirstTurn = !turnInfo || turnInfo.current <= 1;
1374
- const rawDiff = state.snapshotDiffRaw;
1375
- const diffChanges = rawDiff ? rawDiff.added.length + rawDiff.removed.length + rawDiff.changed.length : 0;
1376
- const diffTotal = rawDiff ? diffChanges + rawDiff.unchangedCount : 0;
1377
- const useDiffOnly = isHybrid && samePageAsPrevious && !isFirstTurn
1378
- && rawDiff !== undefined && diffChanges > 0 && diffTotal > 0
1379
- && diffChanges / diffTotal < 0.4;
1380
- let textContent = `GOAL: ${goal}
1381
-
1382
- CURRENT PAGE:
1383
- URL: ${state.url}
1384
- Title: ${state.title}`;
1385
- if (isHybrid && state.snapshot) {
1386
- if (useDiffOnly) {
1387
- // Diff-focused: only what changed since last turn
1388
- const lines = [];
1389
- if (rawDiff.added.length)
1390
- lines.push('ADDED:', ...rawDiff.added);
1391
- if (rawDiff.changed.length)
1392
- lines.push('CHANGED:', ...rawDiff.changed);
1393
- if (rawDiff.removed.length)
1394
- lines.push('REMOVED:', ...rawDiff.removed);
1395
- lines.push(`(${rawDiff.unchangedCount} elements unchanged — refs from previous turn still valid)`);
1396
- textContent += `\n\nPAGE CHANGES (what changed after your last action — this is the important part):\n${lines.join('\n')}`;
1397
- }
1398
- else {
1399
- // Progressive budget reduction: more turns on same page = less snapshot
1400
- // needed (agent has already seen the full page, rely on screenshot + diff).
1401
- const sameTurnCount = samePageAsPrevious ? (turnInfo?.current || 0) : 0;
1402
- const snapshotBudget = samePageAsPrevious
1403
- ? (sameTurnCount >= 8 ? 2_500 : 4_000) // aggressive after 8+ same-page turns
1404
- : 6_000;
1405
- const snap = budgetSnapshot(state.snapshot, snapshotBudget);
1406
- textContent += `\n\nELEMENTS:\n${snap}`;
1407
- }
1408
- }
1409
- if (turnInfo) {
1410
- const remaining = turnInfo.max - turnInfo.current;
1411
- textContent += `\n\nTURN: ${turnInfo.current}/${turnInfo.max} (${remaining} remaining)`;
1412
- if (remaining === 1) {
1413
- textContent += ` — FINAL TURN: return a terminal action only (complete or abort)`;
1414
- }
1415
- else if (remaining <= 3) {
1416
- textContent += ` — RUNNING LOW, prioritize completing the goal or aborting`;
1417
- }
1418
- }
1419
- if (extraContext) {
1420
- textContent += `\n\n${extraContext}`;
1421
- }
1422
- textContent += '\n\nLook at the screenshot. What action should you take?';
1423
- // Screenshot is required for vision-first mode
1424
- if (!state.screenshot) {
1425
- return {
1426
- action: { action: 'wait', ms: 500 },
1427
- reasoning: 'No screenshot available for vision-first mode — waiting for page to render',
1428
- raw: '{"action":{"action":"wait","ms":500}}',
1429
- };
1430
- }
1431
- const userContent = [
1432
- { type: 'text', text: textContent },
1433
- { type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
1434
- ];
1435
- // Strip old screenshots from history; the current screenshot is the only
1436
- // image the model needs for this turn.
1437
- const compacted = this.compactHistory().map((msg) => {
1438
- if (msg.role !== 'user' || !Array.isArray(msg.content))
1439
- return msg;
1440
- const textOnly = msg.content
1441
- .filter((part) => part.type === 'text');
1442
- if (textOnly.length === msg.content.length)
1443
- return msg;
1444
- return { ...msg, content: textOnly };
1445
- });
1446
- const messages = [
1447
- ...compacted,
1448
- { role: 'user', content: userContent },
1449
- ];
1450
- // Vision turns stay on the main model because smaller model routes can
1451
- // count image tokens differently and exhaust the token budget.
1452
- const modelOpts = { provider: this.provider, model: this.modelName };
1453
- const nearingEnd = turnInfo && turnInfo.current >= turnInfo.max - 3;
1454
- const maxTokens = nearingEnd ? 1200 : 600;
1455
- // Hybrid mode uses the unified prompt with both action vocabularies.
1456
- const systemPrompt = isHybrid ? UNIFIED_VISION_DOM_PROMPT : VISION_FIRST_PROMPT;
1457
- const result = await this.generate(systemPrompt, messages, modelOpts, maxTokens);
1458
- const raw = result.text;
1459
- if (!raw) {
1460
- throw new Error('Brain.decideVision: LLM returned empty response');
1461
- }
1462
- if (this.debug) {
1463
- console.log('[Brain/Vision] Response:', raw.slice(0, 300));
1464
- }
1465
- const parsed = this.parse(raw);
1466
- this.history.push({ role: 'user', content: userContent });
1467
- this.history.push({ role: 'assistant', content: raw });
1468
- const maxMessages = this.maxHistoryTurns * 2;
1469
- if (this.history.length > maxMessages) {
1470
- this.history = this.history.slice(-maxMessages);
1471
- }
1472
- return {
1473
- ...parsed,
1474
- raw,
1475
- tokensUsed: result.tokensUsed,
1476
- inputTokens: result.inputTokens,
1477
- outputTokens: result.outputTokens,
1478
- cacheReadInputTokens: result.cacheReadInputTokens,
1479
- cacheCreationInputTokens: result.cacheCreationInputTokens,
1480
- modelUsed: this.modelName,
1481
- };
230
+ return decideVisionImpl(this, goal, state, extraContext, turnInfo);
1482
231
  }
1483
232
  /**
1484
233
  * Generate a structured plan for the entire task with one LLM call.
@@ -1494,224 +243,7 @@ Title: ${state.title}`;
1494
243
  * use per-action loop".
1495
244
  */
1496
245
  async plan(goal, state, options) {
1497
- const startedAt = Date.now();
1498
- const maxSteps = options?.maxSteps ?? 12;
1499
- const extraContext = options?.extraContext;
1500
- // Planner snapshots keep enough context for extraction tasks, especially
1501
- // docs/spec pages with data in `<dl>`, `<code>`, and `<pre>` blocks.
1502
- const snapshot = budgetSnapshot(state.snapshot, 24_000);
1503
- const planSystemPrompt = `You are a planning engine for a browser automation agent.
1504
-
1505
- Given a user goal and the current page state, your job is to generate a complete, ordered plan of actions that the agent will execute deterministically without re-entering you between steps. After each step, the runner verifies your stated post-condition. If verification fails the runner falls back to a per-action loop, so your job is to write a plan that requires the FEWEST steps and where every step's post-condition is reliably observable.
1506
-
1507
- KEY PRINCIPLES:
1508
-
1509
- 1. PREFER BATCH VERBS. The driver supports batch \`fill\` (fill N text fields, set N selects, check N checkboxes in ONE action) and \`clickSequence\` (N sequential clicks). Use these aggressively. A 19-field form should be 2-4 fill steps, not 19 type steps.
1510
-
1511
- CRITICAL EXCLUSION: DO NOT INCLUDE ANY SPINBUTTON OR DATE INPUT IN YOUR PLAN. AT ALL. Period. If you see \`spinbutton\` in the snapshot (year, month, day, hour, minute spinners) or any input that looks like a date/time picker, OMIT it from your plan completely. Playwright's locator.fill() and locator.click() both time out on these elements, and your plan will deviate and fall back to the per-action loop. The per-action fallback knows how to handle them. Just LEAVE THEM OUT. Your plan should silently skip those elements and continue with the rest of the task as if they don't exist. The runner WILL handle them after your plan completes — you do not need to plan a step for them.
1512
-
1513
- 2. ASSUME THE SNAPSHOT IS COMPLETE. The agent will execute your plan deterministically — you only get to see the page state ONCE (now). All @refs you emit must come from the ELEMENTS list below. Do not invent refs. If you don't see an element you'd need, do NOT plan a step for it — leave a gap and the runner will recover.
1514
-
1515
- 3. POST-CONDITIONS MUST BE OBSERVABLE. Each step's expectedEffect should describe a concrete change the runner can see in the next snapshot: a URL change, a new visible element, a status text update. Vague effects like "form is filled" are useless because verification can't check them. Use concrete strings: "Status text shows 'Account Created!'" or "URL contains /confirm/".
1516
-
1517
- 4. NAVIGATION CHANGES THE PAGE. Once you emit a \`navigate\`, \`click\` on a Next button, or any action that loads a new page, the @refs from the current snapshot are NO LONGER VALID. After such an action, your subsequent steps cannot rely on the same refs — they must use natural-language post-conditions until the runner falls back to per-action mode and observes the new page.
1518
-
1519
- 5. MAX ${maxSteps} STEPS. If the task genuinely requires more, plan the first ${maxSteps} and let the runner replan from the resulting state.
1520
-
1521
- 6. ONLY EMIT \`complete\` IF THE FINAL POST-CONDITION IS GENUINELY VERIFIABLE FROM THE PRIOR STEP'S expectedEffect. Do NOT fabricate success. If you cannot reliably know from the initial state alone whether the task succeeded (e.g. you can't predict whether a server submission will succeed, you don't know what the success message will say, or the form has multi-step navigation past your visibility), simply STOP planning at the last step you're confident about. The runner will fall through to the per-action loop after your plan exhausts and that loop will continue toward completion. It is BETTER to plan 5 confident steps and let the per-action loop finish than to plan 12 speculative steps with a fabricated complete at the end.
1522
-
1523
- 7. EXTRACTION TASKS: when the goal asks you to READ, EXTRACT, REPORT, or RETURN values from the page (numbers, text, lists, structured data), the LAST step of your plan MUST be \`runScript\`. Do NOT emit a \`complete\` step after the runScript with literal values in \`result\`, because at planning time you cannot know what runScript will return — any values you write would be fabricated. The runner has a deterministic substitution path: it will use the runScript output as the final result, OR fall through to per-action mode where the LLM can see the script output. Either way is fine. The wrong move is to put placeholder JSON like \`{"x":null,"y":null}\` or \`"<from prior step>"\` in the complete result; the runner detects and replaces those, but it's cleaner if you simply omit the complete step. RIGHT: \`[{action:runScript, script:"..."}]\`. WRONG: \`[{action:runScript,...}, {action:complete, result:"{x:null}"}]\`.
1524
-
1525
- ACTION VERBS (same as the per-action prompt):
1526
- - {"action": "click", "selector": "@REF"} — use when the element has a ref in ELEMENTS
1527
- - {"action": "type", "selector": "@REF", "text": "..."} — type into a ref element
1528
- - {"action": "press", "selector": "@REF", "key": "Enter"}
1529
- - {"action": "select", "selector": "@REF", "value": "..."}
1530
- - {"action": "clickAt", "x": 512, "y": 384} — click at pixel coordinates (use when you can see the element in the screenshot but it has no ref)
1531
- - {"action": "typeAt", "x": 300, "y": 200, "text": "..."} — click at coordinates then type
1532
- - {"action": "scroll", "direction": "up"|"down", "amount": 500}
1533
- - {"action": "navigate", "url": "..."}
1534
- - {"action": "wait", "ms": 1000}
1535
- - {"action": "fill", "fields": {"@a": "v1", "@b": "v2"}, "selects": {"@c": "v3"}, "checks": ["@d", "@e"]}
1536
- - {"action": "clickSequence", "refs": ["@a", "@b", "@c"]}
1537
- - {"action": "runScript", "script": "document.querySelector('.x').textContent"}
1538
- - {"action": "extractWithIndex", "query": "p, span, dd, code", "contains": "downloads"} — return a NUMBERED list of visible elements matching the query with their full textContent. Use this for extraction tasks where the data lives in obscurely-classed wrappers (npm download counts, MDN \`<dl>/<dt>/<dd>\` content, Python docs \`<code>\` blocks, W3C spec content) and the planner cannot guarantee a precise selector. The next step (in plan or per-action mode) reads the result and picks the right index. STRONGLY PREFER THIS OVER runScript on ANY extraction task where the snapshot doesn't already show the value verbatim.
1539
- - {"action": "complete", "result": "..."}
1540
- - {"action": "abort", "reason": "..."}
1541
-
1542
- ${URL_FIRST_RULES}
1543
-
1544
- RESPONSE FORMAT — respond with ONLY this JSON:
1545
- {
1546
- "reasoning": "1-2 sentence strategy summary",
1547
- "steps": [
1548
- {
1549
- "action": { "action": "fill", "fields": { "@t1": "Jordan", "@t2": "Rivera" } },
1550
- "expectedEffect": "First name and last name fields are populated",
1551
- "rationale": "Step 1 of the multi-step form: batch-fill all visible Personal Info text fields"
1552
- },
1553
- ...
1554
- ],
1555
- "finalResult": "Account creation form completed and confirmation visible"
1556
- }
1557
-
1558
- DO NOT include any prose outside the JSON. DO NOT use markdown code blocks. The runner parses your response with JSON.parse() and will fall through to the per-action loop on parse failure.`;
1559
- // Replan path: when the runner re-enters plan() after a previous plan
1560
- // deviated, it injects a deviation summary. The system prompt is byte-
1561
- // stable so prompt cache still hits — only the user message changes.
1562
- const userText = `GOAL: ${goal}
1563
-
1564
- CURRENT PAGE:
1565
- URL: ${state.url}
1566
- Title: ${state.title}
1567
-
1568
- ELEMENTS:
1569
- ${snapshot}
1570
- ${extraContext ? `\n${extraContext}\n` : ''}
1571
- What is the complete plan?`;
1572
- // In vision-capable modes, include the screenshot so the planner can use
1573
- // visual layout when the DOM does not capture form structure.
1574
- const isVisionPlanner = (this.observationMode === 'hybrid' || this.observationMode === 'vision') && !!state.screenshot;
1575
- const userContent = isVisionPlanner
1576
- ? [
1577
- { type: 'text', text: userText },
1578
- { type: 'image', image: state.screenshot, mediaType: 'image/jpeg' },
1579
- ]
1580
- : userText;
1581
- // Planner can use its own model override.
1582
- const planModelOpts = this.plannerModel
1583
- ? { provider: (this.plannerProvider || this.provider), model: this.plannerModel }
1584
- : { provider: this.provider, model: this.modelName };
1585
- const result = await this.generate(planSystemPrompt, [{ role: 'user', content: userContent }], planModelOpts,
1586
- // Plans need more output tokens than decide() — a 10-step plan with
1587
- // batch fills + rationale per step is comfortably over 1000 tokens.
1588
- 2_500).catch((err) => ({
1589
- text: '',
1590
- tokensUsed: undefined,
1591
- inputTokens: undefined,
1592
- outputTokens: undefined,
1593
- cacheReadInputTokens: undefined,
1594
- cacheCreationInputTokens: undefined,
1595
- _error: err instanceof Error ? err.message : String(err),
1596
- }));
1597
- const durationMs = Date.now() - startedAt;
1598
- const raw = result.text;
1599
- if (!raw) {
1600
- return {
1601
- plan: null,
1602
- raw: '',
1603
- durationMs,
1604
- parseError: result._error ?? 'empty response',
1605
- };
1606
- }
1607
- // Reuse the same JSON tolerance as decide(): strip markdown fences,
1608
- // then JSON.parse. On parse failure, return null and let the runner
1609
- // fall through.
1610
- let body = raw.trim();
1611
- if (body.startsWith('```')) {
1612
- body = body.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
1613
- }
1614
- let parsed;
1615
- try {
1616
- parsed = JSON.parse(body);
1617
- }
1618
- catch (err) {
1619
- return {
1620
- plan: null,
1621
- raw,
1622
- durationMs,
1623
- tokensUsed: result.tokensUsed,
1624
- inputTokens: result.inputTokens,
1625
- outputTokens: result.outputTokens,
1626
- cacheReadInputTokens: result.cacheReadInputTokens,
1627
- cacheCreationInputTokens: result.cacheCreationInputTokens,
1628
- parseError: err instanceof Error ? err.message : String(err),
1629
- };
1630
- }
1631
- if (!Array.isArray(parsed.steps) || parsed.steps.length === 0) {
1632
- return {
1633
- plan: null,
1634
- raw,
1635
- durationMs,
1636
- tokensUsed: result.tokensUsed,
1637
- inputTokens: result.inputTokens,
1638
- outputTokens: result.outputTokens,
1639
- cacheReadInputTokens: result.cacheReadInputTokens,
1640
- cacheCreationInputTokens: result.cacheCreationInputTokens,
1641
- parseError: 'plan has zero steps',
1642
- };
1643
- }
1644
- // Validate each step. Each must have a parseable action and a non-empty
1645
- // expectedEffect string. We use the same validateAction helper that the
1646
- // per-action parser uses, so the action shapes stay consistent.
1647
- const steps = [];
1648
- for (const [idx, rawStep] of parsed.steps.entries()) {
1649
- if (!rawStep || typeof rawStep !== 'object') {
1650
- return {
1651
- plan: null,
1652
- raw,
1653
- durationMs,
1654
- tokensUsed: result.tokensUsed,
1655
- inputTokens: result.inputTokens,
1656
- outputTokens: result.outputTokens,
1657
- cacheReadInputTokens: result.cacheReadInputTokens,
1658
- cacheCreationInputTokens: result.cacheCreationInputTokens,
1659
- parseError: `step ${idx + 1}: not an object`,
1660
- };
1661
- }
1662
- const stepObj = rawStep;
1663
- const actionRaw = stepObj.action;
1664
- if (!actionRaw || typeof actionRaw !== 'object') {
1665
- return {
1666
- plan: null,
1667
- raw,
1668
- durationMs,
1669
- parseError: `step ${idx + 1}: missing action`,
1670
- };
1671
- }
1672
- const actionData = actionRaw;
1673
- const actionType = actionData.action;
1674
- if (typeof actionType !== 'string') {
1675
- return {
1676
- plan: null,
1677
- raw,
1678
- durationMs,
1679
- parseError: `step ${idx + 1}: action.action must be a string`,
1680
- };
1681
- }
1682
- let action;
1683
- try {
1684
- action = validateAction(actionType, actionData);
1685
- }
1686
- catch (err) {
1687
- return {
1688
- plan: null,
1689
- raw,
1690
- durationMs,
1691
- parseError: `step ${idx + 1}: ${err instanceof Error ? err.message : String(err)}`,
1692
- };
1693
- }
1694
- const expectedEffect = typeof stepObj.expectedEffect === 'string' && stepObj.expectedEffect.length > 0
1695
- ? stepObj.expectedEffect
1696
- : 'page state advances after this action';
1697
- const rationale = typeof stepObj.rationale === 'string' ? stepObj.rationale : undefined;
1698
- steps.push({ action, expectedEffect, ...(rationale ? { rationale } : {}) });
1699
- }
1700
- const plan = {
1701
- steps: steps.slice(0, maxSteps),
1702
- ...(typeof parsed.finalResult === 'string' ? { finalResult: parsed.finalResult } : {}),
1703
- ...(typeof parsed.reasoning === 'string' ? { reasoning: parsed.reasoning } : {}),
1704
- };
1705
- return {
1706
- plan,
1707
- raw,
1708
- durationMs,
1709
- tokensUsed: result.tokensUsed,
1710
- inputTokens: result.inputTokens,
1711
- outputTokens: result.outputTokens,
1712
- cacheReadInputTokens: result.cacheReadInputTokens,
1713
- cacheCreationInputTokens: result.cacheCreationInputTokens,
1714
- };
246
+ return planImpl(this, goal, state, options);
1715
247
  }
1716
248
  /**
1717
249
  * Evaluate quality of the current page state.
@@ -1719,104 +251,10 @@ What is the complete plan?`;
1719
251
  * design, and professional polish.
1720
252
  */
1721
253
  async evaluate(state, goal) {
1722
- const textContent = `GOAL that was being worked on: ${goal}
1723
-
1724
- CURRENT PAGE:
1725
- URL: ${state.url}
1726
- Title: ${state.title}
1727
-
1728
- Please evaluate the quality of this page/application.`;
1729
- const userContent = this.buildUserContent(textContent, state.screenshot, true);
1730
- const result = await this.generate(EVALUATE_PROMPT, [{ role: 'user', content: userContent }], undefined, 800);
1731
- const raw = result.text;
1732
- const tokensUsed = result.tokensUsed;
1733
- if (this.debug) {
1734
- console.log('[Brain] Evaluation:', raw.slice(0, 300));
1735
- }
1736
- try {
1737
- let text = raw.trim();
1738
- if (text.startsWith('```')) {
1739
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
1740
- }
1741
- const parsed = JSON.parse(text);
1742
- const rawScore = typeof parsed.score === 'number' ? parsed.score : 5;
1743
- return {
1744
- score: Math.max(1, Math.min(10, rawScore)),
1745
- assessment: parsed.assessment ?? 'No assessment provided',
1746
- strengths: Array.isArray(parsed.strengths) ? parsed.strengths : [],
1747
- issues: Array.isArray(parsed.issues) ? parsed.issues : [],
1748
- suggestions: Array.isArray(parsed.suggestions) ? parsed.suggestions : [],
1749
- raw,
1750
- tokensUsed,
1751
- };
1752
- }
1753
- catch {
1754
- return {
1755
- score: 5,
1756
- assessment: 'Failed to parse evaluation response',
1757
- strengths: [],
1758
- issues: [],
1759
- suggestions: [],
1760
- raw,
1761
- tokensUsed,
1762
- };
1763
- }
254
+ return evaluateImpl(this, state, goal);
1764
255
  }
1765
256
  async recommendLinkCandidate(goal, state, candidates, extraContext) {
1766
- const topCandidates = candidates.slice(0, 5);
1767
- // Scout only needs candidates + context, not the full snapshot (saves 2-8k tokens)
1768
- const lines = [
1769
- `GOAL: ${goal}`,
1770
- '',
1771
- `PAGE: ${state.url} — ${state.title}`,
1772
- '',
1773
- 'CANDIDATES:',
1774
- ...topCandidates.map((candidate, index) => `${index + 1}. ${candidate.ref} — ${candidate.text} (score ${candidate.score})`),
1775
- ];
1776
- if (extraContext) {
1777
- lines.push('', extraContext);
1778
- }
1779
- lines.push('', 'Choose the single best next visible link.');
1780
- const userContent = this.buildUserContent(lines.join('\n'), state.screenshot, this.scoutUseVision);
1781
- const provider = this.scoutProvider || this.navProvider || this.provider;
1782
- const model = this.scoutModelName || this.navModelName || this.modelName;
1783
- const result = await this.generate(LINK_SCOUT_PROMPT, [{ role: 'user', content: userContent }], { provider, model }, 300);
1784
- const raw = result.text;
1785
- const tokensUsed = result.tokensUsed;
1786
- try {
1787
- let text = raw.trim();
1788
- if (text.startsWith('```')) {
1789
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
1790
- }
1791
- const parsed = JSON.parse(text);
1792
- const selector = typeof parsed.selector === 'string' ? parsed.selector.trim() : '';
1793
- const candidate = topCandidates.find((entry) => entry.ref === selector);
1794
- if (!candidate) {
1795
- throw new Error('invalid scout selector');
1796
- }
1797
- return {
1798
- selector,
1799
- reasoning: typeof parsed.reasoning === 'string' ? parsed.reasoning : 'No scout reasoning provided.',
1800
- confidence: typeof parsed.confidence === 'number'
1801
- ? Math.max(0, Math.min(1, parsed.confidence))
1802
- : 0.5,
1803
- raw,
1804
- tokensUsed,
1805
- };
1806
- }
1807
- catch {
1808
- const fallback = topCandidates[0];
1809
- if (!fallback) {
1810
- throw new Error('recommendLinkCandidate requires at least one candidate');
1811
- }
1812
- return {
1813
- selector: fallback.ref,
1814
- reasoning: 'Scout fallback: selected the top deterministic candidate after parse failure.',
1815
- confidence: 0.5,
1816
- raw,
1817
- tokensUsed,
1818
- };
1819
- }
257
+ return recommendLinkCandidateImpl(this, goal, state, candidates, extraContext);
1820
258
  }
1821
259
  /**
1822
260
  * Verify whether the goal was actually achieved.
@@ -1824,59 +262,7 @@ Please evaluate the quality of this page/application.`;
1824
262
  * Uses a fresh LLM call (no conversation history) to avoid self-confirmation bias.
1825
263
  */
1826
264
  async verifyGoalCompletion(state, goal, claimedResult) {
1827
- const siteBoundaryNote = buildFirstPartyBoundaryNote(goal, state.url);
1828
- const textContent = `GOAL: ${goal}
1829
-
1830
- AGENT'S CLAIMED RESULT: ${claimedResult}
1831
-
1832
- CURRENT PAGE:
1833
- URL: ${state.url}
1834
- Title: ${state.title}
1835
-
1836
- ELEMENTS:
1837
- ${budgetSnapshot(state.snapshot)}${siteBoundaryNote ? `\n\n${siteBoundaryNote}` : ''}
1838
-
1839
- Was the goal actually achieved? Analyze the current page state carefully.`;
1840
- const userContent = this.buildUserContent(textContent, state.screenshot, true);
1841
- // Verifier can use its own model, then the navigation model, then main.
1842
- const verifyProvider = this.verifierProvider
1843
- ? this.verifierProvider
1844
- : (this.adaptiveModelRouting && this.navModelName ? (this.navProvider || this.provider) : undefined);
1845
- const verifyModel = this.verifierModel
1846
- || (this.adaptiveModelRouting && this.navModelName ? this.navModelName : undefined);
1847
- const result = await this.generate(`Verify whether the browser agent achieved its goal. Respond with ONLY JSON:
1848
- {"achieved":true,"confidence":0.9,"evidence":["observation"],"missing":[]}
1849
-
1850
- Check: page state matches goal, no errors, URL is expected, claimed result matches visible data.
1851
- SUPPLEMENTAL TOOL EVIDENCE / SCRIPT RESULT in claimed results = verified DOM data, trustworthy even if page navigated away. Multi-page data collection is valid.`, [{ role: 'user', content: userContent }], verifyProvider && verifyModel ? { provider: verifyProvider, model: verifyModel } : undefined, 600);
1852
- const raw = result.text;
1853
- if (this.debug) {
1854
- console.log('[Brain] Goal verification:', raw.slice(0, 300));
1855
- }
1856
- try {
1857
- let text = raw.trim();
1858
- if (text.startsWith('```')) {
1859
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
1860
- }
1861
- const parsed = JSON.parse(text);
1862
- return {
1863
- achieved: parsed.achieved === true,
1864
- confidence: typeof parsed.confidence === 'number'
1865
- ? Math.max(0, Math.min(1, parsed.confidence))
1866
- : 0.5,
1867
- evidence: Array.isArray(parsed.evidence) ? parsed.evidence : [],
1868
- missing: Array.isArray(parsed.missing) ? parsed.missing : [],
1869
- };
1870
- }
1871
- catch {
1872
- // Parse failure — assume not verified (conservative)
1873
- return {
1874
- achieved: false,
1875
- confidence: 0,
1876
- evidence: [],
1877
- missing: ['Failed to parse goal verification response'],
1878
- };
1879
- }
265
+ return verifyGoalCompletionImpl(this, state, goal, claimedResult);
1880
266
  }
1881
267
  /**
1882
268
  * Audit design quality of the current page state.
@@ -1884,92 +270,7 @@ SUPPLEMENTAL TOOL EVIDENCE / SCRIPT RESULT in claimed results = verified DOM dat
1884
270
  * Returns structured findings with categories and severity levels.
1885
271
  */
1886
272
  async auditDesign(state, goal, checkpoints, systemPrompt) {
1887
- const textContent = `GOAL: ${goal}
1888
-
1889
- CHECKPOINTS to verify:
1890
- ${checkpoints.map((c, i) => `${i + 1}. ${c}`).join('\n')}
1891
-
1892
- CURRENT PAGE:
1893
- URL: ${state.url}
1894
- Title: ${state.title}
1895
-
1896
- ELEMENTS:
1897
- ${state.snapshot}
1898
-
1899
- Audit this page for design quality, UX issues, and visual bugs.`;
1900
- const userContent = this.buildUserContent(textContent, state.screenshot, true);
1901
- const result = await this.generate(systemPrompt ?? DESIGN_AUDIT_PROMPT, [{ role: 'user', content: userContent }], undefined, 8000);
1902
- const raw = result.text;
1903
- const tokensUsed = result.tokensUsed;
1904
- if (this.debug) {
1905
- console.log('[Brain] Design audit:', raw.slice(0, 300));
1906
- }
1907
- try {
1908
- let text = raw.trim();
1909
- if (text.startsWith('```')) {
1910
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
1911
- }
1912
- // Extract JSON object if surrounded by non-JSON text or truncated
1913
- let parsed;
1914
- try {
1915
- parsed = JSON.parse(text);
1916
- }
1917
- catch {
1918
- const start = text.indexOf('{');
1919
- const end = text.lastIndexOf('}');
1920
- if (start >= 0 && end > start) {
1921
- parsed = JSON.parse(text.slice(start, end + 1));
1922
- }
1923
- else {
1924
- throw new Error('No JSON object found');
1925
- }
1926
- }
1927
- const VALID_CATEGORIES = new Set(['visual-bug', 'layout', 'contrast', 'alignment', 'spacing', 'typography', 'accessibility', 'ux']);
1928
- const VALID_SEVERITIES = new Set(['critical', 'major', 'minor']);
1929
- const VALID_BLAST = new Set(['page', 'section', 'component', 'system']);
1930
- const clampScore = (n) => typeof n === 'number' ? Math.max(1, Math.min(10, n)) : undefined;
1931
- const findings = Array.isArray(parsed.findings)
1932
- ? parsed.findings.map((f) => ({
1933
- category: (VALID_CATEGORIES.has(f.category) ? f.category : 'ux'),
1934
- severity: (VALID_SEVERITIES.has(f.severity) ? f.severity : 'minor'),
1935
- description: String(f.description ?? ''),
1936
- location: String(f.location ?? ''),
1937
- suggestion: String(f.suggestion ?? ''),
1938
- ...(f.cssSelector ? { cssSelector: String(f.cssSelector) } : {}),
1939
- ...(f.cssFix ? { cssFix: String(f.cssFix) } : {}),
1940
- // Optional ROI fields.
1941
- ...(clampScore(f.impact) !== undefined ? { impact: clampScore(f.impact) } : {}),
1942
- ...(clampScore(f.effort) !== undefined ? { effort: clampScore(f.effort) } : {}),
1943
- ...(VALID_BLAST.has(f.blast)
1944
- ? { blast: f.blast }
1945
- : {}),
1946
- // Layer 2 — preserve raw patches array (untyped passthrough). The
1947
- // parsePatches/validatePatch pipeline in build-result.ts converts
1948
- // these into typed, validated Patch objects.
1949
- ...(Array.isArray(f.patches) ? { rawPatches: f.patches } : {}),
1950
- }))
1951
- : [];
1952
- const designSystemScore = parsed.designSystemScore && typeof parsed.designSystemScore === 'object'
1953
- ? parsed.designSystemScore
1954
- : undefined;
1955
- const rawScore = typeof parsed.score === 'number' ? parsed.score : 5;
1956
- return {
1957
- score: Math.max(1, Math.min(10, rawScore)),
1958
- findings,
1959
- raw,
1960
- tokensUsed,
1961
- designSystemScore,
1962
- };
1963
- }
1964
- catch (err) {
1965
- return {
1966
- score: 5,
1967
- findings: [],
1968
- raw,
1969
- tokensUsed,
1970
- parseError: err instanceof Error ? err.message : String(err),
1971
- };
1972
- }
273
+ return auditDesignImpl(this, state, goal, checkpoints, systemPrompt);
1973
274
  }
1974
275
  /**
1975
276
  * Extract reusable knowledge from a completed trajectory.
@@ -1977,49 +278,7 @@ Audit this page for design quality, UX issues, and visual bugs.`;
1977
278
  * and app quirks from a successful run.
1978
279
  */
1979
280
  async extractKnowledge(trajectoryText, domain) {
1980
- const result = await this.generate(`You are analyzing a browser automation trajectory to extract reusable knowledge.
1981
- Extract facts that would help an agent complete similar tasks faster next time.
1982
-
1983
- Respond with ONLY a JSON array of facts:
1984
- [
1985
- {"type": "timing", "key": "page-load", "value": "wait 3000ms after navigation for content to hydrate"},
1986
- {"type": "selector", "key": "send-button", "value": "[data-testid='chat-send-button'] is the reliable send button selector"},
1987
- {"type": "pattern", "key": "auth-flow", "value": "Click sign-in → fill email → fill password → click submit → wait for redirect"},
1988
- {"type": "quirk", "key": "lazy-loading", "value": "File tree loads asynchronously — wait for entries before asserting"}
1989
- ]
1990
-
1991
- Types:
1992
- - timing: wait durations, delays that are necessary
1993
- - selector: reliable selectors for important elements
1994
- - pattern: multi-step interaction sequences
1995
- - quirk: app-specific behaviors or gotchas
1996
-
1997
- Only include facts that are genuinely useful. Quality over quantity. Max 10 facts.`, [{
1998
- role: 'user',
1999
- content: `Domain: ${domain}\n\nTrajectory:\n${trajectoryText}`,
2000
- }], undefined, 800);
2001
- try {
2002
- let text = result.text.trim();
2003
- if (text.startsWith('```')) {
2004
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
2005
- }
2006
- const parsed = JSON.parse(text);
2007
- if (!Array.isArray(parsed))
2008
- return [];
2009
- const VALID_TYPES = new Set(['timing', 'selector', 'pattern', 'quirk']);
2010
- return parsed
2011
- .filter((f) => VALID_TYPES.has(f.type) &&
2012
- typeof f.key === 'string' &&
2013
- typeof f.value === 'string')
2014
- .map((f) => ({
2015
- type: f.type,
2016
- key: f.key,
2017
- value: f.value,
2018
- }));
2019
- }
2020
- catch {
2021
- return [];
2022
- }
281
+ return extractKnowledgeImpl(this, trajectoryText, domain);
2023
282
  }
2024
283
  /**
2025
284
  * Generic text-completion entry point for non-agent uses (GEPA reflective
@@ -2030,490 +289,25 @@ Only include facts that are genuinely useful. Quality over quantity. Max 10 fact
2030
289
  * the configured provider/model.
2031
290
  */
2032
291
  async complete(system, user, options = {}) {
2033
- const result = await this.generate(system, [{ role: 'user', content: user }], undefined, options.maxOutputTokens ?? 1500);
2034
- return { text: result.text, tokensUsed: result.tokensUsed };
292
+ return completeImpl(this, system, user, options);
2035
293
  }
2036
- parse(raw) {
2037
- let text = raw.trim();
2038
- // Strip markdown code blocks
2039
- if (text.startsWith('```')) {
2040
- text = text.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '');
2041
- }
2042
- const VALID_ACTIONS = new Set([
2043
- 'click', 'type', 'press', 'hover', 'select',
2044
- 'scroll', 'navigate', 'wait', 'evaluate', 'runScript',
2045
- 'extractWithIndex',
2046
- 'verifyPreview', 'complete', 'abort',
2047
- 'fill', 'clickSequence',
2048
- 'clickAt', 'typeAt',
2049
- 'clickLabel', 'typeLabel',
2050
- // Macro dispatch. The driver validates the macro name at execute time.
2051
- 'macro',
2052
- // Parallel fan-out. Runner handles dispatch; validator checks shape.
2053
- 'fanOut',
2054
- ]);
2055
- // Parse strategy: exact → first-{/last-} extraction.
2056
- // Some OpenAI-compat gateways (router.tangle.tools, LiteLLM proxies, etc.)
2057
- // wrap model output in prose preambles ("Here's your response:\n{...}")
2058
- // that markdown-fence stripping doesn't catch. Fall back to extracting the
2059
- // outermost object literal before giving up.
2060
- let parsed = null;
2061
- let parseError = '';
2062
- try {
2063
- parsed = JSON.parse(text);
2064
- }
2065
- catch (err) {
2066
- parseError = err instanceof Error ? err.message : String(err);
2067
- const firstBrace = text.indexOf('{');
2068
- const lastBrace = text.lastIndexOf('}');
2069
- if (firstBrace >= 0 && lastBrace > firstBrace) {
2070
- try {
2071
- parsed = JSON.parse(text.slice(firstBrace, lastBrace + 1));
2072
- parseError = '';
2073
- }
2074
- catch { /* fall through to retry fallback */ }
2075
- }
2076
- }
2077
- if (!parsed) {
2078
- return {
2079
- // Do not hard-abort the scenario on transient JSON formatting issues.
2080
- // Waiting one turn lets the loop continue and recover on the next model call.
2081
- action: { action: 'wait', ms: 1000 },
2082
- reasoning: `Malformed LLM JSON response (${parseError}). Retrying next turn.`,
2083
- };
2084
- }
2085
- try {
2086
- const actionObj = parsed.action && typeof parsed.action === 'object' ? parsed.action : parsed;
2087
- const actionType = typeof parsed.action === 'string' ? parsed.action : actionObj?.action;
2088
- if (!actionType) {
2089
- throw new Error('Missing action field');
2090
- }
2091
- if (!VALID_ACTIONS.has(actionType)) {
2092
- throw new Error(`Unknown action "${actionType}". Valid: ${[...VALID_ACTIONS].join(', ')}`);
2093
- }
2094
- const actionData = parsed.action && typeof parsed.action === 'object'
2095
- ? parsed.action
2096
- : parsed;
2097
- const action = validateAction(actionType, actionData);
2098
- return {
2099
- action,
2100
- nextActions: parseNextActions(parsed, VALID_ACTIONS),
2101
- reasoning: (parsed.reasoning || parsed.thought || parsed.thinking),
2102
- plan: Array.isArray(parsed.plan) ? parsed.plan : undefined,
2103
- currentStep: typeof parsed.currentStep === 'number' ? parsed.currentStep : undefined,
2104
- expectedEffect: (parsed.expectedEffect || parsed.expected_effect),
2105
- };
2106
- }
2107
- catch (err) {
2108
- const validationError = err instanceof Error ? err.message : String(err);
2109
- return {
2110
- action: { action: 'wait', ms: 1000 },
2111
- reasoning: `Malformed LLM JSON response (${validationError}). Retrying next turn.`,
2112
- };
2113
- }
2114
- }
2115
- }
2116
- /**
2117
- * Collapse consecutive runs of similar elements (same indent + role, names
2118
- * differing only by a trailing number/short suffix) into a single representative
2119
- * line with a count. Reduces token cost on pages with long pagination, nav
2120
- * lists, or repeated product cards.
2121
- *
2122
- * Skips dialog/alertdialog (agent must see each one) and groups < 3 items.
2123
- */
2124
- function deduplicateSnapshot(snapshot) {
2125
- const lines = snapshot.split('\n');
2126
- const out = [];
2127
- // Extract (indent, role) from a snapshot line. Returns null for non-element lines.
2128
- const parseLine = (line) => {
2129
- const m = line.match(/^(\s*-\s+)(\w+)\s+"([^"]*)"\s*\[ref=(\w+)\]/);
2130
- if (!m)
2131
- return null;
2132
- return { indent: m[1], role: m[2], name: m[3], ref: m[4], full: line };
2133
- };
2134
- // Strip trailing numbers/ordinals to get a "name stem" for grouping.
2135
- // "Page 1" and "Page 20" → "Page ", "Item #3" and "Item #42" → "Item #"
2136
- const nameStem = (name) => name.replace(/\d+/g, '#');
2137
- // Structural fingerprint for a block of lines (element + its children).
2138
- // Used for card-level dedup: two hotel cards have different names but the
2139
- // same structure (listitem > link + img + text + text + button).
2140
- const structuralFingerprint = (startIdx, baseIndent) => {
2141
- const roles = [];
2142
- let j = startIdx;
2143
- while (j < lines.length) {
2144
- const p = parseLine(lines[j]);
2145
- if (!p) {
2146
- j++;
2147
- continue;
2148
- }
2149
- // Stop when we hit an element at the same or shallower indent (sibling or parent)
2150
- if (j > startIdx && p.indent.length <= baseIndent.length)
2151
- break;
2152
- roles.push(p.role);
2153
- j++;
2154
- }
2155
- return { fp: roles.join(','), endIdx: j };
2156
- };
2157
- let i = 0;
2158
- while (i < lines.length) {
2159
- const parsed = parseLine(lines[i]);
2160
- // Non-element line or dialog/alertdialog — emit as-is
2161
- if (!parsed || /\b(?:dialog|alertdialog)\b/i.test(parsed.role)) {
2162
- out.push(lines[i]);
2163
- i++;
2164
- continue;
2165
- }
2166
- // Try block-level dedup first: look for consecutive sibling blocks
2167
- // with the same structural fingerprint (same child-role sequence).
2168
- // This catches card patterns like Booking hotel results, Allrecipes cards.
2169
- if (/\b(?:listitem|article|group|region)\b/i.test(parsed.role)) {
2170
- const { fp: firstFp, endIdx: firstEnd } = structuralFingerprint(i, parsed.indent);
2171
- if (firstEnd > i + 2 && firstFp.includes(',')) { // non-trivial block
2172
- const blocks = [
2173
- { start: i, end: firstEnd, firstLine: lines[i] },
2174
- ];
2175
- let scanIdx = firstEnd;
2176
- while (scanIdx < lines.length) {
2177
- const nextParsed = parseLine(lines[scanIdx]);
2178
- if (!nextParsed || nextParsed.indent !== parsed.indent || nextParsed.role !== parsed.role)
2179
- break;
2180
- const { fp: nextFp, endIdx: nextEnd } = structuralFingerprint(scanIdx, nextParsed.indent);
2181
- if (nextFp !== firstFp)
2182
- break;
2183
- blocks.push({ start: scanIdx, end: nextEnd, firstLine: lines[scanIdx] });
2184
- scanIdx = nextEnd;
2185
- }
2186
- if (blocks.length >= 3) {
2187
- // Emit first 2 blocks fully, summarize the rest
2188
- for (let b = 0; b < Math.min(2, blocks.length); b++) {
2189
- for (let k = blocks[b].start; k < blocks[b].end; k++) {
2190
- out.push(lines[k]);
2191
- }
2192
- }
2193
- const remaining = blocks.length - 2;
2194
- out.push(`${parsed.indent}... [${remaining} more similar ${parsed.role} blocks with same structure]`);
2195
- i = blocks[blocks.length - 1].end;
2196
- continue;
2197
- }
2198
- }
2199
- }
2200
- // Line-level dedup: consecutive runs of same (indent, role, name stem)
2201
- const group = [parsed];
2202
- const stem = nameStem(parsed.name);
2203
- let j = i + 1;
2204
- while (j < lines.length) {
2205
- const next = parseLine(lines[j]);
2206
- if (!next ||
2207
- next.indent !== parsed.indent ||
2208
- next.role !== parsed.role ||
2209
- nameStem(next.name) !== stem)
2210
- break;
2211
- group.push(next);
2212
- j++;
2213
- }
2214
- if (group.length < 3) {
2215
- // Not enough to dedup — emit originals
2216
- for (const g of group)
2217
- out.push(g.full);
2218
- }
2219
- else {
2220
- // Emit first element with a summary of the rest
2221
- const last = group[group.length - 1];
2222
- out.push(`${parsed.full} (+${group.length - 1} similar: "${group[1].name}"\u2026"${last.name}")`);
2223
- }
2224
- i = j;
2225
- }
2226
- return out.join('\n');
2227
- }
2228
- /**
2229
- * Cap snapshot size to control token cost on large pages.
2230
- * Keeps the full snapshot when it fits within budget; otherwise preserves:
2231
- * 1. Interactive elements with refs (buttons, inputs, links — for action targets)
2232
- * 2. Content lines: term/definition/code/pre/paragraph (for extraction tasks
2233
- * like MDN, Python docs, W3C spec where the value the agent needs lives in
2234
- * a `<dl>/<code>/<pre>` block, not in an interactive element)
2235
- *
2236
- * New-page callers use a larger default budget; same-page callers can pass a
2237
- * tighter budget after the model has already seen the full snapshot once.
2238
- */
2239
- export function budgetSnapshot(snapshot, maxChars = 24_000) {
2240
- // Skip dedup on small snapshots — not enough repetition to justify the O(n) scan
2241
- if (snapshot.length > 6_000) {
2242
- snapshot = deduplicateSnapshot(snapshot);
2243
- }
2244
- if (snapshot.length <= maxChars)
2245
- return snapshot;
2246
- // First pass: separate keep-set (interactive + content lines) from decorative.
2247
- // Content roles (term, definition, code, pre, paragraph) carry text the LLM
2248
- // needs for extraction tasks. They have no [ref=] but the text is the data.
2249
- const lines = snapshot.split('\n');
2250
- const interactive = [];
2251
- const content = [];
2252
- const decorative = [];
2253
- for (const line of lines) {
2254
- if (/\b(?:button|link|textbox|combobox|menuitem|checkbox|radio|select|heading|dialog|alertdialog)\b/i.test(line) && /\[ref=/.test(line)) {
2255
- interactive.push(line);
2256
- }
2257
- else if (/^\s*-\s+(?:term|definition|code|pre|paragraph)\b/i.test(line)) {
2258
- content.push(line);
2259
- }
2260
- else {
2261
- decorative.push(line);
2262
- }
2263
- }
2264
- // If interactive + content fits, use both with a truncation note
2265
- const keepSet = interactive.concat(content);
2266
- const keepText = keepSet.join('\n');
2267
- if (keepText.length <= maxChars) {
2268
- return keepText + `\n... [${decorative.length} decorative elements omitted for brevity]`;
2269
- }
2270
- // Second pass: when interactive + content still exceed budget, prioritize:
2271
- // 1. inputs (searchbox/textbox/combobox) — essential for form tasks
2272
- // 2. headings + dialogs — structural navigation
2273
- // 3. content lines (term/definition/code/pre) — extraction data
2274
- // 4. bulk links/buttons — main content
2275
- const priority = [];
2276
- const bulk = [];
2277
- for (const line of interactive) {
2278
- if (/\b(?:searchbox|textbox|combobox|heading|dialog|alertdialog)\b/i.test(line)) {
2279
- priority.push(line);
2280
- }
2281
- else {
2282
- bulk.push(line);
2283
- }
2284
- }
2285
- // Content lines come right after priority interactive (they're the extraction
2286
- // data) and before bulk links/buttons.
2287
- const priorityWithContent = priority.concat(content);
2288
- const priorityText = priorityWithContent.join('\n');
2289
- const remaining = maxChars - priorityText.length - 80; // reserve space for note
2290
- if (remaining > 0) {
2291
- const bulkText = bulk.join('\n');
2292
- const trimmedBulk = bulkText.slice(0, remaining);
2293
- const bulkKept = trimmedBulk.lastIndexOf('\n') > 0
2294
- ? trimmedBulk.slice(0, trimmedBulk.lastIndexOf('\n'))
2295
- : trimmedBulk;
2296
- return priorityText + '\n' + bulkKept +
2297
- `\n... [${interactive.length - priority.length - bulkKept.split('\n').length} interactive + ${decorative.length} decorative elements omitted]`;
2298
- }
2299
- // Hard cap: take the first maxChars of the full snapshot
2300
- return snapshot.slice(0, maxChars) + '\n... [snapshot truncated — large page]';
2301
- }
2302
- function compactFirstTurnSnapshot(snapshot) {
2303
- const compact = AriaSnapshotHelper.formatCompact(snapshot);
2304
- const basis = compact.length > 0 ? compact : snapshot;
2305
- const maxChars = 4000;
2306
- if (basis.length <= maxChars)
2307
- return basis;
2308
- return `${basis.slice(0, maxChars)}\n... [snapshot truncated for first-turn fast path]`;
2309
- }
2310
- function parseNextActions(parsed, validActions) {
2311
- if (!Array.isArray(parsed.nextActions)) {
2312
- return undefined;
2313
- }
2314
- const nextActions = [];
2315
- for (const entry of parsed.nextActions.slice(0, 3)) {
2316
- if (!entry || typeof entry !== 'object')
2317
- continue;
2318
- const rawEntry = entry;
2319
- const actionType = typeof rawEntry.action === 'string' ? rawEntry.action : undefined;
2320
- if (!actionType || !validActions.has(actionType))
2321
- continue;
2322
- try {
2323
- nextActions.push(validateAction(actionType, rawEntry));
2324
- }
2325
- catch {
2326
- // Best effort: ignore malformed follow-up action.
2327
- }
2328
- }
2329
- return nextActions.length > 0 ? nextActions : undefined;
2330
- }
2331
- /**
2332
- * Runtime validation of LLM-parsed action objects.
2333
- * Ensures required fields are present and correctly typed per action variant.
2334
- * Throws on missing/invalid fields so the caller can abort gracefully.
2335
- */
2336
- function validateAction(actionType, data) {
2337
- const requireStr = (field) => {
2338
- const v = data[field];
2339
- if (typeof v !== 'string' || !v)
2340
- throw new Error(`${actionType} action requires "${field}" (string)`);
2341
- return v;
2342
- };
2343
- const optStr = (field) => {
2344
- const v = data[field];
2345
- return typeof v === 'string' ? v : '';
2346
- };
2347
- const num = (v, fallback) => (typeof v === 'number' ? v : fallback);
2348
- switch (actionType) {
2349
- case 'click':
2350
- return { action: 'click', selector: requireStr('selector') };
2351
- case 'type':
2352
- return { action: 'type', selector: requireStr('selector'), text: optStr('text') };
2353
- case 'press':
2354
- return { action: 'press', selector: requireStr('selector'), key: requireStr('key') };
2355
- case 'hover':
2356
- return { action: 'hover', selector: requireStr('selector') };
2357
- case 'select':
2358
- return { action: 'select', selector: requireStr('selector'), value: optStr('value') };
2359
- case 'scroll':
2360
- return {
2361
- action: 'scroll',
2362
- direction: data.direction === 'up' ? 'up' : 'down',
2363
- ...(data.amount != null ? { amount: num(data.amount, 500) } : {}),
2364
- };
2365
- case 'navigate':
2366
- return { action: 'navigate', url: requireStr('url') };
2367
- case 'wait':
2368
- return { action: 'wait', ms: num(data.ms, 1000) };
2369
- case 'evaluate':
2370
- return { action: 'evaluate', criteria: optStr('criteria') };
2371
- case 'runScript':
2372
- return { action: 'runScript', script: requireStr('script') };
2373
- case 'extractWithIndex': {
2374
- const query = requireStr('query');
2375
- const contains = typeof data.contains === 'string' ? data.contains : '';
2376
- return {
2377
- action: 'extractWithIndex',
2378
- query,
2379
- ...(contains ? { contains } : {}),
2380
- };
2381
- }
2382
- case 'verifyPreview':
2383
- return { action: 'verifyPreview' };
2384
- case 'complete':
2385
- return { action: 'complete', result: optStr('result') };
2386
- case 'abort':
2387
- return { action: 'abort', reason: optStr('reason') || 'No reason provided' };
2388
- case 'fill': {
2389
- // Multi-field batch fill — at least one of fields/selects/checks must be non-empty
2390
- const fields = isStringRecord(data.fields) ? data.fields : undefined;
2391
- const selects = isStringRecord(data.selects) ? data.selects : undefined;
2392
- const checks = Array.isArray(data.checks) && data.checks.every((c) => typeof c === 'string')
2393
- ? data.checks
2394
- : undefined;
2395
- const fieldCount = (fields ? Object.keys(fields).length : 0)
2396
- + (selects ? Object.keys(selects).length : 0)
2397
- + (checks ? checks.length : 0);
2398
- if (fieldCount === 0) {
2399
- throw new Error('fill action requires at least one of "fields" (object), "selects" (object), or "checks" (string[])');
2400
- }
2401
- return {
2402
- action: 'fill',
2403
- ...(fields ? { fields } : {}),
2404
- ...(selects ? { selects } : {}),
2405
- ...(checks ? { checks } : {}),
2406
- };
2407
- }
2408
- case 'clickSequence': {
2409
- const refs = Array.isArray(data.refs) && data.refs.every((r) => typeof r === 'string')
2410
- ? data.refs
2411
- : null;
2412
- if (!refs || refs.length === 0) {
2413
- throw new Error('clickSequence action requires "refs" (string[]) with at least one entry');
2414
- }
2415
- return {
2416
- action: 'clickSequence',
2417
- refs,
2418
- ...(typeof data.intervalMs === 'number' ? { intervalMs: data.intervalMs } : {}),
2419
- };
2420
- }
2421
- // Vision-first coordinate actions.
2422
- case 'clickAt':
2423
- return { action: 'clickAt', x: num(data.x, 0), y: num(data.y, 0) };
2424
- case 'typeAt':
2425
- return { action: 'typeAt', x: num(data.x, 0), y: num(data.y, 0), text: optStr('text') };
2426
- // Set-of-Marks label-based actions.
2427
- case 'clickLabel':
2428
- return { action: 'clickLabel', label: num(data.label, 0) };
2429
- case 'typeLabel':
2430
- return { action: 'typeLabel', label: num(data.label, 0), text: optStr('text') };
2431
- // Macro invocation. The driver validates the name and required args at
2432
- // execute time; macros without params may omit args.
2433
- case 'macro': {
2434
- const args = {};
2435
- if (data.args && typeof data.args === 'object' && !Array.isArray(data.args)) {
2436
- for (const [k, v] of Object.entries(data.args)) {
2437
- if (typeof v === 'string')
2438
- args[k] = v;
2439
- }
2440
- }
2441
- return {
2442
- action: 'macro',
2443
- name: requireStr('name'),
2444
- ...(Object.keys(args).length > 0 ? { args } : {}),
2445
- };
2446
- }
2447
- case 'fanOut': {
2448
- // Accept either explicit subGoals[] or the shorthand
2449
- // baseUrl+goalTemplate+items trio. resolveSubGoals() in
2450
- // runner/fan-out.ts expands shorthand → subGoals at execute time.
2451
- const subGoals = Array.isArray(data.subGoals)
2452
- ? data.subGoals
2453
- .filter((s) => s && typeof s === 'object')
2454
- .map((s) => ({
2455
- url: typeof s.url === 'string' ? s.url : '',
2456
- goal: typeof s.goal === 'string' ? s.goal : '',
2457
- ...(typeof s.label === 'string' ? { label: s.label } : {}),
2458
- ...(typeof s.maxTurns === 'number' ? { maxTurns: s.maxTurns } : {}),
2459
- }))
2460
- .filter((s) => s.url && s.goal)
2461
- : undefined;
2462
- const baseUrl = typeof data.baseUrl === 'string' ? data.baseUrl : undefined;
2463
- const goalTemplate = typeof data.goalTemplate === 'string' ? data.goalTemplate : undefined;
2464
- const items = Array.isArray(data.items) && data.items.every((i) => typeof i === 'string')
2465
- ? data.items
2466
- : undefined;
2467
- const hasExplicit = subGoals && subGoals.length > 0;
2468
- const hasShorthand = baseUrl && goalTemplate && items && items.length > 0;
2469
- if (!hasExplicit && !hasShorthand) {
2470
- throw new Error('fanOut action requires either "subGoals" (array of {url,goal}) or the shorthand trio "baseUrl" + "goalTemplate" + "items" (non-empty string[])');
2471
- }
2472
- return {
2473
- action: 'fanOut',
2474
- ...(hasExplicit ? { subGoals } : {}),
2475
- ...(hasShorthand ? { baseUrl, goalTemplate, items } : {}),
2476
- ...(typeof data.summarize === 'string' ? { summarize: data.summarize } : {}),
2477
- };
2478
- }
2479
- default:
2480
- throw new Error(`Unknown action type: ${actionType}`);
294
+ /**
295
+ * Multimodal sibling of {@link complete}: a single round-trip with a system
296
+ * prompt, a user prompt, and one-or-more already-encoded images. The narrow
297
+ * vision seam the reference-grounded taste judge binds to (one Brain per
298
+ * `{ provider, model }` ref), built on the same private `generate` so it goes
299
+ * through the existing provider abstraction — any vision-capable backend works.
300
+ *
301
+ * Images arrive ENCODED (`{ image, mediaType }`, base64/data); the disk read +
302
+ * mediaType inference live in the judge's `createBrainVisionModel` adapter, so
303
+ * this method does no IO. It is NOT `auditDesign` — the page-audit seam stays
304
+ * off-limits to taste comparison by contract.
305
+ */
306
+ async completeVision(system, user, images, options = {}) {
307
+ return completeVisionImpl(this, system, user, images, options);
2481
308
  }
2482
- }
2483
- /** Type guard: value is a Record<string, string> */
2484
- function isStringRecord(value) {
2485
- if (!value || typeof value !== 'object' || Array.isArray(value))
2486
- return false;
2487
- for (const v of Object.values(value)) {
2488
- if (typeof v !== 'string')
2489
- return false;
309
+ parse(raw) {
310
+ return parseDecision(raw);
2490
311
  }
2491
- return true;
2492
- }
2493
- /**
2494
- * Build a fetch replacement that forces `"stream": false` on chat completions
2495
- * bodies for OpenAI-compatible gateways that default to SSE streaming.
2496
- */
2497
- function createForceNonStreamingFetch() {
2498
- return async (input, init) => {
2499
- if (init?.body && typeof init.body === 'string') {
2500
- const body = init.body;
2501
- // Cheap content-sniff so we only rewrite chat-completions shaped bodies,
2502
- // not arbitrary POSTs the caller might make (embeddings, etc.).
2503
- if (body.includes('"messages"') && body.includes('"model"')) {
2504
- try {
2505
- const parsed = JSON.parse(body);
2506
- if (parsed.stream === undefined || parsed.stream === true) {
2507
- parsed.stream = false;
2508
- init = { ...init, body: JSON.stringify(parsed) };
2509
- }
2510
- }
2511
- catch {
2512
- // Non-JSON body — pass through unchanged.
2513
- }
2514
- }
2515
- }
2516
- return fetch(input, init);
2517
- };
2518
312
  }
2519
313
  //# sourceMappingURL=index.js.map