minimal-agent 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (470) hide show
  1. package/README.md +6 -5
  2. package/package.json +60 -4
  3. package/plugins/computer_use/.claude-plugin/plugin.json +1 -1
  4. package/plugins/cron-runner/src/commands.js +4 -4
  5. package/plugins/cron-runner/src/commands.ts +4 -4
  6. package/plugins/workflow-runner/.claude-plugin/plugin.json +2 -2
  7. package/plugins/workflow-runner/commands/workflow.md +57 -90
  8. package/plugins/workflow-runner/commands/workflows.md +4 -4
  9. package/plugins/workflow-runner/plugin.js +24 -13
  10. package/plugins/workflow-runner/plugin.ts +28 -13
  11. package/plugins/workflow-runner/src/discovery.js +160 -0
  12. package/plugins/workflow-runner/src/discovery.ts +194 -0
  13. package/plugins/workflow-runner/src/goalAnchor.js +53 -0
  14. package/plugins/workflow-runner/src/goalAnchor.ts +59 -0
  15. package/plugins/workflow-runner/src/index.js +197 -206
  16. package/plugins/workflow-runner/src/index.ts +248 -249
  17. package/plugins/workflow-runner/src/inputCoercion.js +40 -0
  18. package/plugins/workflow-runner/src/inputCoercion.ts +43 -0
  19. package/plugins/workflow-runner/src/journal.js +224 -0
  20. package/plugins/workflow-runner/src/journal.ts +274 -0
  21. package/plugins/workflow-runner/src/jsonSchema.js +132 -0
  22. package/plugins/workflow-runner/src/jsonSchema.ts +162 -0
  23. package/plugins/workflow-runner/src/moduleResolution.js +64 -0
  24. package/plugins/workflow-runner/src/moduleResolution.ts +71 -0
  25. package/plugins/workflow-runner/src/resourceInputs.js +220 -0
  26. package/plugins/workflow-runner/src/resourceInputs.ts +300 -0
  27. package/plugins/workflow-runner/src/runner.js +400 -489
  28. package/plugins/workflow-runner/src/runner.ts +488 -560
  29. package/plugins/workflow-runner/src/runtime/config.js +46 -0
  30. package/plugins/workflow-runner/src/runtime/config.ts +69 -0
  31. package/plugins/workflow-runner/src/runtime/context.js +1304 -0
  32. package/plugins/workflow-runner/src/runtime/context.ts +1786 -0
  33. package/plugins/workflow-runner/src/runtime/exec.js +34 -0
  34. package/plugins/workflow-runner/src/runtime/exec.ts +53 -0
  35. package/plugins/workflow-runner/src/runtime/flow.js +8 -0
  36. package/plugins/workflow-runner/src/runtime/flow.ts +12 -0
  37. package/plugins/workflow-runner/src/runtime/fs.js +27 -0
  38. package/plugins/workflow-runner/src/runtime/fs.ts +41 -0
  39. package/plugins/workflow-runner/src/runtime/model.js +6 -0
  40. package/plugins/workflow-runner/src/runtime/model.ts +8 -0
  41. package/plugins/workflow-runner/src/runtime/output.js +7 -0
  42. package/plugins/workflow-runner/src/runtime/output.ts +21 -0
  43. package/plugins/workflow-runner/src/runtime/scheduler.js +141 -0
  44. package/plugins/workflow-runner/src/runtime/scheduler.ts +166 -0
  45. package/plugins/workflow-runner/src/runtime/state.js +16 -0
  46. package/plugins/workflow-runner/src/runtime/state.ts +19 -0
  47. package/plugins/workflow-runner/src/runtime/tools.js +3 -0
  48. package/plugins/workflow-runner/src/runtime/tools.ts +6 -0
  49. package/plugins/workflow-runner/src/sandbox.js +141 -0
  50. package/plugins/workflow-runner/src/sandbox.ts +182 -0
  51. package/plugins/workflow-runner/src/scriptGuard.d.mts +38 -0
  52. package/plugins/workflow-runner/src/scriptGuard.mjs +449 -0
  53. package/plugins/workflow-runner/src/taskIdentity.js +151 -307
  54. package/plugins/workflow-runner/src/taskIdentity.ts +207 -355
  55. package/plugins/workflow-runner/src/toolRuntime.js +29 -8
  56. package/plugins/workflow-runner/src/toolRuntime.ts +36 -11
  57. package/plugins/workflow-runner/src/types.js +7 -211
  58. package/plugins/workflow-runner/src/types.ts +154 -623
  59. package/plugins/workflow-runner/src/util.js +47 -0
  60. package/plugins/workflow-runner/src/util.ts +51 -0
  61. package/plugins/workflow-runner/src/workflowContext.js +20 -0
  62. package/plugins/workflow-runner/src/workflowContext.ts +58 -0
  63. package/plugins/workflow-runner/src/workflowDependencies.js +36 -0
  64. package/plugins/workflow-runner/src/workflowDependencies.ts +59 -0
  65. package/plugins/workflow-runner/src/workflowFingerprint.js +118 -0
  66. package/plugins/workflow-runner/src/workflowFingerprint.ts +122 -0
  67. package/plugins/workflow-runner/src/workflowState.js +474 -802
  68. package/plugins/workflow-runner/src/workflowState.ts +575 -970
  69. package/plugins/workflow-runner/test/api.test.ts +223 -0
  70. package/plugins/workflow-runner/test/apiCapabilities.test.ts +614 -0
  71. package/plugins/workflow-runner/test/apiConcurrencyGate.test.ts +104 -0
  72. package/plugins/workflow-runner/test/apiEcho.test.ts +210 -0
  73. package/plugins/workflow-runner/test/apiReadText.test.ts +178 -0
  74. package/plugins/workflow-runner/test/apiResourceLedger.test.ts +184 -0
  75. package/plugins/workflow-runner/test/childWorkflow.test.ts +304 -0
  76. package/plugins/workflow-runner/test/discovery.test.ts +234 -0
  77. package/plugins/workflow-runner/test/goalAnchor.test.ts +106 -0
  78. package/plugins/workflow-runner/test/goalController.test.ts +317 -0
  79. package/plugins/workflow-runner/test/journalLanes.test.ts +198 -0
  80. package/plugins/workflow-runner/test/listResources.test.ts +106 -0
  81. package/plugins/workflow-runner/test/resourceInputs.test.ts +294 -0
  82. package/plugins/workflow-runner/test/sandbox.test.ts +126 -0
  83. package/plugins/workflow-runner/test/scheduler.test.ts +70 -0
  84. package/plugins/workflow-runner/test/scriptGuard.test.ts +276 -0
  85. package/plugins/workflow-runner/test/skillDocSync.test.ts +78 -0
  86. package/plugins/workflow-runner/test/taskIdentity.test.ts +288 -179
  87. package/plugins/workflow-runner/test/workflowFingerprint.test.ts +106 -0
  88. package/plugins/workflow-runner/test/workflowState.test.ts +429 -0
  89. package/skills/pptx/examples/assets/cover-bg.svg +23 -0
  90. package/skills/pptx/examples/assets/icon-spark.svg +5 -0
  91. package/skills/ui-design/SKILL.md +5 -2
  92. package/skills/ui-design/references/layout-composition.md +1 -1
  93. package/skills/workflow-creator/SKILL.md +344 -0
  94. package/skills/workflow-creator/assets/template/workflow.mjs +48 -0
  95. package/skills/workflow-creator/references/api.md +216 -0
  96. package/skills/workflow-creator/references/migration.md +103 -0
  97. package/skills/workflow-creator/references/patterns.md +779 -0
  98. package/skills/workflow-creator/scripts/list-resources.mjs +205 -0
  99. package/skills/workflow-creator/scripts/scaffold.mjs +148 -0
  100. package/skills/workflow-creator/scripts/validate.mjs +275 -0
  101. package/skills/workflow-creator/scripts/workflow-creator.test.mjs +82 -0
  102. package/src/advisor/advisorCoordinator.js +503 -0
  103. package/src/advisor/advisorCoordinator.ts +657 -0
  104. package/src/cli/configMenu.tsx +0 -1
  105. package/src/cli/print.js +27 -6
  106. package/src/cli/print.ts +48 -4
  107. package/src/cli/streamJson.js +5 -6
  108. package/src/cli/streamJson.ts +8 -4
  109. package/src/cli/wizards/advisorWizard.js +4 -2
  110. package/src/cli/wizards/advisorWizard.tsx +8 -2
  111. package/src/cli/wizards/contextAdvisorWizard.js +5 -4
  112. package/src/cli/wizards/contextAdvisorWizard.tsx +7 -6
  113. package/src/collaboration/taskFlow.js +115 -0
  114. package/src/collaboration/taskFlow.ts +169 -0
  115. package/src/config/configDefaults.js +41 -1
  116. package/src/config/configDefaults.ts +43 -1
  117. package/src/config/configDiagnostics.ts +2 -2
  118. package/src/config/configFile.js +40 -9
  119. package/src/config/configFile.ts +60 -11
  120. package/src/config/configService.js +419 -74
  121. package/src/config/configService.ts +703 -145
  122. package/src/config/numericLimits.js +22 -0
  123. package/src/config/numericLimits.ts +24 -0
  124. package/src/config.js +1 -0
  125. package/src/config.ts +3 -2
  126. package/src/context/budgetPlan.js +1 -1
  127. package/src/context/budgetPlan.ts +1 -1
  128. package/src/context/compact.js +7 -2
  129. package/src/context/compact.ts +9 -2
  130. package/src/context/microCompactLite.js +212 -13
  131. package/src/context/microCompactLite.ts +261 -12
  132. package/src/context/sessionRegistry.js +8 -1
  133. package/src/context/sessionRegistry.ts +18 -5
  134. package/src/context/toolResultArchive.js +72 -0
  135. package/src/context/toolResultArchive.ts +95 -0
  136. package/src/goal/errors.js +8 -0
  137. package/src/goal/errors.ts +18 -0
  138. package/src/goal/goalRuntime.js +344 -0
  139. package/src/goal/goalRuntime.ts +466 -0
  140. package/src/goal/goalStore.js +200 -0
  141. package/src/goal/goalStore.ts +219 -0
  142. package/src/goal/index.js +4 -0
  143. package/src/goal/index.ts +4 -0
  144. package/src/goal/statusMapping.js +71 -0
  145. package/src/goal/statusMapping.ts +95 -0
  146. package/src/goal/types.js +7 -0
  147. package/src/goal/types.ts +179 -0
  148. package/src/goal/workflowController.js +306 -0
  149. package/src/goal/workflowController.ts +360 -0
  150. package/src/harness/evidenceLedger.js +2 -0
  151. package/src/harness/evidenceLedger.ts +1 -0
  152. package/src/harness/stallGuard.js +9 -3
  153. package/src/harness/stallGuard.ts +7 -3
  154. package/src/images/imageResolver.js +161 -0
  155. package/src/images/imageResolver.ts +189 -0
  156. package/src/infra/verification.js +34 -5
  157. package/src/infra/verification.ts +39 -4
  158. package/src/llm/client.js +79 -21
  159. package/src/llm/client.ts +182 -35
  160. package/src/llm/gatewayClient.js +201 -0
  161. package/src/llm/gatewayClient.ts +299 -0
  162. package/src/llm/messageProvenance.js +46 -0
  163. package/src/llm/messageProvenance.ts +58 -0
  164. package/src/llm/modelProvider.js +1 -0
  165. package/src/llm/modelProvider.ts +22 -0
  166. package/src/llm/providers/anthropicFormat.js +108 -1
  167. package/src/llm/providers/anthropicFormat.ts +115 -4
  168. package/src/llm/providers/openaiFormat.js +9 -0
  169. package/src/llm/providers/openaiFormat.ts +12 -3
  170. package/src/loop.js +319 -54
  171. package/src/loop.ts +373 -57
  172. package/src/main.js +4 -1
  173. package/src/main.tsx +4 -1
  174. package/src/managedExecution/index.js +2 -0
  175. package/src/managedExecution/index.ts +2 -0
  176. package/src/managedExecution/registry.js +15 -0
  177. package/src/managedExecution/registry.ts +20 -0
  178. package/src/managedExecution/types.js +1 -0
  179. package/src/managedExecution/types.ts +86 -0
  180. package/src/modelRef.js +9 -0
  181. package/src/modelRef.ts +31 -0
  182. package/src/multimodal.js +14 -28
  183. package/src/multimodal.ts +17 -33
  184. package/src/net/lanDiscovery.js +65 -0
  185. package/src/net/lanDiscovery.ts +82 -0
  186. package/src/plugin-sdk.js +28 -3
  187. package/src/plugin-sdk.ts +39 -6
  188. package/src/plugins/pluginApi.ts +16 -1
  189. package/src/plugins/pluginLoader.js +4 -0
  190. package/src/plugins/pluginLoader.ts +6 -0
  191. package/src/plugins/pluginRunner.js +139 -7
  192. package/src/plugins/pluginRunner.ts +173 -9
  193. package/src/plugins/slashFallback.js +113 -0
  194. package/src/plugins/slashFallback.ts +137 -0
  195. package/src/prompts/policyRegistry.js +12 -32
  196. package/src/prompts/policyRegistry.ts +15 -32
  197. package/src/prompts/promptAssembly.js +103 -12
  198. package/src/prompts/promptAssembly.ts +114 -12
  199. package/src/prompts/skillList.js +21 -3
  200. package/src/prompts/skillList.ts +27 -2
  201. package/src/prompts/stableSystemPrompt.js +18 -0
  202. package/src/prompts/stableSystemPrompt.ts +33 -0
  203. package/src/runtime/agentRuntimeContext.js +8 -2
  204. package/src/runtime/agentRuntimeContext.ts +17 -4
  205. package/src/session/runAgentTurn.js +78 -12
  206. package/src/session/runAgentTurn.ts +105 -23
  207. package/src/session/runTurn.js +10 -9
  208. package/src/session/runTurn.ts +17 -8
  209. package/src/session/sessionState.ts +2 -0
  210. package/src/team/auth.js +186 -0
  211. package/src/team/auth.ts +298 -0
  212. package/src/team/cli.js +105 -0
  213. package/src/team/cli.ts +122 -0
  214. package/src/team/onlineToolContract.js +64 -0
  215. package/src/team/onlineToolContract.ts +109 -0
  216. package/src/team/toolPolicy.js +36 -0
  217. package/src/team/toolPolicy.ts +75 -0
  218. package/src/tools/advisor/advisor.js +90 -46
  219. package/src/tools/advisor/advisor.ts +100 -47
  220. package/src/tools/advisor/advisorDecision.js +71 -0
  221. package/src/tools/advisor/advisorDecision.ts +85 -0
  222. package/src/tools/agent/agent.js +18 -5
  223. package/src/tools/agent/agent.ts +21 -7
  224. package/src/tools/agent/agentControl.js +246 -0
  225. package/src/tools/agent/agentControl.ts +284 -0
  226. package/src/tools/agent/agentTaskService.js +40 -1
  227. package/src/tools/agent/agentTaskService.ts +44 -1
  228. package/src/tools/agent/agentTypes.js +14 -2
  229. package/src/tools/agent/agentTypes.ts +15 -2
  230. package/src/tools/applypatch/applyPatch.js +362 -0
  231. package/src/tools/applypatch/applyPatch.ts +428 -0
  232. package/src/tools/bash/bash.js +22 -15
  233. package/src/tools/bash/bash.ts +23 -17
  234. package/src/tools/capabilities.js +10 -5
  235. package/src/tools/capabilities.ts +10 -5
  236. package/src/tools/computeruse/computerIntent.js +32 -0
  237. package/src/tools/computeruse/computerIntent.ts +46 -0
  238. package/src/tools/computeruse/telemetryLog.js +12 -0
  239. package/src/tools/computeruse/telemetryLog.ts +16 -0
  240. package/src/tools/computeruse/vision/telemetry.js +1 -0
  241. package/src/tools/computeruse/vision/telemetry.ts +1 -0
  242. package/src/tools/effectiveToolSet.js +4 -0
  243. package/src/tools/effectiveToolSet.ts +4 -0
  244. package/src/tools/index.js +21 -2
  245. package/src/tools/index.ts +33 -2
  246. package/src/tools/presentresult/presentresult.js +6 -15
  247. package/src/tools/presentresult/presentresult.ts +6 -15
  248. package/src/tools/read/read.js +62 -9
  249. package/src/tools/read/read.ts +63 -9
  250. package/src/tools/shared/fileUtils.js +1 -2
  251. package/src/tools/shared/fileUtils.ts +1 -2
  252. package/src/tools/shared/toolOutputStore.js +23 -1
  253. package/src/tools/shared/toolOutputStore.ts +22 -1
  254. package/src/tools/skill/skill.js +9 -3
  255. package/src/tools/skill/skill.ts +8 -4
  256. package/src/tools/toolPromptGuides.js +117 -0
  257. package/src/tools/toolPromptGuides.ts +137 -0
  258. package/src/tools/webbrowser/agentBrowserPath.js +19 -9
  259. package/src/tools/webbrowser/agentBrowserPath.ts +70 -28
  260. package/src/tools/webbrowser/browser.js +27 -11
  261. package/src/tools/webbrowser/browser.ts +68 -24
  262. package/src/tools/webbrowser/browserAutomation.js +329 -0
  263. package/src/tools/webbrowser/browserAutomation.ts +467 -0
  264. package/src/tools/webbrowser/browserAxState.js +97 -0
  265. package/src/tools/webbrowser/browserAxState.ts +142 -0
  266. package/src/tools/webbrowser/browserBatch.js +1578 -0
  267. package/src/tools/webbrowser/browserBatch.ts +2382 -0
  268. package/src/tools/webbrowser/browserCdp.js +208 -0
  269. package/src/tools/webbrowser/browserCdp.ts +255 -0
  270. package/src/tools/webbrowser/browserConditions.js +24 -0
  271. package/src/tools/webbrowser/browserConditions.ts +45 -0
  272. package/src/tools/webbrowser/browserContent.js +281 -0
  273. package/src/tools/webbrowser/browserContent.ts +438 -0
  274. package/src/tools/webbrowser/browserControl.js +131 -34
  275. package/src/tools/webbrowser/browserControl.ts +210 -56
  276. package/src/tools/webbrowser/browserDaemon.js +22 -14
  277. package/src/tools/webbrowser/browserDaemon.ts +54 -21
  278. package/src/tools/webbrowser/browserFrames.js +47 -0
  279. package/src/tools/webbrowser/browserFrames.ts +54 -0
  280. package/src/tools/webbrowser/browserInteractionResolver.js +17 -0
  281. package/src/tools/webbrowser/browserInteractionResolver.ts +40 -0
  282. package/src/tools/webbrowser/browserLocator.js +140 -0
  283. package/src/tools/webbrowser/browserLocator.ts +194 -0
  284. package/src/tools/webbrowser/browserOutcome.js +10 -17
  285. package/src/tools/webbrowser/browserOutcome.ts +48 -31
  286. package/src/tools/webbrowser/browserRecovery.js +58 -0
  287. package/src/tools/webbrowser/browserRecovery.ts +83 -0
  288. package/src/tools/webbrowser/browserRunRegistry.js +35 -3
  289. package/src/tools/webbrowser/browserRunRegistry.ts +91 -15
  290. package/src/tools/webbrowser/browserService.js +28 -2
  291. package/src/tools/webbrowser/browserService.ts +49 -9
  292. package/src/tools/webbrowser/browserStream.js +132 -33
  293. package/src/tools/webbrowser/browserStream.ts +224 -50
  294. package/src/tools/webbrowser/browserTabs.js +48 -15
  295. package/src/tools/webbrowser/browserTabs.ts +178 -96
  296. package/src/tools/webbrowser/webbrowser.js +679 -144
  297. package/src/tools/webbrowser/webbrowser.ts +975 -236
  298. package/src/tools/webfetch/preapproved.js +0 -2
  299. package/src/tools/webfetch/preapproved.ts +0 -2
  300. package/src/tools/websearch/websearch.js +129 -8
  301. package/src/tools/websearch/websearch.ts +138 -6
  302. package/src/types.ts +440 -38
  303. package/src/ui/App.js +377 -58
  304. package/src/ui/App.tsx +576 -111
  305. package/src/ui/LiveArea.js +14 -8
  306. package/src/ui/LiveArea.tsx +14 -10
  307. package/src/ui/MessageList.js +63 -20
  308. package/src/ui/MessageList.tsx +101 -16
  309. package/src/ui/ModelPicker.js +7 -4
  310. package/src/ui/ModelPicker.tsx +55 -8
  311. package/src/ui/hooks/useChat.js +152 -163
  312. package/src/ui/hooks/useChat.ts +396 -334
  313. package/src/ui/loginCommand.js +56 -0
  314. package/src/ui/loginCommand.ts +63 -0
  315. package/src/ui/toolProgress.js +41 -0
  316. package/src/ui/toolProgress.ts +44 -0
  317. package/src/web/browserBridge.js +20 -2
  318. package/src/web/browserBridge.ts +20 -2
  319. package/src/web/protocol.ts +1897 -194
  320. package/workflows/adaptive-research/workflow.mjs +86 -0
  321. package/workflows/e2e-capabilities/workflow.mjs +102 -0
  322. package/workflows/e2e-llm/workflow.mjs +52 -0
  323. package/workflows/e2e-smoke/workflow.mjs +59 -0
  324. package/workflows/e2e-vision-schema/README.md +48 -0
  325. package/workflows/e2e-vision-schema/workflow.mjs +149 -0
  326. package/workflows/e2e-write-greet/workflow.mjs +37 -0
  327. package/workflows/file-census/README.md +14 -0
  328. package/workflows/file-census/scripts/.gitkeep +2 -0
  329. package/workflows/file-census/workflow.mjs +110 -0
  330. package/workflows/goal/README.md +166 -0
  331. package/workflows/goal/prompts/audit.md +57 -0
  332. package/workflows/goal/prompts/contract.md +56 -0
  333. package/workflows/goal/prompts/iterate.md +37 -0
  334. package/workflows/goal/workflow.mjs +501 -0
  335. package/workflows/illustrate/README.md +90 -0
  336. package/workflows/illustrate/assets/engines.json +20 -0
  337. package/workflows/illustrate/assets/extract-baidu.js +21 -0
  338. package/workflows/illustrate/assets/extract-bing.js +33 -0
  339. package/workflows/illustrate/assets/extract-google.js +31 -0
  340. package/workflows/illustrate/workflow.mjs +301 -0
  341. package/workflows/plan/workflow.mjs +389 -0
  342. package/workflows/question-crops/README.md +35 -0
  343. package/workflows/question-crops/prompts/inventory.md +23 -0
  344. package/workflows/question-crops/scripts/image-ops.mjs +305 -0
  345. package/workflows/question-crops/scripts/image-ops.test.mjs +95 -0
  346. package/workflows/question-crops/workflow.mjs +172 -0
  347. package/workflows/ralph-loop/workflow.mjs +162 -0
  348. package/workflows/research/README.md +74 -0
  349. package/workflows/research/prompts/s0-goal.md +43 -0
  350. package/workflows/research/prompts/s1-early-exit.md +22 -0
  351. package/workflows/research/prompts/s1-map.md +58 -0
  352. package/workflows/research/prompts/s2-options.md +53 -0
  353. package/workflows/research/prompts/s3-deep-dive.md +56 -0
  354. package/workflows/research/prompts/s4-verify.md +74 -0
  355. package/workflows/research/prompts/s5-synthesis.md +81 -0
  356. package/workflows/research/prompts/s5b-fill-gaps.md +32 -0
  357. package/workflows/research/prompts/s6-package.md +32 -0
  358. package/workflows/research/scripts/build-report.mjs +98 -0
  359. package/workflows/research/workflow.mjs +288 -0
  360. package/workflows/wechat-reading-shots/prompts/collect-params.md +16 -0
  361. package/workflows/wechat-reading-shots/prompts/evaluate.md +20 -0
  362. package/workflows/wechat-reading-shots/prompts/open-book.md +28 -0
  363. package/workflows/wechat-reading-shots/prompts/read-page.md +13 -0
  364. package/workflows/wechat-reading-shots/prompts/turn-batch.md +24 -0
  365. package/workflows/wechat-reading-shots/test/control-flow.test.ts +329 -0
  366. package/workflows/wechat-reading-shots/workflow.mjs +266 -0
  367. package/workflows/wrong-question-docx/README.md +55 -0
  368. package/workflows/wrong-question-docx/prompts/figure-qa.md +8 -0
  369. package/workflows/wrong-question-docx/prompts/transcribe.md +25 -0
  370. package/workflows/wrong-question-docx/scripts/workbook.mjs +857 -0
  371. package/workflows/wrong-question-docx/test/control-flow.test.ts +165 -0
  372. package/workflows/wrong-question-docx/test/workbook.test.mjs +303 -0
  373. package/workflows/wrong-question-docx/workflow.mjs +485 -0
  374. package/plugins/workflow-runner/src/agentCommon.js +0 -354
  375. package/plugins/workflow-runner/src/agentCommon.ts +0 -435
  376. package/plugins/workflow-runner/src/controlFlow/branch.js +0 -40
  377. package/plugins/workflow-runner/src/controlFlow/branch.ts +0 -46
  378. package/plugins/workflow-runner/src/controlFlow/loop.js +0 -211
  379. package/plugins/workflow-runner/src/controlFlow/loop.ts +0 -248
  380. package/plugins/workflow-runner/src/controlFlow/pause.js +0 -129
  381. package/plugins/workflow-runner/src/controlFlow/pause.ts +0 -141
  382. package/plugins/workflow-runner/src/controlFlow/signal.js +0 -21
  383. package/plugins/workflow-runner/src/controlFlow/signal.ts +0 -29
  384. package/plugins/workflow-runner/src/controlFlow/types.js +0 -18
  385. package/plugins/workflow-runner/src/controlFlow/types.ts +0 -63
  386. package/plugins/workflow-runner/src/dslManifest.js +0 -72
  387. package/plugins/workflow-runner/src/dslManifest.ts +0 -127
  388. package/plugins/workflow-runner/src/expressions.js +0 -639
  389. package/plugins/workflow-runner/src/expressions.ts +0 -640
  390. package/plugins/workflow-runner/src/loader.js +0 -573
  391. package/plugins/workflow-runner/src/loader.ts +0 -721
  392. package/plugins/workflow-runner/src/multimodal.js +0 -38
  393. package/plugins/workflow-runner/src/multimodal.ts +0 -64
  394. package/plugins/workflow-runner/src/recovery.js +0 -126
  395. package/plugins/workflow-runner/src/recovery.ts +0 -156
  396. package/plugins/workflow-runner/src/stepExecutors/agent.js +0 -173
  397. package/plugins/workflow-runner/src/stepExecutors/agent.ts +0 -234
  398. package/plugins/workflow-runner/src/stepExecutors/assert.js +0 -42
  399. package/plugins/workflow-runner/src/stepExecutors/assert.ts +0 -52
  400. package/plugins/workflow-runner/src/stepExecutors/echo.js +0 -31
  401. package/plugins/workflow-runner/src/stepExecutors/echo.ts +0 -38
  402. package/plugins/workflow-runner/src/stepExecutors/llm.js +0 -99
  403. package/plugins/workflow-runner/src/stepExecutors/llm.ts +0 -133
  404. package/plugins/workflow-runner/src/stepExecutors/set.js +0 -49
  405. package/plugins/workflow-runner/src/stepExecutors/set.ts +0 -56
  406. package/plugins/workflow-runner/src/stepExecutors/skill.js +0 -111
  407. package/plugins/workflow-runner/src/stepExecutors/skill.ts +0 -140
  408. package/plugins/workflow-runner/src/stepExecutors/tool.js +0 -227
  409. package/plugins/workflow-runner/src/stepExecutors/tool.ts +0 -274
  410. package/plugins/workflow-runner/src/stepExecutors/verify.js +0 -61
  411. package/plugins/workflow-runner/src/stepExecutors/verify.ts +0 -72
  412. package/plugins/workflow-runner/src/stepPipeline.js +0 -277
  413. package/plugins/workflow-runner/src/stepPipeline.ts +0 -366
  414. package/plugins/workflow-runner/test/agent.test.ts +0 -587
  415. package/plugins/workflow-runner/test/agentLlmError.test.ts +0 -373
  416. package/plugins/workflow-runner/test/bashBlacklist.sweep.test.ts +0 -65
  417. package/plugins/workflow-runner/test/captureMisbind.test.ts +0 -99
  418. package/plugins/workflow-runner/test/cli.e2e.test.ts +0 -167
  419. package/plugins/workflow-runner/test/context-files.test.ts +0 -320
  420. package/plugins/workflow-runner/test/e2e.test.ts +0 -370
  421. package/plugins/workflow-runner/test/e2eNewLayout.test.ts +0 -314
  422. package/plugins/workflow-runner/test/engineRobustness.test.ts +0 -234
  423. package/plugins/workflow-runner/test/expressions.test.ts +0 -439
  424. package/plugins/workflow-runner/test/foundation.test.ts +0 -126
  425. package/plugins/workflow-runner/test/graceful.test.ts +0 -139
  426. package/plugins/workflow-runner/test/greedyInputs.load.test.ts +0 -47
  427. package/plugins/workflow-runner/test/hostCapability.test.ts +0 -108
  428. package/plugins/workflow-runner/test/illustrate.load.test.ts +0 -183
  429. package/plugins/workflow-runner/test/jsonPayload.test.ts +0 -83
  430. package/plugins/workflow-runner/test/loader.test.ts +0 -617
  431. package/plugins/workflow-runner/test/maxTurnsRetryDelay.test.ts +0 -33
  432. package/plugins/workflow-runner/test/multimodal.test.ts +0 -342
  433. package/plugins/workflow-runner/test/output-schema.test.ts +0 -213
  434. package/plugins/workflow-runner/test/parseWorkflowArgs.test.ts +0 -106
  435. package/plugins/workflow-runner/test/pluginRunner.isolation.test.ts +0 -269
  436. package/plugins/workflow-runner/test/ppt.load.test.ts +0 -1220
  437. package/plugins/workflow-runner/test/ralphLoop.e2e.test.ts +0 -221
  438. package/plugins/workflow-runner/test/recovery.test.ts +0 -265
  439. package/plugins/workflow-runner/test/research.load.test.ts +0 -328
  440. package/plugins/workflow-runner/test/runner.test.ts +0 -911
  441. package/plugins/workflow-runner/test/runnerPreflight.test.ts +0 -436
  442. package/plugins/workflow-runner/test/skillTurnDone.test.ts +0 -71
  443. package/plugins/workflow-runner/test/stateLifecycle.test.ts +0 -198
  444. package/plugins/workflow-runner/test/stepPipeline.test.ts +0 -114
  445. package/plugins/workflow-runner/test/taskArbitration.test.ts +0 -354
  446. package/plugins/workflow-runner/test/taskLifecycle.test.ts +0 -324
  447. package/plugins/workflow-runner/test/toolArtifactOverwrite.test.ts +0 -221
  448. package/plugins/workflow-runner/test/toolCapabilityBoundary.test.ts +0 -29
  449. package/plugins/workflow-runner/test/toolRuntimeIntegration.test.ts +0 -168
  450. package/plugins/workflow-runner/test/webppt.load.test.ts +0 -250
  451. package/plugins/workflow-runner/test/word.load.test.ts +0 -208
  452. package/plugins/workflow-runner/test/workflows.smoke.test.ts +0 -76
  453. package/plugins/workflow-runner/test/yamlGuardConditions.test.ts +0 -207
  454. package/src/advisorState.js +0 -22
  455. package/src/cli/configWizard.js +0 -293
  456. package/src/tools/toolProgressBridge.js +0 -48
  457. package/workflows/adaptive-research.yaml +0 -50
  458. package/workflows/book-review-short.yaml +0 -125
  459. package/workflows/e2e-write-greet.yaml +0 -31
  460. package/workflows/illustrate.yaml +0 -227
  461. package/workflows/plan.yaml +0 -470
  462. package/workflows/ppt.yaml +0 -1670
  463. package/workflows/ralph-loop.yaml +0 -201
  464. package/workflows/research.yaml +0 -873
  465. package/workflows/schema.json +0 -176
  466. package/workflows/webppt.yaml +0 -496
  467. package/workflows/wechat-reading-shots.yaml +0 -315
  468. package/workflows/word.yaml +0 -720
  469. package/workflows/youtube-shorts.yaml +0 -202
  470. /package/workflows/{research.md → research/DESIGN.md} +0 -0
@@ -0,0 +1,2382 @@
1
+ import { createHash, randomUUID } from 'node:crypto';
2
+ import { createReadStream } from 'node:fs';
3
+ import {
4
+ copyFile,
5
+ readdir,
6
+ realpath,
7
+ rename,
8
+ rm,
9
+ stat,
10
+ writeFile,
11
+ } from 'node:fs/promises';
12
+ import { basename, extname, isAbsolute, join, relative } from 'node:path';
13
+ import { z } from 'zod';
14
+
15
+ import { getWorkingDir } from '../../bootstrap/workingDir.js';
16
+ import { configService } from '../../config/configService.js';
17
+ import type {
18
+ DeliverableManifest,
19
+ Tool,
20
+ ToolCallContext,
21
+ ToolResult,
22
+ } from '../../types.js';
23
+ import { toToolParameters } from '../../utils/zodToJson.js';
24
+ import { DEFAULT_MAX_RESULT_SIZE_CHARS } from '../types.js';
25
+ import { resolveAgentBrowserPath } from './agentBrowserPath.js';
26
+ import {
27
+ BROWSER_SESSION,
28
+ browserArtifactDir,
29
+ createArtifactPath,
30
+ sessionArgs,
31
+ spawnRunner,
32
+ } from './browser.js';
33
+ import {
34
+ BROWSER_TAB_MARKER_KEY,
35
+ type CdpEvent,
36
+ configureDownloads,
37
+ getBrowserAutomation,
38
+ } from './browserAutomation.js';
39
+ import {
40
+ type BrowserAxNode,
41
+ conciseAxTree,
42
+ diffAxTree,
43
+ normalizeAxTree,
44
+ } from './browserAxState.js';
45
+ import {
46
+ type BrowserCondition,
47
+ conditionToScript,
48
+ } from './browserConditions.js';
49
+ import {
50
+ type BrowserPageAsset,
51
+ type BrowserPageContent,
52
+ CLIPBOARD_READ_EXPRESSION,
53
+ CLIPBOARD_READ_TEXT_EXPRESSION,
54
+ PAGE_ASSETS_EXPRESSION,
55
+ browserAssetFetchExpression,
56
+ bundlePageAssets,
57
+ clipboardWriteExpression,
58
+ clipboardWriteTextExpression,
59
+ googleWorkspaceExportExpression,
60
+ markdownFromPageContent,
61
+ pageContentExpression,
62
+ readUrlsInBackground,
63
+ } from './browserContent.js';
64
+ import { detectBrowserChallenge } from './browserControl.js';
65
+ import { ensureDaemon } from './browserDaemon.js';
66
+ import {
67
+ type BrowserFrameInfo,
68
+ normalizeFrameInfo,
69
+ normalizeFrameTree,
70
+ } from './browserFrames.js';
71
+ import { resolveInteractionStrategy } from './browserInteractionResolver.js';
72
+ import {
73
+ type BrowserLocator,
74
+ buildLocatorExpression,
75
+ buildTargetExpression,
76
+ } from './browserLocator.js';
77
+ import { withBrowserRecovery } from './browserRecovery.js';
78
+ import { browserRunRegistry } from './browserRunRegistry.js';
79
+ import { browserService } from './browserService.js';
80
+ import { getBrowserStream } from './browserStream.js';
81
+ import {
82
+ claimConversationTabById,
83
+ ensureConversationTab,
84
+ parseTabList,
85
+ } from './browserTabs.js';
86
+
87
+ export type { BrowserLocator } from './browserLocator.js';
88
+
89
+ const locatorLeafSchema = z.object({
90
+ by: z.enum([
91
+ 'css',
92
+ 'ax',
93
+ 'role',
94
+ 'text',
95
+ 'label',
96
+ 'placeholder',
97
+ 'alt',
98
+ 'title',
99
+ 'testid',
100
+ ]),
101
+ value: z.string(),
102
+ name: z.string().optional(),
103
+ exact: z.boolean().optional(),
104
+ strict: z.boolean().optional(),
105
+ nth: z.number().int().nonnegative().optional(),
106
+ has_text: z.string().optional(),
107
+ generation: z.number().int().positive().optional(),
108
+ frameId: z.string().min(1).optional(),
109
+ frame_id: z.string().min(1).optional(),
110
+ });
111
+ // Keep the public JSON Schema finite. A single nested scope and a finite frame
112
+ // chain cover the browser pages we support while avoiding recursive-schema
113
+ // fallbacks that turn the model contract into `any`.
114
+ const locatorSchema = z.object({
115
+ ...locatorLeafSchema.shape,
116
+ within: locatorLeafSchema.optional(),
117
+ frame: z.array(locatorLeafSchema).optional(),
118
+ framePath: z.array(locatorLeafSchema).optional(),
119
+ frame_path: z.array(locatorLeafSchema).optional(),
120
+ }) as z.ZodType<BrowserLocator>;
121
+ const expectationSchema = z.object({
122
+ navigation: z
123
+ .object({
124
+ url: z.string().optional(),
125
+ wait_until: z.enum(['commit', 'domcontentloaded', 'load']).optional(),
126
+ })
127
+ .optional(),
128
+ new_tab: z.object({ url: z.string().optional() }).optional(),
129
+ download: z.object({ filename: z.string().optional() }).optional(),
130
+ file_chooser: z.object({ files: z.array(z.string()).optional() }).optional(),
131
+ conditions: z
132
+ .array(
133
+ z.union([
134
+ z.object({ type: z.literal('url'), pattern: z.string() }),
135
+ z.object({
136
+ type: z.literal('text'),
137
+ value: z.string(),
138
+ exact: z.boolean().optional(),
139
+ }),
140
+ z.object({
141
+ type: z.literal('element-visible'),
142
+ locator: locatorSchema.optional(),
143
+ selector: z.string().optional(),
144
+ }),
145
+ z.object({
146
+ type: z.literal('element-hidden'),
147
+ locator: locatorSchema.optional(),
148
+ selector: z.string().optional(),
149
+ }),
150
+ z.object({
151
+ type: z.literal('load-state'),
152
+ state: z.enum(['domcontentloaded', 'load', 'networkidle']),
153
+ }),
154
+ z.object({ type: z.literal('page-change') }),
155
+ z.object({
156
+ type: z.literal('download'),
157
+ filename: z.string().optional(),
158
+ }),
159
+ z.object({ type: z.literal('new-tab'), url: z.string().optional() }),
160
+ z.object({
161
+ type: z.literal('dialog'),
162
+ dialogType: z
163
+ .enum(['alert', 'confirm', 'prompt', 'beforeunload'])
164
+ .optional(),
165
+ }),
166
+ ]),
167
+ )
168
+ .max(8)
169
+ .optional(),
170
+ });
171
+ const stepSchema = z.object({
172
+ id: z.string().min(1).max(100).optional(),
173
+ action: z.enum([
174
+ 'navigate',
175
+ 'click',
176
+ 'doubleClick',
177
+ 'fill',
178
+ 'type',
179
+ 'hover',
180
+ 'press',
181
+ 'check',
182
+ 'uncheck',
183
+ 'setChecked',
184
+ 'getState',
185
+ 'drag',
186
+ 'wait',
187
+ 'waitForURL',
188
+ 'waitForDownload',
189
+ 'waitForFileChooser',
190
+ 'snapshot',
191
+ 'frameInfo',
192
+ 'visibleDom',
193
+ 'clickNode',
194
+ 'typeNode',
195
+ 'scrollNode',
196
+ 'upload',
197
+ 'pageAssets',
198
+ 'listPageAssets',
199
+ 'bundlePageAssets',
200
+ 'exportContent',
201
+ 'readUrls',
202
+ 'clipboardReadText',
203
+ 'clipboardWriteText',
204
+ 'clipboardRead',
205
+ 'clipboardWrite',
206
+ ]),
207
+ selector: z.string().optional(),
208
+ locator: locatorSchema.optional(),
209
+ trigger_selector: z.string().optional(),
210
+ trigger_locator: locatorSchema.optional(),
211
+ value: z.string().optional(),
212
+ files: z.array(z.string()).optional(),
213
+ key: z.string().optional(),
214
+ node_id: z.number().int().nonnegative().optional(),
215
+ checked: z.boolean().optional(),
216
+ url: z.string().optional(),
217
+ wait_ms: z.number().int().min(1).max(120_000).optional(),
218
+ drag_path: z
219
+ .array(z.object({ x: z.number(), y: z.number() }))
220
+ .min(2)
221
+ .optional(),
222
+ expect: expectationSchema.optional(),
223
+ intent: z.enum(['normal', 'submit']).optional(),
224
+ export_format: z
225
+ .enum([
226
+ 'text',
227
+ 'txt',
228
+ 'markdown',
229
+ 'md',
230
+ 'html',
231
+ 'json',
232
+ 'page',
233
+ 'google-workspace',
234
+ 'youtube-transcript',
235
+ 'pdf',
236
+ 'docx',
237
+ 'xlsx',
238
+ 'csv',
239
+ 'pptx',
240
+ ])
241
+ .optional(),
242
+ urls: z.array(z.string().url()).max(20).optional(),
243
+ concurrency: z.number().int().min(1).max(8).optional(),
244
+ content_type: z.enum(['text', 'html', 'domSnapshot']).optional(),
245
+ bundle_dir: z.string().optional(),
246
+ bundle_assets: z.boolean().optional(),
247
+ clipboard_text: z.string().optional(),
248
+ clipboard_items: z
249
+ .array(z.object({ type: z.string().min(1), base64: z.string() }))
250
+ .optional(),
251
+ is_multiple: z.boolean().optional(),
252
+ artifact: z
253
+ .object({
254
+ save_as: z.string().min(1).max(255),
255
+ label: z.string().max(200).optional(),
256
+ })
257
+ .optional(),
258
+ });
259
+ type Step = z.infer<typeof stepSchema>;
260
+ const inputSchema = z.object({
261
+ tab_id: z.string().optional(),
262
+ retention: z
263
+ .enum(['ephemeral', 'conversation', 'handoff', 'deliverable'])
264
+ .optional(),
265
+ tab_title: z.string().max(200).optional(),
266
+ steps: z.array(stepSchema).min(1).max(50),
267
+ stop_on_error: z.boolean().default(true),
268
+ timeout_ms: z.number().int().min(1_000).max(300_000).default(120_000),
269
+ final_snapshot: z.enum(['none', 'auto', 'full', 'diff']).default('auto'),
270
+ capture: z.enum(['none', 'state', 'screenshot', 'both']).default('state'),
271
+ });
272
+ type Input = z.infer<typeof inputSchema>;
273
+
274
+ type Snapshot = {
275
+ id: string;
276
+ revision: number;
277
+ generation: number;
278
+ url: string;
279
+ text: string;
280
+ nodes: Map<string, BrowserAxNode>;
281
+ updatedAt: number;
282
+ };
283
+ type FrameInfo = BrowserFrameInfo;
284
+ type DownloadItem = {
285
+ id: string;
286
+ url: string;
287
+ filename: string;
288
+ path: string;
289
+ state: 'start' | 'progress' | 'complete' | 'canceled';
290
+ receivedBytes?: number;
291
+ totalBytes?: number;
292
+ error?: string;
293
+ startedAt: number;
294
+ };
295
+ const snapshots = new Map<string, Snapshot>();
296
+ const MAX_SNAPSHOTS = 256;
297
+ const SNAPSHOT_TTL_MS = 15 * 60_000;
298
+
299
+ function jsString(value: string): string {
300
+ return JSON.stringify(value);
301
+ }
302
+
303
+ const DOWNLOAD_MIME_BY_EXT: Record<string, string> = {
304
+ '.csv': 'text/csv',
305
+ '.docx':
306
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
307
+ '.pdf': 'application/pdf',
308
+ '.png': 'image/png',
309
+ '.jpg': 'image/jpeg',
310
+ '.jpeg': 'image/jpeg',
311
+ '.pptx':
312
+ 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
313
+ '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
314
+ '.zip': 'application/zip',
315
+ };
316
+
317
+ async function hashFile(path: string): Promise<string> {
318
+ const hash = createHash('sha256');
319
+ for await (const chunk of createReadStream(path)) hash.update(chunk);
320
+ return hash.digest('hex');
321
+ }
322
+
323
+ async function materializeDownloadArtifact(
324
+ sourcePath: string,
325
+ saveAs: string,
326
+ ): Promise<DeliverableManifest> {
327
+ const name = basename(saveAs);
328
+ if (!name || name !== saveAs || name === '.' || name === '..')
329
+ throw new Error('artifact.save_as 只能是工作区根目录下的文件名');
330
+ const root = await realpath(getWorkingDir()).catch(() => getWorkingDir());
331
+ const destination = join(root, name);
332
+ const source = await realpath(sourcePath);
333
+ await stat(source);
334
+ const temporary = join(root, `.${name}.${randomUUID()}.partial`);
335
+ try {
336
+ await copyFile(source, temporary);
337
+ await rename(temporary, destination);
338
+ } finally {
339
+ await rm(temporary, { force: true }).catch(() => undefined);
340
+ }
341
+ const info = await stat(destination);
342
+ const relativePath = name;
343
+ return {
344
+ status: 'final',
345
+ summary: `已下载并提交 ${name}`,
346
+ files: [
347
+ {
348
+ relativePath,
349
+ sha256: await hashFile(destination),
350
+ mime:
351
+ DOWNLOAD_MIME_BY_EXT[extname(name).toLowerCase()] ??
352
+ 'application/octet-stream',
353
+ size: info.size,
354
+ modifiedAt: info.mtimeMs,
355
+ verification: 'not_run',
356
+ purpose: 'deliverable',
357
+ },
358
+ ],
359
+ warnings: [],
360
+ };
361
+ }
362
+
363
+ async function waitForDownloadedFile(
364
+ directory: string,
365
+ item: DownloadItem,
366
+ timeout: number,
367
+ signal?: AbortSignal,
368
+ ): Promise<string> {
369
+ const deadline = Date.now() + timeout;
370
+ let previousPath = '';
371
+ let previousSize = -1;
372
+ let stableSamples = 0;
373
+ while (Date.now() < deadline) {
374
+ const entries = await readdir(directory, { withFileTypes: true }).catch(
375
+ () => [],
376
+ );
377
+ const candidates = await Promise.all(
378
+ entries
379
+ .filter(
380
+ (entry) => entry.isFile() && !entry.name.endsWith('.crdownload'),
381
+ )
382
+ .map(async (entry) => {
383
+ const path = join(directory, entry.name);
384
+ const info = await stat(path).catch(() => null);
385
+ return info
386
+ ? { path, name: entry.name, mtimeMs: info.mtimeMs, size: info.size }
387
+ : null;
388
+ }),
389
+ );
390
+ const usable = candidates
391
+ .filter(
392
+ (candidate): candidate is NonNullable<typeof candidate> =>
393
+ candidate !== null,
394
+ )
395
+ .filter((candidate) => candidate.mtimeMs >= item.startedAt - 1_000)
396
+ .sort((a, b) => b.mtimeMs - a.mtimeMs);
397
+ const exact = usable.find((candidate) => candidate.name === item.filename);
398
+ const candidate = exact ?? usable[0];
399
+ if (candidate) {
400
+ const expectedSize = item.totalBytes;
401
+ const sizeMatches =
402
+ expectedSize === undefined ||
403
+ expectedSize <= 0 ||
404
+ candidate.size === expectedSize;
405
+ if (sizeMatches) {
406
+ if (candidate.path === previousPath && candidate.size === previousSize)
407
+ stableSamples += 1;
408
+ else {
409
+ previousPath = candidate.path;
410
+ previousSize = candidate.size;
411
+ stableSamples = 1;
412
+ }
413
+ if (stableSamples >= 2) return candidate.path;
414
+ }
415
+ }
416
+ await sleepWithSignal(100, signal);
417
+ }
418
+ throw new Error(`下载已完成但文件未稳定落盘:${item.filename}`);
419
+ }
420
+
421
+ async function materializeExportArtifact(
422
+ value: unknown,
423
+ format: string,
424
+ saveAs: string,
425
+ ): Promise<DeliverableManifest> {
426
+ const name = basename(saveAs);
427
+ if (!name || name !== saveAs || name === '.' || name === '..')
428
+ throw new Error('artifact.save_as 只能是工作区根目录下的文件名');
429
+ const root = await realpath(getWorkingDir()).catch(() => getWorkingDir());
430
+ const destination = join(root, name);
431
+ let bytes: Buffer;
432
+ if (
433
+ value &&
434
+ typeof value === 'object' &&
435
+ '__browser_pdf_base64' in value &&
436
+ typeof (value as { __browser_pdf_base64?: unknown })
437
+ .__browser_pdf_base64 === 'string'
438
+ ) {
439
+ bytes = Buffer.from(
440
+ (value as { __browser_pdf_base64: string }).__browser_pdf_base64,
441
+ 'base64',
442
+ );
443
+ } else if (
444
+ value &&
445
+ typeof value === 'object' &&
446
+ '__browser_binary_base64' in value &&
447
+ typeof (value as { __browser_binary_base64?: unknown })
448
+ .__browser_binary_base64 === 'string'
449
+ ) {
450
+ bytes = Buffer.from(
451
+ (value as { __browser_binary_base64: string }).__browser_binary_base64,
452
+ 'base64',
453
+ );
454
+ } else if (format === 'html' && value && typeof value === 'object') {
455
+ bytes = Buffer.from(
456
+ String((value as { html?: unknown }).html ?? ''),
457
+ 'utf8',
458
+ );
459
+ } else if (format === 'json') {
460
+ bytes = Buffer.from(`${JSON.stringify(value, null, 2)}\n`, 'utf8');
461
+ } else if (typeof value === 'string') {
462
+ bytes = Buffer.from(value.endsWith('\n') ? value : `${value}\n`, 'utf8');
463
+ } else if (value && typeof value === 'object' && 'text' in value) {
464
+ bytes = Buffer.from(
465
+ `${String((value as { text?: unknown }).text ?? '')}\n`,
466
+ 'utf8',
467
+ );
468
+ } else {
469
+ bytes = Buffer.from(`${JSON.stringify(value, null, 2)}\n`, 'utf8');
470
+ }
471
+ const temporary = join(root, `.${name}.${randomUUID()}.partial`);
472
+ try {
473
+ await writeFile(temporary, bytes, { mode: 0o600 });
474
+ await rename(temporary, destination);
475
+ } finally {
476
+ await rm(temporary, { force: true }).catch(() => undefined);
477
+ }
478
+ const info = await stat(destination);
479
+ return {
480
+ status: 'final',
481
+ summary: `已导出并提交 ${name}`,
482
+ files: [
483
+ {
484
+ relativePath: name,
485
+ sha256: await hashFile(destination),
486
+ mime:
487
+ DOWNLOAD_MIME_BY_EXT[extname(name).toLowerCase()] ??
488
+ 'application/octet-stream',
489
+ size: info.size,
490
+ modifiedAt: info.mtimeMs,
491
+ verification: 'not_run',
492
+ purpose: 'deliverable',
493
+ },
494
+ ],
495
+ warnings: [],
496
+ };
497
+ }
498
+
499
+ export function mergeArtifactManifests(
500
+ current: DeliverableManifest | undefined,
501
+ next: DeliverableManifest,
502
+ ): DeliverableManifest {
503
+ if (!current) return next;
504
+ const files = new Map<string, DeliverableManifest['files'][number]>();
505
+ for (const file of [...current.files, ...next.files]) {
506
+ // A repeated save_as represents the latest version of the same artifact;
507
+ // distinct paths remain distinct formal deliverables.
508
+ files.set(file.relativePath, file);
509
+ }
510
+ const links = new Map<
511
+ string,
512
+ NonNullable<DeliverableManifest['links']>[number]
513
+ >();
514
+ for (const link of [...(current.links ?? []), ...(next.links ?? [])])
515
+ links.set(link.url, link);
516
+ const allFiles = [...files.values()];
517
+ const warnings = [...new Set([...current.warnings, ...next.warnings])];
518
+ return {
519
+ status:
520
+ current.status === 'final' && next.status === 'final' && !warnings.length
521
+ ? 'final'
522
+ : 'draft',
523
+ summary: `已提交 ${allFiles.length} 个成果文件`,
524
+ files: allFiles,
525
+ ...(links.size ? { links: [...links.values()] } : {}),
526
+ warnings,
527
+ };
528
+ }
529
+
530
+ async function evaluate(
531
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
532
+ expression: string,
533
+ signal?: AbortSignal,
534
+ frameId?: string,
535
+ ): Promise<unknown> {
536
+ if (frameId) {
537
+ const result = (await cdp.evaluateInFrame(frameId, expression, signal)) as {
538
+ result?: { value?: unknown; description?: string };
539
+ };
540
+ return result.result?.value ?? result.result?.description;
541
+ }
542
+ const result = (await cdp.send(
543
+ 'Runtime.evaluate',
544
+ {
545
+ expression,
546
+ returnByValue: true,
547
+ awaitPromise: true,
548
+ },
549
+ undefined,
550
+ 30_000,
551
+ signal,
552
+ )) as {
553
+ result?: { value?: unknown; description?: string };
554
+ };
555
+ return result.result?.value ?? result.result?.description;
556
+ }
557
+ async function waitFor(
558
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
559
+ predicate: (event: CdpEvent) => boolean,
560
+ timeout: number,
561
+ afterSequence = 0,
562
+ signal?: AbortSignal,
563
+ ): Promise<CdpEvent> {
564
+ const pageEvent = (event: CdpEvent): boolean => {
565
+ // Browser.* events are browser-scoped and intentionally remain visible
566
+ // (downloads do not carry a page session in all Chromium versions).
567
+ // Page/DOM/Input/etc. events must belong to this connection's attached
568
+ // page, otherwise another tab can satisfy a navigation or dialog wait.
569
+ const pageDomain =
570
+ /^(Accessibility|DOM|Input|Log|Network|Page|Performance|Runtime|Security)\./;
571
+ return (
572
+ !pageDomain.test(event.method) ||
573
+ event.sessionId === cdp.attachedPageSessionId
574
+ );
575
+ };
576
+ return new Promise((resolve, reject) => {
577
+ if (signal?.aborted) {
578
+ reject(
579
+ signal.reason instanceof Error
580
+ ? signal.reason
581
+ : new Error('Browser event wait aborted'),
582
+ );
583
+ return;
584
+ }
585
+ const existing = cdp
586
+ .read(afterSequence)
587
+ .find((event) => pageEvent(event) && predicate(event));
588
+ if (existing) return resolve(existing);
589
+ let settled = false;
590
+ let off: () => void = () => {};
591
+ const cleanup = () => {
592
+ clearTimeout(timer);
593
+ off();
594
+ signal?.removeEventListener('abort', onAbort);
595
+ };
596
+ const finish = (callback: () => void) => {
597
+ if (settled) return;
598
+ settled = true;
599
+ cleanup();
600
+ callback();
601
+ };
602
+ const onAbort = () =>
603
+ finish(() =>
604
+ reject(
605
+ signal?.reason instanceof Error
606
+ ? signal.reason
607
+ : new Error('Browser event wait aborted'),
608
+ ),
609
+ );
610
+ const timer = setTimeout(
611
+ () => finish(() => reject(new Error('Browser event wait timed out'))),
612
+ timeout,
613
+ );
614
+ off = cdp.subscribe((event) => {
615
+ if (pageEvent(event) && predicate(event)) finish(() => resolve(event));
616
+ });
617
+ signal?.addEventListener('abort', onAbort, { once: true });
618
+ // Close the tiny read/subscribe race without changing the event cursor contract.
619
+ const raced = cdp
620
+ .read(afterSequence)
621
+ .find((event) => pageEvent(event) && predicate(event));
622
+ if (raced) finish(() => resolve(raced));
623
+ });
624
+ }
625
+
626
+ // Some agent-browser/CDP endpoints expose Runtime/Page but not the optional
627
+ // Accessibility domain. Keep a deterministic DOM-semantic fallback so a
628
+ // missing AX domain does not make an otherwise usable supervised browser fail.
629
+ const domSemanticSnapshotExpression = `(() => Array.from(document.querySelectorAll('h1,h2,h3,h4,h5,h6,p,li,a,button,input,textarea,select,[role],[contenteditable="true"]')).map((el,index) => { const r=el.getBoundingClientRect(); const s=getComputedStyle(el); const visible=!!(r.width||r.height)&&s.display!=='none'&&s.visibility!=='hidden'&&s.pointerEvents!=='none'; const tag=el.tagName.toLowerCase(); const explicit=el.getAttribute('role'); const role=explicit||(['button','a','textarea','select'].includes(tag)?({button:'button',a:'link',textarea:'textbox',select:'combobox'})[tag]:(tag==='input'?(['checkbox','radio','button','submit','reset'].includes((el.getAttribute('type')||'text').toLowerCase())?(el.getAttribute('type')||'text').toLowerCase():'textbox'):tag)); const name=(el.getAttribute('aria-label')||el.textContent||el.getAttribute('placeholder')||'').trim().replace(/\\s+/g,' ').slice(0,240); const value=el instanceof HTMLInputElement||el instanceof HTMLTextAreaElement||el instanceof HTMLSelectElement?el.value:undefined; return {node_id:index,role,text:name,value,visible,x:r.x,y:r.y,width:r.width,height:r.height}; }).filter((node) => node.visible).slice(0,500))()`;
630
+
631
+ async function readBrowserAxTree(
632
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
633
+ signal?: AbortSignal,
634
+ ): Promise<BrowserAxNode[]> {
635
+ try {
636
+ return normalizeAxTree(
637
+ await cdp.send(
638
+ 'Accessibility.getFullAXTree',
639
+ {},
640
+ undefined,
641
+ 30_000,
642
+ signal,
643
+ ),
644
+ );
645
+ } catch (error) {
646
+ const message = error instanceof Error ? error.message : String(error);
647
+ if (!/-32601|not found|unknown method/i.test(message)) throw error;
648
+ const domNodes = await evaluate(cdp, domSemanticSnapshotExpression, signal);
649
+ if (!Array.isArray(domNodes)) return [];
650
+ return normalizeAxTree({
651
+ nodes: domNodes.map((value) => {
652
+ const node =
653
+ value && typeof value === 'object'
654
+ ? (value as Record<string, unknown>)
655
+ : {};
656
+ const number = (key: string) =>
657
+ typeof node[key] === 'number' ? node[key] : undefined;
658
+ return {
659
+ nodeId: `dom-${String(node.node_id ?? '')}`,
660
+ role: { value: String(node.role ?? 'generic') },
661
+ name: { value: String(node.text ?? '') },
662
+ ...(node.value !== undefined
663
+ ? { properties: [{ name: 'value', value: { value: node.value } }] }
664
+ : {}),
665
+ bounds: {
666
+ x: number('x') ?? 0,
667
+ y: number('y') ?? 0,
668
+ width: number('width') ?? 0,
669
+ height: number('height') ?? 0,
670
+ },
671
+ };
672
+ }),
673
+ });
674
+ }
675
+ }
676
+
677
+ async function snapshot(
678
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
679
+ key: string,
680
+ mode: Input['final_snapshot'],
681
+ signal?: AbortSignal,
682
+ ): Promise<{
683
+ mode: 'full' | 'diff';
684
+ snapshot_id: string;
685
+ base_snapshot_id?: string;
686
+ page_revision: number;
687
+ generation: number;
688
+ tree: string;
689
+ frames: FrameInfo[];
690
+ }> {
691
+ const now = Date.now();
692
+ for (const [staleKey, stale] of snapshots) {
693
+ if (now - stale.updatedAt > SNAPSHOT_TTL_MS) snapshots.delete(staleKey);
694
+ }
695
+ const tree = await readBrowserAxTree(cdp, signal);
696
+ const url = String(
697
+ await evaluate(cdp, 'location.href', signal).catch(() => ''),
698
+ );
699
+ const frameResult = (await cdp
700
+ .send('Page.getFrameTree', {}, undefined, 30_000, signal)
701
+ .catch(() => ({}))) as {
702
+ frameTree?: unknown;
703
+ };
704
+ const frames: FrameInfo[] = normalizeFrameTree(frameResult.frameTree);
705
+ const prior = snapshots.get(key);
706
+ const revision = (prior?.revision ?? 0) + 1;
707
+ const id = randomUUID();
708
+ const generation =
709
+ prior && prior.url !== url
710
+ ? prior.generation + 1
711
+ : (prior?.generation ?? 1);
712
+ const samePageNodes = prior && prior.url === url ? prior.nodes : undefined;
713
+ const diff = diffAxTree(
714
+ samePageNodes,
715
+ tree,
716
+ generation,
717
+ revision,
718
+ mode === 'full' || (!samePageNodes && mode !== 'diff'),
719
+ );
720
+ snapshots.set(key, {
721
+ id,
722
+ revision,
723
+ generation,
724
+ url,
725
+ text: JSON.stringify(tree),
726
+ nodes: new Map(tree.map((node) => [node.id, node])),
727
+ updatedAt: now,
728
+ });
729
+ if (snapshots.size > MAX_SNAPSHOTS) {
730
+ const stale = [...snapshots.entries()]
731
+ .sort((left, right) => left[1].updatedAt - right[1].updatedAt)
732
+ .slice(0, snapshots.size - MAX_SNAPSHOTS);
733
+ for (const [staleKey] of stale) snapshots.delete(staleKey);
734
+ }
735
+ const body =
736
+ mode === 'full' || !samePageNodes ? { nodes: tree, ...diff } : diff;
737
+ const compact = JSON.stringify(body);
738
+ return {
739
+ mode: diff.full ? 'full' : 'diff',
740
+ snapshot_id: id,
741
+ ...(prior ? { base_snapshot_id: prior.id } : {}),
742
+ page_revision: revision,
743
+ generation,
744
+ tree: (mode === 'full' || !prior ? conciseAxTree(tree) : compact).slice(
745
+ 0,
746
+ DEFAULT_MAX_RESULT_SIZE_CHARS,
747
+ ),
748
+ frames,
749
+ };
750
+ }
751
+
752
+ async function resolveAxLocator(
753
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
754
+ locator: BrowserLocator | undefined,
755
+ snapshotKey: string,
756
+ signal?: AbortSignal,
757
+ ): Promise<BrowserLocator | undefined> {
758
+ if (!locator || locator.by !== 'ax') return locator;
759
+ const current = await readBrowserAxTree(cdp, signal);
760
+ const prior = snapshots.get(snapshotKey);
761
+ if (locator.generation !== undefined && !prior) {
762
+ throw new Error(
763
+ 'AX 引用缺少对应 snapshot;请先 snapshot 后再使用 generation',
764
+ );
765
+ }
766
+ if (
767
+ locator.generation !== undefined &&
768
+ prior &&
769
+ locator.generation !== prior.generation
770
+ ) {
771
+ throw new Error(
772
+ `AX 引用已过期:引用 generation=${locator.generation},当前页面 generation=${prior.generation};请重新 snapshot`,
773
+ );
774
+ }
775
+ if (locator.generation !== undefined && prior) {
776
+ const currentUrl = String(
777
+ await evaluate(cdp, 'location.href', signal).catch(() => ''),
778
+ );
779
+ if (currentUrl !== prior.url) {
780
+ throw new Error(
781
+ `AX 引用已过期:页面已从 ${prior.url} 导航到 ${currentUrl};请重新 snapshot`,
782
+ );
783
+ }
784
+ }
785
+ const indexMatch = /^index:(\d+)$/.exec(locator.value);
786
+ const numeric = indexMatch ? Number(indexMatch[1]) : Number(locator.value);
787
+ const node =
788
+ current.find((item) => item.id === locator.value) ??
789
+ (Number.isInteger(numeric) && numeric >= 0 && numeric < current.length
790
+ ? current[numeric]
791
+ : undefined);
792
+ if (!node)
793
+ throw new Error(`AX 引用未找到:${locator.value};请重新 snapshot`);
794
+ if (!node.name && !node.role)
795
+ throw new Error(`AX 引用没有可执行的语义信息:${locator.value}`);
796
+ const resolved: BrowserLocator = node.name
797
+ ? {
798
+ by: 'role',
799
+ value: node.role ?? 'generic',
800
+ name: node.name,
801
+ exact: true,
802
+ strict: true,
803
+ }
804
+ : { by: 'text', value: node.value ?? '', exact: true, strict: true };
805
+ const framePath = locator.frame ?? locator.framePath ?? locator.frame_path;
806
+ const frameId = locator.frameId ?? locator.frame_id;
807
+ return {
808
+ ...resolved,
809
+ ...(framePath?.length ? { frame: framePath } : {}),
810
+ ...(frameId ? { frameId } : {}),
811
+ };
812
+ }
813
+
814
+ function eventMatchesCondition(
815
+ condition: BrowserCondition,
816
+ event: CdpEvent,
817
+ ): boolean {
818
+ if (condition.type === 'download') {
819
+ return (
820
+ event.method === 'Browser.downloadWillBegin' &&
821
+ (!condition.filename ||
822
+ String(event.params.suggestedFilename ?? '') === condition.filename)
823
+ );
824
+ }
825
+ if (condition.type === 'new-tab') {
826
+ return (
827
+ event.method === 'Target.targetCreated' &&
828
+ (!condition.url || JSON.stringify(event.params).includes(condition.url))
829
+ );
830
+ }
831
+ if (condition.type === 'dialog') {
832
+ return (
833
+ event.method === 'Page.javascriptDialogOpening' &&
834
+ (!condition.dialogType ||
835
+ String(event.params.type ?? '') === condition.dialogType)
836
+ );
837
+ }
838
+ return false;
839
+ }
840
+
841
+ const visibleDomExpression = `(() => Array.from(document.querySelectorAll('a,button,input,textarea,select,[role],[contenteditable="true"]')).map((el, index) => { const r=el.getBoundingClientRect(); const s=getComputedStyle(el); const visible=!!(r.width||r.height)&&s.display!=='none'&&s.visibility!=='hidden'&&s.pointerEvents!=='none'; const label=el.labels?.length?Array.from(el.labels).map((item)=>String(item.textContent||'').trim()).filter(Boolean).join(' '):undefined; const role=el.getAttribute('role')||el.tagName.toLowerCase(); const editable=el instanceof HTMLInputElement||el instanceof HTMLTextAreaElement||el.isContentEditable; const clickable=el instanceof HTMLButtonElement||el instanceof HTMLAnchorElement||role==='button'||typeof el.onclick==='function'; return {node_id:index,tag:el.tagName.toLowerCase(),role,text:(el.innerText||el.getAttribute('aria-label')||label||el.getAttribute('placeholder')||'').trim().slice(0,240),label,placeholder:el.getAttribute('placeholder')||undefined,testId:el.getAttribute('data-testid')||undefined,cssCandidates:[el.id?'#'+CSS.escape(el.id):'',el.getAttribute('data-testid')?'[data-testid="'+CSS.escape(el.getAttribute('data-testid'))+'"]':''].filter(Boolean),framePath:[],x:r.x,y:r.y,width:r.width,height:r.height,visible,enabled:!el.disabled,editable,clickable,confidence:el.id||el.getAttribute('data-testid')?'high':'medium'}; }).filter((node) => node.visible).slice(0,500))()`;
842
+ export const visibleDomTarget = (nodeId: number) =>
843
+ `(() => { const nodes=${visibleDomExpression}; const n=nodes[${nodeId}]; if(!n) throw new Error('visible DOM node not found'); const all=Array.from(document.querySelectorAll('a,button,input,textarea,select,[role],[contenteditable="true"]')); const e=all[n.node_id]; if(!e) throw new Error('visible DOM node is stale'); return e; })()`;
844
+
845
+ async function waitForCondition(
846
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
847
+ condition: BrowserCondition,
848
+ timeout: number,
849
+ baseline?: unknown,
850
+ signal?: AbortSignal,
851
+ ): Promise<unknown> {
852
+ const deadline = Date.now() + timeout;
853
+ const script = conditionToScript(condition, (locator, selector) =>
854
+ buildTargetExpression(locator, selector),
855
+ );
856
+ let lastValue: unknown;
857
+ while (Date.now() < deadline) {
858
+ if (signal?.aborted)
859
+ throw signal.reason instanceof Error
860
+ ? signal.reason
861
+ : new Error('Browser condition wait aborted');
862
+ const challenge = await readBrowserChallenge(cdp, signal);
863
+ if (challenge) throw new Error(`页面需要人工验证:${challenge.summary}`);
864
+ const value = await evaluate(cdp, script, signal);
865
+ lastValue = value;
866
+ if (condition.type === 'page-change' ? value !== baseline : value === true)
867
+ return value;
868
+ await new Promise<void>((resolve, reject) => {
869
+ const timer = setTimeout(() => {
870
+ signal?.removeEventListener('abort', onAbort);
871
+ resolve();
872
+ }, 100);
873
+ const onAbort = () => {
874
+ clearTimeout(timer);
875
+ reject(
876
+ signal?.reason instanceof Error
877
+ ? signal.reason
878
+ : new Error('Browser condition wait aborted'),
879
+ );
880
+ };
881
+ if (signal?.aborted) onAbort();
882
+ else signal?.addEventListener('abort', onAbort, { once: true });
883
+ });
884
+ }
885
+ throw new Error(
886
+ `Browser condition timed out: ${condition.type} (last=${JSON.stringify(lastValue)})`,
887
+ );
888
+ }
889
+
890
+ async function readBrowserChallenge(
891
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
892
+ signal?: AbortSignal,
893
+ ): Promise<ReturnType<typeof detectBrowserChallenge>> {
894
+ const text = String(
895
+ await evaluate(
896
+ cdp,
897
+ `location.href + '\\n' + (document.body?.innerText || '').slice(0, 20000)`,
898
+ signal,
899
+ ).catch(() => ''),
900
+ );
901
+ return detectBrowserChallenge(text);
902
+ }
903
+
904
+ async function waitForBrowserChallenge(
905
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
906
+ timeout: number,
907
+ signal?: AbortSignal,
908
+ ): Promise<NonNullable<ReturnType<typeof detectBrowserChallenge>> | null> {
909
+ const deadline = Date.now() + timeout;
910
+ while (Date.now() < deadline && !signal?.aborted) {
911
+ const challenge = await readBrowserChallenge(cdp, signal);
912
+ if (challenge) return challenge;
913
+ await sleepWithSignal(250, signal).catch(() => undefined);
914
+ }
915
+ return null;
916
+ }
917
+
918
+ async function waitForEventOrBrowserChallenge(
919
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
920
+ predicate: (event: CdpEvent) => boolean,
921
+ timeout: number,
922
+ afterSequence = 0,
923
+ signal?: AbortSignal,
924
+ ): Promise<CdpEvent> {
925
+ const challengeAbort = new AbortController();
926
+ const challengeRace = waitForBrowserChallenge(
927
+ cdp,
928
+ timeout,
929
+ challengeAbort.signal,
930
+ ).then((challenge) => {
931
+ if (challenge) throw new Error(`页面需要人工验证:${challenge.summary}`);
932
+ return new Promise<never>(() => undefined);
933
+ });
934
+ try {
935
+ return await Promise.race([
936
+ waitFor(cdp, predicate, timeout, afterSequence, signal),
937
+ challengeRace,
938
+ ]);
939
+ } finally {
940
+ challengeAbort.abort();
941
+ }
942
+ }
943
+
944
+ type BrowserPoint = { x: number; y: number };
945
+
946
+ async function elementPoint(
947
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
948
+ target: string,
949
+ framePath: BrowserLocator[] = [],
950
+ signal?: AbortSignal,
951
+ frameId?: string,
952
+ ): Promise<BrowserPoint> {
953
+ if (frameId) {
954
+ const value = await evaluate(
955
+ cdp,
956
+ `(() => { const e=${target}; e.scrollIntoView({block:'center',inline:'center'}); const r=e.getBoundingClientRect(); const s=getComputedStyle(e); if (!r.width || !r.height || s.visibility==='hidden' || s.pointerEvents==='none') throw new Error('Locator matched a non-actionable element'); return {x:r.left+r.width/2,y:r.top+r.height/2}; })()`,
957
+ signal,
958
+ frameId,
959
+ );
960
+ if (!value || typeof value !== 'object')
961
+ throw new Error('无法取得浏览器元素坐标');
962
+ const point = value as { x?: unknown; y?: unknown };
963
+ if (typeof point.x !== 'number' || typeof point.y !== 'number')
964
+ throw new Error('浏览器元素坐标无效');
965
+ const offset = await cdp.frameOwnerOffset(frameId, signal);
966
+ return { x: point.x + offset.x, y: point.y + offset.y };
967
+ }
968
+ let frameOffsets = '';
969
+ let frameRoot = 'document';
970
+ for (const [index, frame] of framePath.entries()) {
971
+ const frameExpression = buildLocatorExpression(frame, undefined, frameRoot);
972
+ frameOffsets += `const __frame${index}=(${frameExpression})[0]; if(!__frame${index}||!__frame${index}.contentDocument) throw new Error('frameLocator could not resolve a same-origin iframe'); const __frameRect${index}=__frame${index}.getBoundingClientRect(); __offsetX+=__frameRect${index}.left; __offsetY+=__frameRect${index}.top; __frameRoot=__frame${index}.contentDocument;`;
973
+ frameRoot = '__frameRoot';
974
+ }
975
+ const value = await evaluate(
976
+ cdp,
977
+ `(() => { let __offsetX=0,__offsetY=0,__frameRoot=document; ${frameOffsets} const e=${target}; e.scrollIntoView({block:'center',inline:'center'}); const r=e.getBoundingClientRect(); const s=getComputedStyle(e); if (!r.width || !r.height || s.visibility==='hidden' || s.pointerEvents==='none') throw new Error('Locator matched a non-actionable element'); return {x:r.left+r.width/2+__offsetX,y:r.top+r.height/2+__offsetY}; })()`,
978
+ signal,
979
+ );
980
+ if (!value || typeof value !== 'object')
981
+ throw new Error('无法取得浏览器元素坐标');
982
+ const point = value as { x?: unknown; y?: unknown };
983
+ if (typeof point.x !== 'number' || typeof point.y !== 'number')
984
+ throw new Error('浏览器元素坐标无效');
985
+ return { x: point.x, y: point.y };
986
+ }
987
+
988
+ async function trustedClick(
989
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
990
+ target: string,
991
+ framePath: BrowserLocator[] = [],
992
+ signal?: AbortSignal,
993
+ clickCount = 1,
994
+ frameId?: string,
995
+ ): Promise<void> {
996
+ const point = await elementPoint(cdp, target, framePath, signal, frameId);
997
+ await cdp.send(
998
+ 'Input.dispatchMouseEvent',
999
+ {
1000
+ type: 'mouseMoved',
1001
+ x: point.x,
1002
+ y: point.y,
1003
+ },
1004
+ undefined,
1005
+ 30_000,
1006
+ signal,
1007
+ );
1008
+ await cdp.send(
1009
+ 'Input.dispatchMouseEvent',
1010
+ {
1011
+ type: 'mousePressed',
1012
+ x: point.x,
1013
+ y: point.y,
1014
+ button: 'left',
1015
+ buttons: 1,
1016
+ clickCount,
1017
+ },
1018
+ undefined,
1019
+ 30_000,
1020
+ signal,
1021
+ );
1022
+ await cdp.send(
1023
+ 'Input.dispatchMouseEvent',
1024
+ {
1025
+ type: 'mouseReleased',
1026
+ x: point.x,
1027
+ y: point.y,
1028
+ button: 'left',
1029
+ buttons: 0,
1030
+ clickCount,
1031
+ },
1032
+ undefined,
1033
+ 30_000,
1034
+ signal,
1035
+ );
1036
+ }
1037
+
1038
+ async function focusTarget(
1039
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
1040
+ target: string,
1041
+ signal?: AbortSignal,
1042
+ frameId?: string,
1043
+ ): Promise<void> {
1044
+ await evaluate(
1045
+ cdp,
1046
+ `(()=>{const e=${target};e.scrollIntoView({block:'center',inline:'nearest'});e.focus();})()`,
1047
+ signal,
1048
+ frameId,
1049
+ );
1050
+ }
1051
+
1052
+ async function insertText(
1053
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
1054
+ target: string,
1055
+ value: string,
1056
+ replace: boolean,
1057
+ signal?: AbortSignal,
1058
+ frameId?: string,
1059
+ ): Promise<void> {
1060
+ await focusTarget(cdp, target, signal, frameId);
1061
+ if (replace) {
1062
+ const mac = await evaluate(
1063
+ cdp,
1064
+ `navigator.platform.includes('Mac')`,
1065
+ signal,
1066
+ frameId,
1067
+ );
1068
+ const modifiers = mac === true ? 4 : 2;
1069
+ await cdp.send(
1070
+ 'Input.dispatchKeyEvent',
1071
+ {
1072
+ type: 'keyDown',
1073
+ key: 'a',
1074
+ code: 'KeyA',
1075
+ modifiers,
1076
+ },
1077
+ undefined,
1078
+ 30_000,
1079
+ signal,
1080
+ );
1081
+ await cdp.send(
1082
+ 'Input.dispatchKeyEvent',
1083
+ {
1084
+ type: 'keyUp',
1085
+ key: 'a',
1086
+ code: 'KeyA',
1087
+ modifiers,
1088
+ },
1089
+ undefined,
1090
+ 30_000,
1091
+ signal,
1092
+ );
1093
+ }
1094
+ await cdp.send(
1095
+ 'Input.insertText',
1096
+ { text: value },
1097
+ undefined,
1098
+ 30_000,
1099
+ signal,
1100
+ );
1101
+ }
1102
+
1103
+ function sleepWithSignal(ms: number, signal?: AbortSignal): Promise<void> {
1104
+ return new Promise((resolve, reject) => {
1105
+ if (signal?.aborted) {
1106
+ reject(
1107
+ signal.reason instanceof Error
1108
+ ? signal.reason
1109
+ : new Error('Browser step aborted'),
1110
+ );
1111
+ return;
1112
+ }
1113
+ const timer = setTimeout(() => {
1114
+ signal?.removeEventListener('abort', onAbort);
1115
+ resolve();
1116
+ }, ms);
1117
+ const onAbort = () => {
1118
+ clearTimeout(timer);
1119
+ reject(
1120
+ signal?.reason instanceof Error
1121
+ ? signal.reason
1122
+ : new Error('Browser step aborted'),
1123
+ );
1124
+ };
1125
+ signal?.addEventListener('abort', onAbort, { once: true });
1126
+ });
1127
+ }
1128
+
1129
+ async function resumeBatchAfterHuman(
1130
+ stream: ReturnType<typeof getBrowserStream>,
1131
+ ctx: ToolCallContext,
1132
+ signal: AbortSignal | undefined,
1133
+ summary: string,
1134
+ ): Promise<'resumed' | 'stopped'> {
1135
+ ctx.reportProgress?.(`${summary},等待你在浏览器中处理`);
1136
+ const resolution = await stream.waitForHuman(signal);
1137
+ if (resolution === 'resumed')
1138
+ ctx.reportProgress?.('人工处理完成,正在重新读取页面状态并继续批量任务');
1139
+ return resolution;
1140
+ }
1141
+
1142
+ async function executeStep(
1143
+ cdp: Awaited<ReturnType<typeof getBrowserAutomation>>,
1144
+ step: Step,
1145
+ timeout: number,
1146
+ afterSequence = 0,
1147
+ snapshotKey = 'step',
1148
+ signal?: AbortSignal,
1149
+ ): Promise<unknown> {
1150
+ await cdp
1151
+ .send('Page.enable', {}, undefined, 30_000, signal)
1152
+ .catch(() => undefined);
1153
+ const resolvedLocator = await resolveAxLocator(
1154
+ cdp,
1155
+ step.locator,
1156
+ snapshotKey,
1157
+ signal,
1158
+ );
1159
+ const frameId = resolvedLocator?.frameId ?? resolvedLocator?.frame_id;
1160
+ const targetLocator: BrowserLocator | undefined =
1161
+ resolvedLocator && frameId
1162
+ ? {
1163
+ ...resolvedLocator,
1164
+ frame: undefined,
1165
+ framePath: undefined,
1166
+ frame_path: undefined,
1167
+ }
1168
+ : resolvedLocator;
1169
+ const target = () => {
1170
+ if (targetLocator && step.selector) {
1171
+ const semantic = buildTargetExpression(targetLocator, undefined);
1172
+ const cssFallback = buildTargetExpression(undefined, step.selector);
1173
+ return `(() => { try { return ${semantic}; } catch { return ${cssFallback}; } })()`;
1174
+ }
1175
+ return buildTargetExpression(targetLocator, step.selector);
1176
+ };
1177
+ switch (step.action) {
1178
+ case 'navigate':
1179
+ if (!step.url) throw new Error('navigate requires url');
1180
+ return cdp.send(
1181
+ 'Page.navigate',
1182
+ { url: step.url },
1183
+ undefined,
1184
+ 30_000,
1185
+ signal,
1186
+ );
1187
+ case 'click':
1188
+ return trustedClick(
1189
+ cdp,
1190
+ target(),
1191
+ resolvedLocator?.frame ??
1192
+ resolvedLocator?.framePath ??
1193
+ resolvedLocator?.frame_path ??
1194
+ [],
1195
+ signal,
1196
+ 1,
1197
+ frameId,
1198
+ );
1199
+ case 'doubleClick':
1200
+ return trustedClick(
1201
+ cdp,
1202
+ target(),
1203
+ resolvedLocator?.frame ??
1204
+ resolvedLocator?.framePath ??
1205
+ resolvedLocator?.frame_path ??
1206
+ [],
1207
+ signal,
1208
+ 2,
1209
+ frameId,
1210
+ );
1211
+ case 'fill':
1212
+ if (step.value === undefined) throw new Error('fill requires value');
1213
+ return insertText(cdp, target(), step.value, true, signal, frameId);
1214
+ case 'type':
1215
+ if (step.value === undefined) throw new Error('type requires value');
1216
+ return insertText(cdp, target(), step.value, false, signal, frameId);
1217
+ case 'hover': {
1218
+ const point = await elementPoint(
1219
+ cdp,
1220
+ target(),
1221
+ resolvedLocator?.frame ??
1222
+ resolvedLocator?.framePath ??
1223
+ resolvedLocator?.frame_path ??
1224
+ [],
1225
+ signal,
1226
+ frameId,
1227
+ );
1228
+ return cdp.send(
1229
+ 'Input.dispatchMouseEvent',
1230
+ {
1231
+ type: 'mouseMoved',
1232
+ x: point.x,
1233
+ y: point.y,
1234
+ },
1235
+ undefined,
1236
+ 30_000,
1237
+ signal,
1238
+ );
1239
+ }
1240
+ case 'press':
1241
+ if (!step.key) throw new Error('press requires key');
1242
+ if (step.locator || step.selector)
1243
+ await focusTarget(cdp, target(), signal, frameId);
1244
+ return cdp
1245
+ .send(
1246
+ 'Input.dispatchKeyEvent',
1247
+ { type: 'keyDown', key: step.key },
1248
+ undefined,
1249
+ 30_000,
1250
+ signal,
1251
+ )
1252
+ .then(() =>
1253
+ cdp.send(
1254
+ 'Input.dispatchKeyEvent',
1255
+ { type: 'keyUp', key: step.key },
1256
+ undefined,
1257
+ 30_000,
1258
+ signal,
1259
+ ),
1260
+ );
1261
+ case 'check':
1262
+ return evaluate(
1263
+ cdp,
1264
+ `Boolean(${target()}.checked)`,
1265
+ signal,
1266
+ frameId,
1267
+ ).then((checked) =>
1268
+ checked
1269
+ ? undefined
1270
+ : trustedClick(
1271
+ cdp,
1272
+ target(),
1273
+ resolvedLocator?.frame ??
1274
+ resolvedLocator?.framePath ??
1275
+ resolvedLocator?.frame_path ??
1276
+ [],
1277
+ signal,
1278
+ 1,
1279
+ frameId,
1280
+ ),
1281
+ );
1282
+ case 'uncheck':
1283
+ return evaluate(
1284
+ cdp,
1285
+ `Boolean(${target()}.checked)`,
1286
+ signal,
1287
+ frameId,
1288
+ ).then((checked) =>
1289
+ checked
1290
+ ? trustedClick(
1291
+ cdp,
1292
+ target(),
1293
+ resolvedLocator?.frame ??
1294
+ resolvedLocator?.framePath ??
1295
+ resolvedLocator?.frame_path ??
1296
+ [],
1297
+ signal,
1298
+ 1,
1299
+ frameId,
1300
+ )
1301
+ : undefined,
1302
+ );
1303
+ case 'setChecked':
1304
+ if (step.checked === undefined)
1305
+ throw new Error('setChecked requires checked');
1306
+ return evaluate(
1307
+ cdp,
1308
+ `Boolean(${target()}.checked)`,
1309
+ signal,
1310
+ frameId,
1311
+ ).then((checked) =>
1312
+ checked === step.checked
1313
+ ? undefined
1314
+ : trustedClick(
1315
+ cdp,
1316
+ target(),
1317
+ resolvedLocator?.frame ??
1318
+ resolvedLocator?.framePath ??
1319
+ resolvedLocator?.frame_path ??
1320
+ [],
1321
+ signal,
1322
+ 1,
1323
+ frameId,
1324
+ ),
1325
+ );
1326
+ case 'getState':
1327
+ return evaluate(
1328
+ cdp,
1329
+ `(()=>{const e=${target()};const s=getComputedStyle(e);return {visible:!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)&&s.visibility!=='hidden',enabled:!e.disabled,checked:!!e.checked}})()`,
1330
+ signal,
1331
+ frameId,
1332
+ );
1333
+ case 'drag': {
1334
+ const points = step.drag_path;
1335
+ if (!points?.length) throw new Error('drag requires drag_path');
1336
+ for (const [index, point] of points.entries())
1337
+ await cdp.send(
1338
+ 'Input.dispatchMouseEvent',
1339
+ {
1340
+ type:
1341
+ index === 0
1342
+ ? 'mousePressed'
1343
+ : index === points.length - 1
1344
+ ? 'mouseReleased'
1345
+ : 'mouseMoved',
1346
+ x: point.x,
1347
+ y: point.y,
1348
+ button: 'left',
1349
+ buttons: index === 0 ? 1 : index === points.length - 1 ? 0 : 1,
1350
+ },
1351
+ undefined,
1352
+ 30_000,
1353
+ signal,
1354
+ );
1355
+ return { points: points.length };
1356
+ }
1357
+ case 'wait':
1358
+ await sleepWithSignal(step.wait_ms ?? 250, signal);
1359
+ return { waited_ms: step.wait_ms ?? 250 };
1360
+ case 'waitForURL':
1361
+ if (!step.url) throw new Error('waitForURL requires url');
1362
+ return waitFor(
1363
+ cdp,
1364
+ (event) =>
1365
+ /Page\.frame(Navigated|StartedLoading)/.test(event.method) &&
1366
+ JSON.stringify(event.params).includes(step.url!),
1367
+ timeout,
1368
+ afterSequence,
1369
+ signal,
1370
+ );
1371
+ case 'waitForDownload':
1372
+ return waitFor(
1373
+ cdp,
1374
+ (event) =>
1375
+ event.method === 'Browser.downloadWillBegin' ||
1376
+ event.method === 'Browser.downloadProgress',
1377
+ timeout,
1378
+ afterSequence,
1379
+ signal,
1380
+ );
1381
+ case 'waitForFileChooser':
1382
+ return waitFor(
1383
+ cdp,
1384
+ (event) => event.method === 'Page.fileChooserOpened',
1385
+ timeout,
1386
+ afterSequence,
1387
+ signal,
1388
+ );
1389
+ case 'snapshot':
1390
+ return snapshot(cdp, snapshotKey, 'full', signal);
1391
+ case 'frameInfo':
1392
+ return normalizeFrameInfo(
1393
+ normalizeFrameTree(
1394
+ (
1395
+ (await cdp.send(
1396
+ 'Page.getFrameTree',
1397
+ {},
1398
+ undefined,
1399
+ 30_000,
1400
+ signal,
1401
+ )) as { frameTree?: unknown }
1402
+ ).frameTree,
1403
+ ),
1404
+ );
1405
+ case 'visibleDom':
1406
+ return evaluate(cdp, visibleDomExpression, signal, frameId);
1407
+ case 'clickNode':
1408
+ if (step.node_id === undefined)
1409
+ throw new Error('clickNode requires node_id');
1410
+ return trustedClick(cdp, visibleDomTarget(step.node_id), [], signal);
1411
+ case 'typeNode':
1412
+ if (step.node_id === undefined || step.value === undefined)
1413
+ throw new Error('typeNode requires node_id and value');
1414
+ return insertText(
1415
+ cdp,
1416
+ visibleDomTarget(step.node_id),
1417
+ step.value,
1418
+ false,
1419
+ signal,
1420
+ frameId,
1421
+ );
1422
+ case 'scrollNode':
1423
+ if (step.node_id === undefined)
1424
+ throw new Error('scrollNode requires node_id');
1425
+ return evaluate(
1426
+ cdp,
1427
+ `(()=>{const e=${visibleDomTarget(step.node_id)};e.scrollIntoView({block:'center',inline:'nearest'});return {node_id:${step.node_id}}})()`,
1428
+ signal,
1429
+ frameId,
1430
+ );
1431
+ case 'upload': {
1432
+ if (!step.files?.length) throw new Error('upload requires files');
1433
+ if (step.trigger_selector || step.trigger_locator) {
1434
+ const trigger = buildTargetExpression(
1435
+ step.trigger_locator,
1436
+ step.trigger_selector,
1437
+ );
1438
+ await trustedClick(cdp, trigger, [], signal);
1439
+ return { files: step.files, chooser_triggered: true };
1440
+ }
1441
+ if (!step.selector)
1442
+ throw new Error('upload requires selector or trigger_selector');
1443
+ const documentResult = (await cdp.send(
1444
+ 'DOM.getDocument',
1445
+ {
1446
+ depth: -1,
1447
+ },
1448
+ undefined,
1449
+ 30_000,
1450
+ signal,
1451
+ )) as {
1452
+ root?: { nodeId?: number };
1453
+ };
1454
+ if (!documentResult.root?.nodeId) throw new Error('无法取得 DOM 根节点');
1455
+ const found = (await cdp.send(
1456
+ 'DOM.querySelector',
1457
+ {
1458
+ nodeId: documentResult.root.nodeId,
1459
+ selector: step.selector,
1460
+ },
1461
+ undefined,
1462
+ 30_000,
1463
+ signal,
1464
+ )) as { nodeId?: number };
1465
+ if (!found.nodeId) throw new Error('upload selector matched no input');
1466
+ await cdp.send(
1467
+ 'DOM.setFileInputFiles',
1468
+ {
1469
+ files: step.files,
1470
+ nodeId: found.nodeId,
1471
+ },
1472
+ undefined,
1473
+ 30_000,
1474
+ signal,
1475
+ );
1476
+ return { files: step.files, uploaded: true };
1477
+ }
1478
+ case 'pageAssets':
1479
+ case 'listPageAssets':
1480
+ case 'bundlePageAssets': {
1481
+ const assets = await evaluate(cdp, PAGE_ASSETS_EXPRESSION, signal);
1482
+ if (!Array.isArray(assets)) throw new Error('页面资产清单格式无效');
1483
+ if (step.bundle_assets || step.action === 'bundlePageAssets') {
1484
+ if (!step.bundle_dir)
1485
+ throw new Error('bundle_assets requires bundle_dir');
1486
+ const bundle = await bundlePageAssets(
1487
+ assets as BrowserPageAsset[],
1488
+ step.bundle_dir,
1489
+ async (url) => {
1490
+ const fetched = await evaluate(
1491
+ cdp,
1492
+ browserAssetFetchExpression(url),
1493
+ signal,
1494
+ );
1495
+ if (
1496
+ !fetched ||
1497
+ typeof fetched !== 'object' ||
1498
+ (fetched as { ok?: unknown }).ok !== true ||
1499
+ typeof (fetched as { base64?: unknown }).base64 !== 'string'
1500
+ ) {
1501
+ const status =
1502
+ fetched && typeof fetched === 'object'
1503
+ ? String(
1504
+ (fetched as { status?: unknown }).status ?? 'unknown',
1505
+ )
1506
+ : 'unknown';
1507
+ throw new Error(`浏览器上下文抓取资源失败 HTTP ${status}`);
1508
+ }
1509
+ const value = fetched as {
1510
+ base64: string;
1511
+ type?: string;
1512
+ };
1513
+ return new Response(Buffer.from(value.base64, 'base64'), {
1514
+ status: 200,
1515
+ headers: value.type ? { 'content-type': value.type } : undefined,
1516
+ });
1517
+ },
1518
+ );
1519
+ return { assets, bundle };
1520
+ }
1521
+ return { assets };
1522
+ }
1523
+ case 'exportContent': {
1524
+ const format = step.export_format ?? 'page';
1525
+ if (format === 'pdf') {
1526
+ const pdf = (await cdp.send(
1527
+ 'Page.printToPDF',
1528
+ {
1529
+ printBackground: true,
1530
+ preferCSSPageSize: true,
1531
+ },
1532
+ undefined,
1533
+ 30_000,
1534
+ signal,
1535
+ )) as { data?: string };
1536
+ if (!pdf.data) throw new Error('浏览器未返回 PDF 内容');
1537
+ return { __browser_pdf_base64: pdf.data };
1538
+ }
1539
+ if (['docx', 'xlsx', 'csv', 'pptx'].includes(format)) {
1540
+ const exported = await evaluate(
1541
+ cdp,
1542
+ googleWorkspaceExportExpression(
1543
+ format as 'docx' | 'xlsx' | 'csv' | 'pptx',
1544
+ ),
1545
+ signal,
1546
+ );
1547
+ if (exported && typeof exported === 'object' && 'error' in exported)
1548
+ throw new Error(String((exported as { error?: unknown }).error));
1549
+ if (
1550
+ !exported ||
1551
+ typeof exported !== 'object' ||
1552
+ typeof (exported as { __browser_binary_base64?: unknown })
1553
+ .__browser_binary_base64 !== 'string'
1554
+ )
1555
+ throw new Error('Workspace 未返回可保存的文件内容');
1556
+ return exported;
1557
+ }
1558
+ const content = await evaluate(
1559
+ cdp,
1560
+ pageContentExpression(format, step.selector),
1561
+ signal,
1562
+ );
1563
+ if (!content || typeof content !== 'object')
1564
+ throw new Error('页面内容导出失败');
1565
+ if (format === 'markdown' || format === 'md')
1566
+ return markdownFromPageContent(content as BrowserPageContent);
1567
+ return content;
1568
+ }
1569
+ case 'readUrls': {
1570
+ if (!step.urls?.length) throw new Error('readUrls requires urls');
1571
+ return readUrlsInBackground(
1572
+ cdp,
1573
+ step.urls,
1574
+ timeout,
1575
+ step.content_type,
1576
+ step.concurrency,
1577
+ signal,
1578
+ );
1579
+ }
1580
+ case 'clipboardReadText':
1581
+ return evaluate(cdp, CLIPBOARD_READ_TEXT_EXPRESSION, signal);
1582
+ case 'clipboardWriteText':
1583
+ if (step.clipboard_text === undefined)
1584
+ throw new Error('clipboardWriteText requires clipboard_text');
1585
+ return evaluate(
1586
+ cdp,
1587
+ clipboardWriteTextExpression(step.clipboard_text),
1588
+ signal,
1589
+ );
1590
+ case 'clipboardRead':
1591
+ return evaluate(cdp, CLIPBOARD_READ_EXPRESSION, signal);
1592
+ case 'clipboardWrite':
1593
+ if (!step.clipboard_items?.length)
1594
+ throw new Error('clipboardWrite requires clipboard_items');
1595
+ return evaluate(
1596
+ cdp,
1597
+ clipboardWriteExpression(step.clipboard_items),
1598
+ signal,
1599
+ );
1600
+ }
1601
+ }
1602
+
1603
+ async function call(
1604
+ input: Input,
1605
+ signal?: AbortSignal,
1606
+ ctx: ToolCallContext = {},
1607
+ ): Promise<ToolResult> {
1608
+ // Tool runners normally apply the Zod default, but callWithContext is also
1609
+ // used by the local runtime and deterministic E2E harnesses directly. Keep
1610
+ // the execution core safe when that adapter is intentionally bypassed.
1611
+ const timeoutMs = input.timeout_ms ?? 120_000;
1612
+ const stopOnError = input.stop_on_error ?? true;
1613
+ const finalSnapshotMode = input.final_snapshot ?? 'auto';
1614
+ const captureMode = input.capture ?? 'state';
1615
+ if (!(await configService.browser()).enabled)
1616
+ return { ok: false, error: '内置浏览器已在设置中关闭' };
1617
+ if (ctx.browserRun?.supervised !== true)
1618
+ return {
1619
+ ok: false,
1620
+ error: 'BrowserBatch 仅可在 Desktop 的受监督浏览器会话中使用',
1621
+ };
1622
+ const bin = await resolveAgentBrowserPath();
1623
+ if (!bin)
1624
+ return { ok: false, error: '内置浏览器不可用:未找到 agent-browser' };
1625
+ const runId = ctx.browserRun?.runId ?? ctx.toolCallId ?? randomUUID();
1626
+ const conversationKey =
1627
+ ctx.browserRun?.conversationKey ?? `headless:${process.pid}`;
1628
+ const run = browserRunRegistry.ensure({
1629
+ conversationKey,
1630
+ runId,
1631
+ grants: ctx.browserRun?.grants,
1632
+ allowedUploadRoots: ctx.browserRun?.allowedUploadRoots,
1633
+ });
1634
+ try {
1635
+ browserRunRegistry.requireGrant(run, 'interact');
1636
+ if (input.steps.some((step) => step.intent === 'submit'))
1637
+ browserRunRegistry.requireGrant(run, 'submit');
1638
+ const uploadSteps = input.steps.filter((step) => step.action === 'upload');
1639
+ if (uploadSteps.length) {
1640
+ browserRunRegistry.requireGrant(run, 'upload');
1641
+ const roots = await Promise.all(
1642
+ run.allowedUploadRoots.map(async (root) => {
1643
+ try {
1644
+ return await realpath(root);
1645
+ } catch {
1646
+ return root;
1647
+ }
1648
+ }),
1649
+ );
1650
+ if (!roots.length)
1651
+ throw new Error('当前浏览器任务没有获批的文件上传范围');
1652
+ for (const file of uploadSteps.flatMap((step) => step.files ?? [])) {
1653
+ const canonical = await realpath(file).catch(() => '');
1654
+ if (!canonical) throw new Error(`上传文件不存在或不可读取:${file}`);
1655
+ const allowed = roots.some((root) => {
1656
+ const child = relative(root, canonical);
1657
+ return (
1658
+ child === '' || (!child.startsWith('..') && !isAbsolute(child))
1659
+ );
1660
+ });
1661
+ if (!allowed) throw new Error(`上传文件超出当前任务批准范围:${file}`);
1662
+ }
1663
+ }
1664
+ const bundleDirs = input.steps
1665
+ .filter(
1666
+ (step) =>
1667
+ ['pageAssets', 'bundlePageAssets'].includes(step.action) &&
1668
+ (step.bundle_assets || step.action === 'bundlePageAssets'),
1669
+ )
1670
+ .map((step) => step.bundle_dir)
1671
+ .filter((value): value is string => Boolean(value));
1672
+ for (const bundleDir of bundleDirs) {
1673
+ const roots = [browserArtifactDir(), getWorkingDir()];
1674
+ const target = await realpath(bundleDir).catch(() => bundleDir);
1675
+ const allowed = roots.some((root) => {
1676
+ const child = relative(root, target);
1677
+ return child === '' || (!child.startsWith('..') && !isAbsolute(child));
1678
+ });
1679
+ if (!allowed)
1680
+ throw new Error(
1681
+ `资产打包目录不在浏览器产物或工作区范围内:${bundleDir}`,
1682
+ );
1683
+ }
1684
+ } catch (error) {
1685
+ return {
1686
+ ok: false,
1687
+ error: error instanceof Error ? error.message : String(error),
1688
+ };
1689
+ }
1690
+ try {
1691
+ let claimedTabId = input.tab_id
1692
+ ? await claimConversationTabById(
1693
+ {
1694
+ context: run,
1695
+ sessionName: BROWSER_SESSION,
1696
+ bin,
1697
+ globals: sessionArgs(BROWSER_SESSION),
1698
+ runner: spawnRunner,
1699
+ signal,
1700
+ },
1701
+ input.tab_id,
1702
+ )
1703
+ : await ensureConversationTab({
1704
+ context: run,
1705
+ sessionName: BROWSER_SESSION,
1706
+ bin,
1707
+ globals: sessionArgs(BROWSER_SESSION),
1708
+ runner: spawnRunner,
1709
+ signal,
1710
+ });
1711
+ if (input.retention) {
1712
+ browserRunRegistry.markTab(run, claimedTabId, {
1713
+ retention: input.retention,
1714
+ ...(input.tab_title ? { title: input.tab_title } : {}),
1715
+ });
1716
+ }
1717
+ const serviceInput = {
1718
+ sessionName: BROWSER_SESSION,
1719
+ bin,
1720
+ runner: spawnRunner,
1721
+ signal,
1722
+ };
1723
+ const runExclusive = <T>(task: () => Promise<T>): Promise<T> =>
1724
+ browserService.exclusive(serviceInput, async () => task());
1725
+ // Do not hold the process-wide browser mutation lock while waiting for a
1726
+ // human. Desktop must be able to click/type during waiting_human; only
1727
+ // each setup/action/final-observation critical section is serialized.
1728
+ return await (async () => {
1729
+ const tabMarker = `${conversationKey}:${runId}:${randomUUID()}`;
1730
+ const markActiveTab = async (
1731
+ markerSignal?: AbortSignal,
1732
+ ): Promise<void> => {
1733
+ const marked = await spawnRunner(
1734
+ bin,
1735
+ [
1736
+ ...sessionArgs(BROWSER_SESSION),
1737
+ 'eval',
1738
+ `globalThis[${JSON.stringify(BROWSER_TAB_MARKER_KEY)}]=${JSON.stringify(tabMarker)}`,
1739
+ ],
1740
+ markerSignal,
1741
+ );
1742
+ if (marked.code !== 0)
1743
+ throw new Error(marked.stderr || '无法标记当前浏览器会话标签');
1744
+ };
1745
+ // Reuse the same owned tab policy as WebBrowser before opening CDP. This
1746
+ // makes a batch one atomic lease instead of an accidental shared tab.
1747
+ // Setup is serialized, but this lock is released before any human wait.
1748
+ const setup = await runExclusive(async () => {
1749
+ const switched = await spawnRunner(
1750
+ bin,
1751
+ [...sessionArgs(BROWSER_SESSION), 'tab', claimedTabId],
1752
+ signal,
1753
+ );
1754
+ if (switched.code !== 0)
1755
+ throw new Error(switched.stderr || '无法切换到已认领的浏览器标签页');
1756
+ await markActiveTab(signal);
1757
+ // The daemon exposes a browser-level CDP endpoint. Prefer the URL of
1758
+ // the tab we just claimed when attaching a page target, otherwise a
1759
+ // multi-tab session could silently bind the last unrelated tab.
1760
+ const currentPage = await spawnRunner(
1761
+ bin,
1762
+ [...sessionArgs(BROWSER_SESSION), 'get', 'url'],
1763
+ signal,
1764
+ );
1765
+ const lastKnownUrl =
1766
+ currentPage.code === 0 ? currentPage.stdout.trim() : undefined;
1767
+ const cdp = await getBrowserAutomation(
1768
+ BROWSER_SESSION,
1769
+ bin,
1770
+ spawnRunner,
1771
+ signal,
1772
+ lastKnownUrl,
1773
+ tabMarker,
1774
+ );
1775
+ return { cdp, lastKnownUrl };
1776
+ });
1777
+ let { cdp, lastKnownUrl } = setup;
1778
+ const snapshotKey = () => `${conversationKey}:${runId}:${claimedTabId}`;
1779
+
1780
+ /**
1781
+ * Rebind both sides of the tab lease after a daemon/page restart.
1782
+ * A CDP reconnect alone is insufficient: the old tab id may have
1783
+ * disappeared with the browser process, and attaching the last page
1784
+ * would silently send the next action to another conversation.
1785
+ */
1786
+ const rebindTab = async (recoverySignal?: AbortSignal): Promise<void> => {
1787
+ const rebound = await spawnRunner(
1788
+ bin,
1789
+ [...sessionArgs(BROWSER_SESSION), 'tab', claimedTabId],
1790
+ recoverySignal,
1791
+ );
1792
+ if (rebound.code === 0) {
1793
+ const active = await spawnRunner(
1794
+ bin,
1795
+ [...sessionArgs(BROWSER_SESSION), 'get', 'url'],
1796
+ recoverySignal,
1797
+ );
1798
+ if (active.code === 0) lastKnownUrl = active.stdout.trim();
1799
+ } else {
1800
+ const beforeResult = await spawnRunner(
1801
+ bin,
1802
+ [...sessionArgs(BROWSER_SESSION), 'tab', 'list', '--json'],
1803
+ recoverySignal,
1804
+ );
1805
+ if (beforeResult.code !== 0)
1806
+ throw new Error(
1807
+ beforeResult.stderr || '浏览器重启后无法读取标签列表',
1808
+ );
1809
+ const beforeIds = new Set(
1810
+ parseTabList(beforeResult.stdout).map((tab) => tab.id),
1811
+ );
1812
+ const created = await spawnRunner(
1813
+ bin,
1814
+ [
1815
+ ...sessionArgs(BROWSER_SESSION),
1816
+ 'tab',
1817
+ 'new',
1818
+ lastKnownUrl || 'about:blank',
1819
+ ],
1820
+ recoverySignal,
1821
+ );
1822
+ if (created.code !== 0)
1823
+ throw new Error(created.stderr || '浏览器重启后无法创建恢复标签');
1824
+ const afterResult = await spawnRunner(
1825
+ bin,
1826
+ [...sessionArgs(BROWSER_SESSION), 'tab', 'list', '--json'],
1827
+ recoverySignal,
1828
+ );
1829
+ if (afterResult.code !== 0)
1830
+ throw new Error(
1831
+ afterResult.stderr || '浏览器重启后无法确认恢复标签',
1832
+ );
1833
+ const added = parseTabList(afterResult.stdout).filter(
1834
+ (tab) => !beforeIds.has(tab.id),
1835
+ );
1836
+ const replacement = added.find((tab) => tab.active) ?? added.at(-1);
1837
+ if (!replacement)
1838
+ throw new Error('浏览器重启后未返回新的恢复标签 ID');
1839
+ browserRunRegistry.releaseTab(run.conversationKey, claimedTabId);
1840
+ browserRunRegistry.claimTab(run, replacement.id);
1841
+ claimedTabId = replacement.id;
1842
+ }
1843
+ const active = await spawnRunner(
1844
+ bin,
1845
+ [...sessionArgs(BROWSER_SESSION), 'get', 'url'],
1846
+ recoverySignal,
1847
+ );
1848
+ if (active.code === 0) lastKnownUrl = active.stdout.trim();
1849
+ await markActiveTab(recoverySignal);
1850
+ };
1851
+ const rebindTabAndPage = async (
1852
+ recoverySignal?: AbortSignal,
1853
+ ): Promise<void> => {
1854
+ await rebindTab(recoverySignal);
1855
+ await cdp.attachToPage(lastKnownUrl, true, tabMarker);
1856
+ };
1857
+ const downloadDir = await runExclusive(() =>
1858
+ configureDownloads(cdp, conversationKey, runId),
1859
+ );
1860
+ const stream = getBrowserStream(BROWSER_SESSION);
1861
+ let currentStepAbort: AbortController | undefined;
1862
+ let offControl: (() => void) | undefined;
1863
+ let offDownload: (() => void) | undefined;
1864
+ // ⚠️ 取锁必须是 try 的第一行,释放必须在同一个 finally 内——这把锁是进程级
1865
+ // 单例,唯一释放路径是下面的 finally。此前 beginAgentAction 与
1866
+ // stream.finishAgentAction 分属两个不同的 try 块(中间横跨整段初始化 +
1867
+ // 整个逐步执行循环),ctx.reportProgress(宿主注入回调)等任何一步抛错都会
1868
+ // 跳过 finishAgentAction,锁永久不释放——对齐 webbrowser.ts 的写法(见其头注)。
1869
+ try {
1870
+ stream.beginAgentAction({ runId, conversationKey, action: 'batch' });
1871
+ offControl = stream.subscribeControl((state) => {
1872
+ if (state.status === 'waiting_human')
1873
+ currentStepAbort?.abort(new Error('浏览器已交给用户处理'));
1874
+ });
1875
+ stream.setAgentActing(true, 'batch', `${input.steps.length} 个步骤`);
1876
+ ctx.reportProgress?.(
1877
+ `开始浏览器批量任务,共 ${input.steps.length} 个步骤`,
1878
+ );
1879
+ const downloads = new Map<string, DownloadItem>();
1880
+ const subscribeDownloadEvents = (
1881
+ connection: Awaited<ReturnType<typeof getBrowserAutomation>>,
1882
+ ): (() => void) =>
1883
+ connection.subscribe((event) => {
1884
+ if (event.method === 'Browser.downloadWillBegin') {
1885
+ const id = String(event.params.guid ?? randomUUID());
1886
+ const item = {
1887
+ id,
1888
+ url: String(event.params.url ?? ''),
1889
+ filename:
1890
+ basename(
1891
+ String(event.params.suggestedFilename ?? 'download'),
1892
+ ) || 'download',
1893
+ path: join(
1894
+ downloadDir,
1895
+ basename(
1896
+ String(event.params.suggestedFilename ?? 'download'),
1897
+ ) || 'download',
1898
+ ),
1899
+ state: 'start' as const,
1900
+ startedAt: Date.now(),
1901
+ };
1902
+ downloads.set(id, item);
1903
+ stream.publish({ type: 'download', ...item, state: 'start' });
1904
+ }
1905
+ if (event.method === 'Browser.downloadProgress') {
1906
+ const id = String(event.params.guid ?? '');
1907
+ const item = downloads.get(id);
1908
+ if (!item) return;
1909
+ const state =
1910
+ event.params.state === 'completed'
1911
+ ? 'complete'
1912
+ : event.params.state === 'canceled'
1913
+ ? 'canceled'
1914
+ : 'progress';
1915
+ item.state = state;
1916
+ item.receivedBytes =
1917
+ Number(event.params.receivedBytes ?? 0) || undefined;
1918
+ item.totalBytes = Number(event.params.totalBytes ?? 0) || undefined;
1919
+ item.error =
1920
+ typeof event.params.error === 'string'
1921
+ ? event.params.error
1922
+ : undefined;
1923
+ stream.publish({
1924
+ type: 'download',
1925
+ ...item,
1926
+ state,
1927
+ receivedBytes: item.receivedBytes,
1928
+ totalBytes: item.totalBytes,
1929
+ ...(item.error ? { error: item.error } : {}),
1930
+ });
1931
+ }
1932
+ });
1933
+ offDownload = subscribeDownloadEvents(cdp);
1934
+ const results: Array<Record<string, unknown>> = [];
1935
+ let artifactManifest: DeliverableManifest | undefined;
1936
+ for (const [index, step] of input.steps.entries()) {
1937
+ const started = Date.now();
1938
+ const stepAbort = new AbortController();
1939
+ currentStepAbort = stepAbort;
1940
+ const abortFromParent = () => stepAbort.abort(signal?.reason);
1941
+ if (signal?.aborted) stepAbort.abort(signal.reason);
1942
+ else signal?.addEventListener('abort', abortFromParent, { once: true });
1943
+ const waits: Promise<CdpEvent>[] = [];
1944
+ try {
1945
+ ctx.reportProgress?.(
1946
+ `浏览器批量任务 ${index + 1}/${input.steps.length}:${step.action}`,
1947
+ );
1948
+ const conditions = step.expect?.conditions ?? [];
1949
+ const preHuman = await runExclusive(async () => {
1950
+ const eventCursor = cdp.read(0).at(-1)?.sequence ?? 0;
1951
+ const baseline = await evaluate(
1952
+ cdp,
1953
+ `location.href + '|' + (document.body?.innerText || '').slice(0, 4000)`,
1954
+ stepAbort.signal,
1955
+ ).catch(() => undefined);
1956
+ if (step.expect?.navigation)
1957
+ waits.push(
1958
+ waitFor(
1959
+ cdp,
1960
+ (e) =>
1961
+ e.method.startsWith('Page.frame') &&
1962
+ (!step.expect?.navigation?.url ||
1963
+ JSON.stringify(e.params).includes(
1964
+ step.expect.navigation.url,
1965
+ )),
1966
+ timeoutMs,
1967
+ eventCursor,
1968
+ stepAbort.signal,
1969
+ ),
1970
+ );
1971
+ if (step.expect?.new_tab)
1972
+ waits.push(
1973
+ waitFor(
1974
+ cdp,
1975
+ (e) =>
1976
+ e.method === 'Target.targetCreated' &&
1977
+ (!step.expect?.new_tab?.url ||
1978
+ JSON.stringify(e.params).includes(
1979
+ step.expect.new_tab.url,
1980
+ )),
1981
+ timeoutMs,
1982
+ eventCursor,
1983
+ stepAbort.signal,
1984
+ ),
1985
+ );
1986
+ if (step.expect?.download)
1987
+ waits.push(
1988
+ waitFor(
1989
+ cdp,
1990
+ (e) =>
1991
+ e.method === 'Browser.downloadWillBegin' &&
1992
+ (!step.expect?.download?.filename ||
1993
+ String(e.params.suggestedFilename ?? '').includes(
1994
+ step.expect.download.filename,
1995
+ )),
1996
+ timeoutMs,
1997
+ eventCursor,
1998
+ stepAbort.signal,
1999
+ ),
2000
+ );
2001
+ const uploadChooser =
2002
+ step.action === 'upload' &&
2003
+ (step.trigger_selector || step.trigger_locator)
2004
+ ? { files: step.files }
2005
+ : undefined;
2006
+ if (step.expect?.file_chooser || uploadChooser)
2007
+ waits.push(
2008
+ waitFor(
2009
+ cdp,
2010
+ (e) => e.method === 'Page.fileChooserOpened',
2011
+ timeoutMs,
2012
+ eventCursor,
2013
+ stepAbort.signal,
2014
+ ),
2015
+ );
2016
+ for (const condition of step.expect?.conditions ?? []) {
2017
+ if (
2018
+ condition.type === 'download' ||
2019
+ condition.type === 'new-tab' ||
2020
+ condition.type === 'dialog'
2021
+ ) {
2022
+ waits.push(
2023
+ waitFor(
2024
+ cdp,
2025
+ (event) =>
2026
+ eventMatchesCondition(
2027
+ condition as BrowserCondition,
2028
+ event,
2029
+ ),
2030
+ timeoutMs,
2031
+ eventCursor,
2032
+ stepAbort.signal,
2033
+ ),
2034
+ );
2035
+ }
2036
+ }
2037
+ const value = await withBrowserRecovery(
2038
+ () =>
2039
+ executeStep(
2040
+ cdp,
2041
+ step,
2042
+ timeoutMs,
2043
+ eventCursor,
2044
+ snapshotKey(),
2045
+ stepAbort.signal,
2046
+ ),
2047
+ {
2048
+ signal: stepAbort.signal,
2049
+ reconnect: async () => {
2050
+ await ensureDaemon(BROWSER_SESSION, {
2051
+ signal: stepAbort.signal,
2052
+ forceRestart: true,
2053
+ });
2054
+ browserRunRegistry.invalidateAllTabs();
2055
+ // A daemon restart can publish a new browser-level WebSocket
2056
+ // URL. Rebind the tab first, then mark that exact tab, and
2057
+ // only then create a fresh page-scoped CDP connection.
2058
+ await rebindTab(stepAbort.signal);
2059
+ const previousCdp = cdp;
2060
+ offDownload?.();
2061
+ cdp = await getBrowserAutomation(
2062
+ BROWSER_SESSION,
2063
+ bin,
2064
+ spawnRunner,
2065
+ stepAbort.signal,
2066
+ lastKnownUrl,
2067
+ tabMarker,
2068
+ );
2069
+ offDownload = subscribeDownloadEvents(cdp);
2070
+ previousCdp.close();
2071
+ },
2072
+ rebindTab: async () => {
2073
+ await rebindTabAndPage(stepAbort.signal);
2074
+ },
2075
+ },
2076
+ );
2077
+ if (step.action === 'navigate' && step.url) lastKnownUrl = step.url;
2078
+ let events: CdpEvent[] = [];
2079
+ if (waits.length) {
2080
+ const challengeAbort = new AbortController();
2081
+ const challengeRace = waitForBrowserChallenge(
2082
+ cdp,
2083
+ timeoutMs,
2084
+ challengeAbort.signal,
2085
+ ).then((challenge) => {
2086
+ if (challenge)
2087
+ throw new Error(`页面需要人工验证:${challenge.summary}`);
2088
+ return new Promise<never>(() => undefined);
2089
+ });
2090
+ try {
2091
+ events = await Promise.race([
2092
+ Promise.all(waits),
2093
+ challengeRace,
2094
+ ]);
2095
+ } finally {
2096
+ challengeAbort.abort();
2097
+ }
2098
+ }
2099
+ if (step.artifact && step.action === 'exportContent') {
2100
+ const nextManifest = await materializeExportArtifact(
2101
+ value,
2102
+ step.export_format ?? 'page',
2103
+ step.artifact.save_as,
2104
+ );
2105
+ if (step.artifact.label)
2106
+ nextManifest.files[0].label = step.artifact.label;
2107
+ artifactManifest = mergeArtifactManifests(
2108
+ artifactManifest,
2109
+ nextManifest,
2110
+ );
2111
+ } else if (step.artifact) {
2112
+ const startedDownload = events.find(
2113
+ (event) => event.method === 'Browser.downloadWillBegin',
2114
+ );
2115
+ const downloadId = String(startedDownload?.params.guid ?? '');
2116
+ const item = downloadId
2117
+ ? downloads.get(downloadId)
2118
+ : [...downloads.values()].at(-1);
2119
+ if (!item) throw new Error('未捕获到要保存为成果的下载');
2120
+ if (item.state !== 'complete') {
2121
+ const completed = await waitForEventOrBrowserChallenge(
2122
+ cdp,
2123
+ (event) =>
2124
+ event.method === 'Browser.downloadProgress' &&
2125
+ String(event.params.guid ?? '') === item.id &&
2126
+ ['completed', 'canceled'].includes(
2127
+ String(event.params.state ?? ''),
2128
+ ),
2129
+ timeoutMs,
2130
+ eventCursor,
2131
+ stepAbort.signal,
2132
+ );
2133
+ if (completed.params.state !== 'completed')
2134
+ throw new Error(
2135
+ `下载未完成:${String(completed.params.state ?? 'unknown')}`,
2136
+ );
2137
+ item.state = 'complete';
2138
+ }
2139
+ item.path = await waitForDownloadedFile(
2140
+ downloadDir,
2141
+ item,
2142
+ timeoutMs,
2143
+ stepAbort.signal,
2144
+ );
2145
+ const nextManifest = await materializeDownloadArtifact(
2146
+ item.path,
2147
+ step.artifact.save_as,
2148
+ );
2149
+ if (step.artifact.label) {
2150
+ nextManifest.files[0].label = step.artifact.label;
2151
+ }
2152
+ artifactManifest = mergeArtifactManifests(
2153
+ artifactManifest,
2154
+ nextManifest,
2155
+ );
2156
+ }
2157
+ const chooser = step.expect?.file_chooser ?? uploadChooser;
2158
+ if (chooser?.files?.length) {
2159
+ const opened = events.find(
2160
+ (event) => event.method === 'Page.fileChooserOpened',
2161
+ );
2162
+ const backendNodeId = Number(opened?.params.backendNodeId ?? 0);
2163
+ if (!backendNodeId)
2164
+ throw new Error(
2165
+ 'Browser did not provide the file chooser node',
2166
+ );
2167
+ await cdp.send('DOM.setFileInputFiles', {
2168
+ files: chooser.files,
2169
+ backendNodeId,
2170
+ });
2171
+ }
2172
+ for (const condition of conditions) {
2173
+ if (
2174
+ condition.type === 'download' ||
2175
+ condition.type === 'new-tab' ||
2176
+ condition.type === 'dialog'
2177
+ )
2178
+ continue;
2179
+ await waitForCondition(
2180
+ cdp,
2181
+ condition as BrowserCondition,
2182
+ timeoutMs,
2183
+ baseline,
2184
+ stepAbort.signal,
2185
+ );
2186
+ }
2187
+ const challenge = await readBrowserChallenge(cdp, stepAbort.signal);
2188
+ return { value, events, challenge };
2189
+ });
2190
+ const { value, events, challenge } = preHuman;
2191
+ let humanSnapshot: Awaited<ReturnType<typeof snapshot>> | undefined;
2192
+ if (challenge) {
2193
+ stream.requestHuman(challenge.reason, {
2194
+ runId,
2195
+ conversationKey,
2196
+ ownerId: 'local-owner',
2197
+ summary: challenge.summary,
2198
+ });
2199
+ const resolution = await resumeBatchAfterHuman(
2200
+ stream,
2201
+ ctx,
2202
+ signal,
2203
+ challenge.summary,
2204
+ );
2205
+ if (resolution === 'stopped')
2206
+ throw new Error('浏览器批量任务已由用户停止');
2207
+ humanSnapshot = await runExclusive(() =>
2208
+ snapshot(cdp, snapshotKey(), 'full', signal),
2209
+ );
2210
+ }
2211
+ results.push({
2212
+ id: step.id ?? String(index + 1),
2213
+ ok: true,
2214
+ duration_ms: Date.now() - started,
2215
+ result: value,
2216
+ resolution_strategy: resolveInteractionStrategy(step),
2217
+ events,
2218
+ ...(humanSnapshot
2219
+ ? {
2220
+ human_intervention: 'resumed',
2221
+ post_intervention_snapshot: humanSnapshot,
2222
+ }
2223
+ : {}),
2224
+ ...(conditions.length
2225
+ ? {
2226
+ conditions_satisfied: conditions.map(
2227
+ (condition) => condition.type,
2228
+ ),
2229
+ }
2230
+ : {}),
2231
+ });
2232
+ } catch (error) {
2233
+ await Promise.allSettled(waits);
2234
+ let errorMessage =
2235
+ error instanceof Error ? error.message : String(error);
2236
+ if (stream.getControlState().status !== 'waiting_human') {
2237
+ const challenge = await runExclusive(() =>
2238
+ readBrowserChallenge(cdp).catch(() => null),
2239
+ );
2240
+ if (challenge) {
2241
+ stream.requestHuman(challenge.reason, {
2242
+ runId,
2243
+ conversationKey,
2244
+ ownerId: 'local-owner',
2245
+ summary: challenge.summary,
2246
+ });
2247
+ }
2248
+ }
2249
+ if (stream.getControlState().status === 'waiting_human') {
2250
+ const resolution = await resumeBatchAfterHuman(
2251
+ stream,
2252
+ ctx,
2253
+ signal,
2254
+ '浏览器批量步骤已暂停',
2255
+ );
2256
+ if (resolution === 'resumed') {
2257
+ const recovered = await runExclusive(() =>
2258
+ snapshot(cdp, snapshotKey(), 'full', signal),
2259
+ );
2260
+ results.push({
2261
+ id: step.id ?? String(index + 1),
2262
+ ok: false,
2263
+ duration_ms: Date.now() - started,
2264
+ resolution_strategy: resolveInteractionStrategy(step),
2265
+ error:
2266
+ '人工接管中断了当前步骤;浏览器已恢复,但该步骤是否完成无法自动确认。请依据最新快照重新确认后再继续。',
2267
+ result: {
2268
+ human_intervention: 'resumed_unverified',
2269
+ post_intervention_snapshot: recovered,
2270
+ },
2271
+ });
2272
+ // Never continue subsequent steps with an unknown side effect:
2273
+ // a click may have landed before the handoff, while a form
2274
+ // submission may still be pending. The next model turn can
2275
+ // inspect the fresh snapshot and deliberately resume.
2276
+ break;
2277
+ }
2278
+ errorMessage = '浏览器批量任务已由用户停止';
2279
+ }
2280
+ results.push({
2281
+ id: step.id ?? String(index + 1),
2282
+ ok: false,
2283
+ duration_ms: Date.now() - started,
2284
+ resolution_strategy: resolveInteractionStrategy(step),
2285
+ error: errorMessage,
2286
+ });
2287
+ if (stopOnError) break;
2288
+ } finally {
2289
+ stepAbort.abort();
2290
+ signal?.removeEventListener('abort', abortFromParent);
2291
+ if (currentStepAbort === stepAbort) currentStepAbort = undefined;
2292
+ }
2293
+ }
2294
+ let final: Awaited<ReturnType<typeof snapshot>> | undefined;
2295
+ let finalScreenshot: string | undefined;
2296
+ await runExclusive(async () => {
2297
+ final =
2298
+ finalSnapshotMode === 'none'
2299
+ ? undefined
2300
+ : await snapshot(
2301
+ cdp,
2302
+ `${conversationKey}:${runId}:${claimedTabId}`,
2303
+ finalSnapshotMode,
2304
+ signal,
2305
+ );
2306
+ if (captureMode === 'screenshot' || captureMode === 'both') {
2307
+ const path = await createArtifactPath('browser-batch-final', 'png');
2308
+ const captured = (await cdp.send('Page.captureScreenshot', {
2309
+ format: 'png',
2310
+ fromSurface: true,
2311
+ })) as { data?: string };
2312
+ if (!captured.data) throw new Error('浏览器未返回最终截图');
2313
+ await writeFile(path, Buffer.from(captured.data, 'base64'), {
2314
+ mode: 0o600,
2315
+ });
2316
+ finalScreenshot = path;
2317
+ }
2318
+ });
2319
+ const failed = results.some((result) => result.ok === false);
2320
+ const content = JSON.stringify({
2321
+ tab_id: claimedTabId,
2322
+ download_dir: downloadDir,
2323
+ results,
2324
+ ...(final ? { final_snapshot: final } : {}),
2325
+ ...(finalScreenshot ? { final_screenshot: finalScreenshot } : {}),
2326
+ });
2327
+ // A direct export (PDF/MD/Workspace formats) has no Browser.download*
2328
+ // event, but it is still a formal browser deliverable. Keep one
2329
+ // metadata envelope for the Desktop delivery ledger and allow an empty
2330
+ // downloads list for that case.
2331
+ const metadata =
2332
+ downloads.size || artifactManifest
2333
+ ? {
2334
+ kind: 'browser_download' as const,
2335
+ downloads: [...downloads.values()],
2336
+ ...(artifactManifest ? { artifact: artifactManifest } : {}),
2337
+ }
2338
+ : undefined;
2339
+ return failed
2340
+ ? { ok: false, error: content, ...(metadata ? { metadata } : {}) }
2341
+ : { ok: true, content, ...(metadata ? { metadata } : {}) };
2342
+ } finally {
2343
+ offDownload?.();
2344
+ offControl?.();
2345
+ currentStepAbort?.abort();
2346
+ currentStepAbort = undefined;
2347
+ cdp.close();
2348
+ stream.finishAgentAction(runId);
2349
+ try {
2350
+ stream.setAgentActing(false);
2351
+ } catch {
2352
+ // UI decoration failure must not hide the batch result or leak control state.
2353
+ }
2354
+ }
2355
+ })();
2356
+ } catch (error) {
2357
+ return {
2358
+ ok: false,
2359
+ error: error instanceof Error ? error.message : String(error),
2360
+ };
2361
+ } finally {
2362
+ // AX snapshots are part of the BrowserBatch contract: the model may use
2363
+ // the returned generation in the next tool call. Keep them behind the
2364
+ // bounded TTL/capacity eviction in captureFinalSnapshot instead of
2365
+ // invalidating the reference as soon as this call returns.
2366
+ browserRunRegistry.finish(conversationKey, runId);
2367
+ }
2368
+ }
2369
+
2370
+ // biome-ignore lint/suspicious/noExplicitAny: Tool's schema parser supplies the inferred output type.
2371
+ export const browserBatchTool: Tool<any> = {
2372
+ name: 'BrowserBatch',
2373
+ description:
2374
+ 'Executes multiple visible-browser actions under one tab claim and one persistent CDP connection. Supports semantic locators, event expectations, downloads, waits and final snapshots.',
2375
+ inputSchema,
2376
+ parameters: toToolParameters(inputSchema),
2377
+ isReadOnly: false,
2378
+ isConcurrencySafe: false,
2379
+ maxResultSizeChars: DEFAULT_MAX_RESULT_SIZE_CHARS,
2380
+ call,
2381
+ callWithContext: call,
2382
+ };