minimal-agent 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (472) hide show
  1. package/README.md +6 -5
  2. package/package.json +62 -5
  3. package/plugins/computer_use/.claude-plugin/plugin.json +1 -1
  4. package/plugins/cron-runner/src/commands.js +4 -4
  5. package/plugins/cron-runner/src/commands.ts +4 -4
  6. package/plugins/workflow-runner/.claude-plugin/plugin.json +2 -2
  7. package/plugins/workflow-runner/commands/workflow.md +57 -90
  8. package/plugins/workflow-runner/commands/workflows.md +4 -4
  9. package/plugins/workflow-runner/plugin.js +24 -13
  10. package/plugins/workflow-runner/plugin.ts +28 -13
  11. package/plugins/workflow-runner/src/discovery.js +160 -0
  12. package/plugins/workflow-runner/src/discovery.ts +194 -0
  13. package/plugins/workflow-runner/src/goalAnchor.js +53 -0
  14. package/plugins/workflow-runner/src/goalAnchor.ts +59 -0
  15. package/plugins/workflow-runner/src/index.js +204 -185
  16. package/plugins/workflow-runner/src/index.ts +249 -224
  17. package/plugins/workflow-runner/src/inputCoercion.js +40 -0
  18. package/plugins/workflow-runner/src/inputCoercion.ts +43 -0
  19. package/plugins/workflow-runner/src/journal.js +224 -0
  20. package/plugins/workflow-runner/src/journal.ts +274 -0
  21. package/plugins/workflow-runner/src/jsonSchema.js +132 -0
  22. package/plugins/workflow-runner/src/jsonSchema.ts +162 -0
  23. package/plugins/workflow-runner/src/moduleResolution.js +64 -0
  24. package/plugins/workflow-runner/src/moduleResolution.ts +71 -0
  25. package/plugins/workflow-runner/src/resourceInputs.js +220 -0
  26. package/plugins/workflow-runner/src/resourceInputs.ts +300 -0
  27. package/plugins/workflow-runner/src/runner.js +404 -467
  28. package/plugins/workflow-runner/src/runner.ts +491 -535
  29. package/plugins/workflow-runner/src/runtime/config.js +46 -0
  30. package/plugins/workflow-runner/src/runtime/config.ts +69 -0
  31. package/plugins/workflow-runner/src/runtime/context.js +1304 -0
  32. package/plugins/workflow-runner/src/runtime/context.ts +1786 -0
  33. package/plugins/workflow-runner/src/runtime/exec.js +34 -0
  34. package/plugins/workflow-runner/src/runtime/exec.ts +53 -0
  35. package/plugins/workflow-runner/src/runtime/flow.js +8 -0
  36. package/plugins/workflow-runner/src/runtime/flow.ts +12 -0
  37. package/plugins/workflow-runner/src/runtime/fs.js +27 -0
  38. package/plugins/workflow-runner/src/runtime/fs.ts +41 -0
  39. package/plugins/workflow-runner/src/runtime/model.js +6 -0
  40. package/plugins/workflow-runner/src/runtime/model.ts +8 -0
  41. package/plugins/workflow-runner/src/runtime/output.js +7 -0
  42. package/plugins/workflow-runner/src/runtime/output.ts +21 -0
  43. package/plugins/workflow-runner/src/runtime/scheduler.js +141 -0
  44. package/plugins/workflow-runner/src/runtime/scheduler.ts +166 -0
  45. package/plugins/workflow-runner/src/runtime/state.js +16 -0
  46. package/plugins/workflow-runner/src/runtime/state.ts +19 -0
  47. package/plugins/workflow-runner/src/runtime/tools.js +3 -0
  48. package/plugins/workflow-runner/src/runtime/tools.ts +6 -0
  49. package/plugins/workflow-runner/src/sandbox.js +141 -0
  50. package/plugins/workflow-runner/src/sandbox.ts +182 -0
  51. package/plugins/workflow-runner/src/scriptGuard.d.mts +38 -0
  52. package/plugins/workflow-runner/src/scriptGuard.mjs +449 -0
  53. package/plugins/workflow-runner/src/taskIdentity.js +172 -152
  54. package/plugins/workflow-runner/src/taskIdentity.ts +233 -200
  55. package/plugins/workflow-runner/src/toolRuntime.js +29 -8
  56. package/plugins/workflow-runner/src/toolRuntime.ts +36 -11
  57. package/plugins/workflow-runner/src/types.js +7 -105
  58. package/plugins/workflow-runner/src/types.ts +155 -520
  59. package/plugins/workflow-runner/src/util.js +47 -0
  60. package/plugins/workflow-runner/src/util.ts +51 -0
  61. package/plugins/workflow-runner/src/workflowContext.js +20 -0
  62. package/plugins/workflow-runner/src/workflowContext.ts +58 -0
  63. package/plugins/workflow-runner/src/workflowDependencies.js +36 -0
  64. package/plugins/workflow-runner/src/workflowDependencies.ts +59 -0
  65. package/plugins/workflow-runner/src/workflowFingerprint.js +118 -0
  66. package/plugins/workflow-runner/src/workflowFingerprint.ts +122 -0
  67. package/plugins/workflow-runner/src/workflowState.js +542 -469
  68. package/plugins/workflow-runner/src/workflowState.ts +639 -589
  69. package/plugins/workflow-runner/test/api.test.ts +223 -0
  70. package/plugins/workflow-runner/test/apiCapabilities.test.ts +614 -0
  71. package/plugins/workflow-runner/test/apiConcurrencyGate.test.ts +104 -0
  72. package/plugins/workflow-runner/test/apiEcho.test.ts +210 -0
  73. package/plugins/workflow-runner/test/apiReadText.test.ts +178 -0
  74. package/plugins/workflow-runner/test/apiResourceLedger.test.ts +184 -0
  75. package/plugins/workflow-runner/test/childWorkflow.test.ts +304 -0
  76. package/plugins/workflow-runner/test/discovery.test.ts +234 -0
  77. package/plugins/workflow-runner/test/goalAnchor.test.ts +106 -0
  78. package/plugins/workflow-runner/test/goalController.test.ts +317 -0
  79. package/plugins/workflow-runner/test/journalLanes.test.ts +198 -0
  80. package/plugins/workflow-runner/test/listResources.test.ts +106 -0
  81. package/plugins/workflow-runner/test/resourceInputs.test.ts +294 -0
  82. package/plugins/workflow-runner/test/sandbox.test.ts +126 -0
  83. package/plugins/workflow-runner/test/scheduler.test.ts +70 -0
  84. package/plugins/workflow-runner/test/scriptGuard.test.ts +276 -0
  85. package/plugins/workflow-runner/test/skillDocSync.test.ts +78 -0
  86. package/plugins/workflow-runner/test/taskIdentity.test.ts +288 -179
  87. package/plugins/workflow-runner/test/workflowFingerprint.test.ts +106 -0
  88. package/plugins/workflow-runner/test/workflowState.test.ts +429 -0
  89. package/skills/pptx/examples/assets/cover-bg.svg +23 -0
  90. package/skills/pptx/examples/assets/icon-spark.svg +5 -0
  91. package/skills/skill-creator/scripts/run_eval.py +3 -2
  92. package/skills/ui-design/SKILL.md +5 -2
  93. package/skills/ui-design/references/layout-composition.md +1 -1
  94. package/skills/workflow-creator/SKILL.md +344 -0
  95. package/skills/workflow-creator/assets/template/workflow.mjs +48 -0
  96. package/skills/workflow-creator/references/api.md +216 -0
  97. package/skills/workflow-creator/references/migration.md +103 -0
  98. package/skills/workflow-creator/references/patterns.md +779 -0
  99. package/skills/workflow-creator/scripts/list-resources.mjs +205 -0
  100. package/skills/workflow-creator/scripts/scaffold.mjs +148 -0
  101. package/skills/workflow-creator/scripts/validate.mjs +275 -0
  102. package/skills/workflow-creator/scripts/workflow-creator.test.mjs +82 -0
  103. package/src/advisor/advisorCoordinator.js +503 -0
  104. package/src/advisor/advisorCoordinator.ts +657 -0
  105. package/src/cli/configMenu.tsx +0 -1
  106. package/src/cli/print.js +44 -6
  107. package/src/cli/print.ts +73 -4
  108. package/src/cli/streamJson.js +5 -6
  109. package/src/cli/streamJson.ts +8 -4
  110. package/src/cli/wizards/advisorWizard.js +4 -2
  111. package/src/cli/wizards/advisorWizard.tsx +8 -2
  112. package/src/cli/wizards/contextAdvisorWizard.js +5 -4
  113. package/src/cli/wizards/contextAdvisorWizard.tsx +7 -6
  114. package/src/collaboration/taskFlow.js +115 -0
  115. package/src/collaboration/taskFlow.ts +169 -0
  116. package/src/config/configDefaults.js +41 -1
  117. package/src/config/configDefaults.ts +43 -1
  118. package/src/config/configDiagnostics.ts +2 -2
  119. package/src/config/configFile.js +40 -9
  120. package/src/config/configFile.ts +60 -11
  121. package/src/config/configService.js +443 -74
  122. package/src/config/configService.ts +723 -142
  123. package/src/config/numericLimits.js +22 -0
  124. package/src/config/numericLimits.ts +24 -0
  125. package/src/config.js +1 -0
  126. package/src/config.ts +3 -2
  127. package/src/context/budgetPlan.js +1 -1
  128. package/src/context/budgetPlan.ts +1 -1
  129. package/src/context/compact.js +7 -2
  130. package/src/context/compact.ts +9 -2
  131. package/src/context/microCompactLite.js +212 -13
  132. package/src/context/microCompactLite.ts +261 -12
  133. package/src/context/sessionRegistry.js +8 -1
  134. package/src/context/sessionRegistry.ts +18 -5
  135. package/src/context/toolResultArchive.js +72 -0
  136. package/src/context/toolResultArchive.ts +95 -0
  137. package/src/goal/errors.js +8 -0
  138. package/src/goal/errors.ts +18 -0
  139. package/src/goal/goalRuntime.js +344 -0
  140. package/src/goal/goalRuntime.ts +466 -0
  141. package/src/goal/goalStore.js +200 -0
  142. package/src/goal/goalStore.ts +219 -0
  143. package/src/goal/index.js +4 -0
  144. package/src/goal/index.ts +4 -0
  145. package/src/goal/statusMapping.js +71 -0
  146. package/src/goal/statusMapping.ts +95 -0
  147. package/src/goal/types.js +7 -0
  148. package/src/goal/types.ts +179 -0
  149. package/src/goal/workflowController.js +306 -0
  150. package/src/goal/workflowController.ts +360 -0
  151. package/src/harness/evidenceLedger.js +2 -0
  152. package/src/harness/evidenceLedger.ts +1 -0
  153. package/src/harness/stallGuard.js +9 -3
  154. package/src/harness/stallGuard.ts +7 -3
  155. package/src/images/imageResolver.js +161 -0
  156. package/src/images/imageResolver.ts +189 -0
  157. package/src/infra/verification.js +56 -20
  158. package/src/infra/verification.ts +87 -20
  159. package/src/llm/client.js +79 -21
  160. package/src/llm/client.ts +182 -35
  161. package/src/llm/gatewayClient.js +201 -0
  162. package/src/llm/gatewayClient.ts +299 -0
  163. package/src/llm/messageProvenance.js +46 -0
  164. package/src/llm/messageProvenance.ts +58 -0
  165. package/src/llm/modelProvider.js +1 -0
  166. package/src/llm/modelProvider.ts +22 -0
  167. package/src/llm/providers/anthropicFormat.js +108 -1
  168. package/src/llm/providers/anthropicFormat.ts +115 -4
  169. package/src/llm/providers/openaiFormat.js +9 -0
  170. package/src/llm/providers/openaiFormat.ts +12 -3
  171. package/src/loop.js +319 -54
  172. package/src/loop.ts +373 -57
  173. package/src/main.js +4 -1
  174. package/src/main.tsx +4 -1
  175. package/src/managedExecution/index.js +2 -0
  176. package/src/managedExecution/index.ts +2 -0
  177. package/src/managedExecution/registry.js +15 -0
  178. package/src/managedExecution/registry.ts +20 -0
  179. package/src/managedExecution/types.js +1 -0
  180. package/src/managedExecution/types.ts +86 -0
  181. package/src/modelRef.js +9 -0
  182. package/src/modelRef.ts +31 -0
  183. package/src/multimodal.js +14 -28
  184. package/src/multimodal.ts +17 -33
  185. package/src/net/lanDiscovery.js +65 -0
  186. package/src/net/lanDiscovery.ts +82 -0
  187. package/src/plugin-sdk.js +28 -3
  188. package/src/plugin-sdk.ts +39 -6
  189. package/src/plugins/pluginApi.ts +16 -1
  190. package/src/plugins/pluginLoader.js +4 -0
  191. package/src/plugins/pluginLoader.ts +6 -0
  192. package/src/plugins/pluginRunner.js +152 -9
  193. package/src/plugins/pluginRunner.ts +186 -8
  194. package/src/plugins/slashFallback.js +113 -0
  195. package/src/plugins/slashFallback.ts +137 -0
  196. package/src/prompts/policyRegistry.js +15 -8
  197. package/src/prompts/policyRegistry.ts +18 -8
  198. package/src/prompts/promptAssembly.js +103 -12
  199. package/src/prompts/promptAssembly.ts +114 -12
  200. package/src/prompts/skillList.js +21 -3
  201. package/src/prompts/skillList.ts +27 -2
  202. package/src/prompts/stableSystemPrompt.js +18 -0
  203. package/src/prompts/stableSystemPrompt.ts +33 -0
  204. package/src/runtime/agentRuntimeContext.js +8 -2
  205. package/src/runtime/agentRuntimeContext.ts +17 -4
  206. package/src/session/runAgentTurn.js +78 -12
  207. package/src/session/runAgentTurn.ts +105 -23
  208. package/src/session/runTurn.js +10 -9
  209. package/src/session/runTurn.ts +17 -8
  210. package/src/session/sessionState.ts +2 -0
  211. package/src/team/auth.js +186 -0
  212. package/src/team/auth.ts +298 -0
  213. package/src/team/cli.js +105 -0
  214. package/src/team/cli.ts +122 -0
  215. package/src/team/onlineToolContract.js +64 -0
  216. package/src/team/onlineToolContract.ts +109 -0
  217. package/src/team/toolPolicy.js +36 -0
  218. package/src/team/toolPolicy.ts +75 -0
  219. package/src/tools/advisor/advisor.js +148 -37
  220. package/src/tools/advisor/advisor.ts +161 -38
  221. package/src/tools/advisor/advisorDecision.js +71 -0
  222. package/src/tools/advisor/advisorDecision.ts +85 -0
  223. package/src/tools/agent/agent.js +18 -5
  224. package/src/tools/agent/agent.ts +21 -7
  225. package/src/tools/agent/agentControl.js +246 -0
  226. package/src/tools/agent/agentControl.ts +284 -0
  227. package/src/tools/agent/agentTaskService.js +40 -1
  228. package/src/tools/agent/agentTaskService.ts +44 -1
  229. package/src/tools/agent/agentTypes.js +14 -2
  230. package/src/tools/agent/agentTypes.ts +15 -2
  231. package/src/tools/applypatch/applyPatch.js +362 -0
  232. package/src/tools/applypatch/applyPatch.ts +428 -0
  233. package/src/tools/bash/bash.js +70 -18
  234. package/src/tools/bash/bash.ts +75 -20
  235. package/src/tools/capabilities.js +10 -5
  236. package/src/tools/capabilities.ts +10 -5
  237. package/src/tools/computeruse/computerIntent.js +32 -0
  238. package/src/tools/computeruse/computerIntent.ts +46 -0
  239. package/src/tools/computeruse/telemetryLog.js +12 -0
  240. package/src/tools/computeruse/telemetryLog.ts +16 -0
  241. package/src/tools/computeruse/vision/telemetry.js +1 -0
  242. package/src/tools/computeruse/vision/telemetry.ts +1 -0
  243. package/src/tools/effectiveToolSet.js +4 -0
  244. package/src/tools/effectiveToolSet.ts +4 -0
  245. package/src/tools/index.js +21 -2
  246. package/src/tools/index.ts +33 -2
  247. package/src/tools/powershell/powershell.js +18 -1
  248. package/src/tools/powershell/powershell.ts +18 -2
  249. package/src/tools/presentresult/presentresult.js +6 -15
  250. package/src/tools/presentresult/presentresult.ts +6 -15
  251. package/src/tools/read/read.js +69 -11
  252. package/src/tools/read/read.ts +70 -11
  253. package/src/tools/shared/fileUtils.js +1 -2
  254. package/src/tools/shared/fileUtils.ts +1 -2
  255. package/src/tools/shared/permanentDeleteGuard.js +34 -3
  256. package/src/tools/shared/permanentDeleteGuard.ts +43 -6
  257. package/src/tools/shared/toolOutputStore.js +23 -1
  258. package/src/tools/shared/toolOutputStore.ts +22 -1
  259. package/src/tools/skill/skill.js +9 -3
  260. package/src/tools/skill/skill.ts +8 -4
  261. package/src/tools/toolPromptGuides.js +117 -0
  262. package/src/tools/toolPromptGuides.ts +137 -0
  263. package/src/tools/webbrowser/agentBrowserPath.js +19 -9
  264. package/src/tools/webbrowser/agentBrowserPath.ts +70 -28
  265. package/src/tools/webbrowser/browser.js +27 -11
  266. package/src/tools/webbrowser/browser.ts +68 -24
  267. package/src/tools/webbrowser/browserAutomation.js +329 -0
  268. package/src/tools/webbrowser/browserAutomation.ts +467 -0
  269. package/src/tools/webbrowser/browserAxState.js +97 -0
  270. package/src/tools/webbrowser/browserAxState.ts +142 -0
  271. package/src/tools/webbrowser/browserBatch.js +1578 -0
  272. package/src/tools/webbrowser/browserBatch.ts +2382 -0
  273. package/src/tools/webbrowser/browserCdp.js +208 -0
  274. package/src/tools/webbrowser/browserCdp.ts +255 -0
  275. package/src/tools/webbrowser/browserConditions.js +24 -0
  276. package/src/tools/webbrowser/browserConditions.ts +45 -0
  277. package/src/tools/webbrowser/browserContent.js +281 -0
  278. package/src/tools/webbrowser/browserContent.ts +438 -0
  279. package/src/tools/webbrowser/browserControl.js +131 -30
  280. package/src/tools/webbrowser/browserControl.ts +210 -50
  281. package/src/tools/webbrowser/browserDaemon.js +22 -14
  282. package/src/tools/webbrowser/browserDaemon.ts +54 -21
  283. package/src/tools/webbrowser/browserFrames.js +47 -0
  284. package/src/tools/webbrowser/browserFrames.ts +54 -0
  285. package/src/tools/webbrowser/browserInteractionResolver.js +17 -0
  286. package/src/tools/webbrowser/browserInteractionResolver.ts +40 -0
  287. package/src/tools/webbrowser/browserLocator.js +140 -0
  288. package/src/tools/webbrowser/browserLocator.ts +194 -0
  289. package/src/tools/webbrowser/browserOutcome.js +10 -17
  290. package/src/tools/webbrowser/browserOutcome.ts +48 -31
  291. package/src/tools/webbrowser/browserRecovery.js +58 -0
  292. package/src/tools/webbrowser/browserRecovery.ts +83 -0
  293. package/src/tools/webbrowser/browserRunRegistry.js +35 -3
  294. package/src/tools/webbrowser/browserRunRegistry.ts +91 -15
  295. package/src/tools/webbrowser/browserService.js +28 -2
  296. package/src/tools/webbrowser/browserService.ts +49 -9
  297. package/src/tools/webbrowser/browserStream.js +132 -33
  298. package/src/tools/webbrowser/browserStream.ts +224 -50
  299. package/src/tools/webbrowser/browserTabs.js +48 -15
  300. package/src/tools/webbrowser/browserTabs.ts +178 -96
  301. package/src/tools/webbrowser/webbrowser.js +683 -144
  302. package/src/tools/webbrowser/webbrowser.ts +979 -236
  303. package/src/tools/webfetch/preapproved.js +0 -2
  304. package/src/tools/webfetch/preapproved.ts +0 -2
  305. package/src/tools/websearch/websearch.js +129 -8
  306. package/src/tools/websearch/websearch.ts +138 -6
  307. package/src/types.ts +450 -42
  308. package/src/ui/App.js +377 -58
  309. package/src/ui/App.tsx +576 -111
  310. package/src/ui/LiveArea.js +14 -8
  311. package/src/ui/LiveArea.tsx +14 -10
  312. package/src/ui/MessageList.js +63 -20
  313. package/src/ui/MessageList.tsx +101 -16
  314. package/src/ui/ModelPicker.js +7 -4
  315. package/src/ui/ModelPicker.tsx +55 -8
  316. package/src/ui/hooks/useChat.js +152 -163
  317. package/src/ui/hooks/useChat.ts +396 -334
  318. package/src/ui/loginCommand.js +56 -0
  319. package/src/ui/loginCommand.ts +63 -0
  320. package/src/ui/toolProgress.js +41 -0
  321. package/src/ui/toolProgress.ts +44 -0
  322. package/src/web/browserBridge.js +20 -2
  323. package/src/web/browserBridge.ts +20 -2
  324. package/src/web/protocol.ts +1897 -194
  325. package/workflows/adaptive-research/workflow.mjs +86 -0
  326. package/workflows/e2e-capabilities/workflow.mjs +102 -0
  327. package/workflows/e2e-llm/workflow.mjs +52 -0
  328. package/workflows/e2e-smoke/workflow.mjs +59 -0
  329. package/workflows/e2e-vision-schema/README.md +48 -0
  330. package/workflows/e2e-vision-schema/workflow.mjs +149 -0
  331. package/workflows/e2e-write-greet/workflow.mjs +37 -0
  332. package/workflows/file-census/README.md +14 -0
  333. package/workflows/file-census/scripts/.gitkeep +2 -0
  334. package/workflows/file-census/workflow.mjs +110 -0
  335. package/workflows/goal/README.md +166 -0
  336. package/workflows/goal/prompts/audit.md +57 -0
  337. package/workflows/goal/prompts/contract.md +56 -0
  338. package/workflows/goal/prompts/iterate.md +37 -0
  339. package/workflows/goal/workflow.mjs +501 -0
  340. package/workflows/illustrate/README.md +90 -0
  341. package/workflows/illustrate/assets/engines.json +20 -0
  342. package/workflows/illustrate/assets/extract-baidu.js +21 -0
  343. package/workflows/illustrate/assets/extract-bing.js +33 -0
  344. package/workflows/illustrate/assets/extract-google.js +31 -0
  345. package/workflows/illustrate/workflow.mjs +301 -0
  346. package/workflows/plan/workflow.mjs +389 -0
  347. package/workflows/question-crops/README.md +35 -0
  348. package/workflows/question-crops/prompts/inventory.md +23 -0
  349. package/workflows/question-crops/scripts/image-ops.mjs +305 -0
  350. package/workflows/question-crops/scripts/image-ops.test.mjs +95 -0
  351. package/workflows/question-crops/workflow.mjs +172 -0
  352. package/workflows/ralph-loop/workflow.mjs +162 -0
  353. package/workflows/research/DESIGN.md +449 -0
  354. package/workflows/research/README.md +74 -0
  355. package/workflows/research/prompts/s0-goal.md +43 -0
  356. package/workflows/research/prompts/s1-early-exit.md +22 -0
  357. package/workflows/research/prompts/s1-map.md +58 -0
  358. package/workflows/research/prompts/s2-options.md +53 -0
  359. package/workflows/research/prompts/s3-deep-dive.md +56 -0
  360. package/workflows/research/prompts/s4-verify.md +74 -0
  361. package/workflows/research/prompts/s5-synthesis.md +81 -0
  362. package/workflows/research/prompts/s5b-fill-gaps.md +32 -0
  363. package/workflows/research/prompts/s6-package.md +32 -0
  364. package/workflows/research/scripts/build-report.mjs +98 -0
  365. package/workflows/research/workflow.mjs +288 -0
  366. package/workflows/wechat-reading-shots/prompts/collect-params.md +16 -0
  367. package/workflows/wechat-reading-shots/prompts/evaluate.md +20 -0
  368. package/workflows/wechat-reading-shots/prompts/open-book.md +28 -0
  369. package/workflows/wechat-reading-shots/prompts/read-page.md +13 -0
  370. package/workflows/wechat-reading-shots/prompts/turn-batch.md +24 -0
  371. package/workflows/wechat-reading-shots/test/control-flow.test.ts +329 -0
  372. package/workflows/wechat-reading-shots/workflow.mjs +266 -0
  373. package/workflows/wrong-question-docx/README.md +55 -0
  374. package/workflows/wrong-question-docx/prompts/figure-qa.md +8 -0
  375. package/workflows/wrong-question-docx/prompts/transcribe.md +25 -0
  376. package/workflows/wrong-question-docx/scripts/workbook.mjs +857 -0
  377. package/workflows/wrong-question-docx/test/control-flow.test.ts +165 -0
  378. package/workflows/wrong-question-docx/test/workbook.test.mjs +303 -0
  379. package/workflows/wrong-question-docx/workflow.mjs +485 -0
  380. package/plugins/workflow-runner/src/agentCommon.js +0 -354
  381. package/plugins/workflow-runner/src/agentCommon.ts +0 -435
  382. package/plugins/workflow-runner/src/controlFlow/branch.js +0 -40
  383. package/plugins/workflow-runner/src/controlFlow/branch.ts +0 -46
  384. package/plugins/workflow-runner/src/controlFlow/loop.js +0 -211
  385. package/plugins/workflow-runner/src/controlFlow/loop.ts +0 -248
  386. package/plugins/workflow-runner/src/controlFlow/pause.js +0 -129
  387. package/plugins/workflow-runner/src/controlFlow/pause.ts +0 -141
  388. package/plugins/workflow-runner/src/controlFlow/signal.js +0 -21
  389. package/plugins/workflow-runner/src/controlFlow/signal.ts +0 -29
  390. package/plugins/workflow-runner/src/controlFlow/types.js +0 -18
  391. package/plugins/workflow-runner/src/controlFlow/types.ts +0 -63
  392. package/plugins/workflow-runner/src/dslManifest.js +0 -72
  393. package/plugins/workflow-runner/src/dslManifest.ts +0 -127
  394. package/plugins/workflow-runner/src/expressions.js +0 -618
  395. package/plugins/workflow-runner/src/expressions.ts +0 -622
  396. package/plugins/workflow-runner/src/loader.js +0 -573
  397. package/plugins/workflow-runner/src/loader.ts +0 -721
  398. package/plugins/workflow-runner/src/multimodal.js +0 -38
  399. package/plugins/workflow-runner/src/multimodal.ts +0 -64
  400. package/plugins/workflow-runner/src/recovery.js +0 -119
  401. package/plugins/workflow-runner/src/recovery.ts +0 -149
  402. package/plugins/workflow-runner/src/stepExecutors/agent.js +0 -173
  403. package/plugins/workflow-runner/src/stepExecutors/agent.ts +0 -234
  404. package/plugins/workflow-runner/src/stepExecutors/assert.js +0 -41
  405. package/plugins/workflow-runner/src/stepExecutors/assert.ts +0 -50
  406. package/plugins/workflow-runner/src/stepExecutors/echo.js +0 -31
  407. package/plugins/workflow-runner/src/stepExecutors/echo.ts +0 -38
  408. package/plugins/workflow-runner/src/stepExecutors/llm.js +0 -99
  409. package/plugins/workflow-runner/src/stepExecutors/llm.ts +0 -133
  410. package/plugins/workflow-runner/src/stepExecutors/set.js +0 -49
  411. package/plugins/workflow-runner/src/stepExecutors/set.ts +0 -56
  412. package/plugins/workflow-runner/src/stepExecutors/skill.js +0 -111
  413. package/plugins/workflow-runner/src/stepExecutors/skill.ts +0 -140
  414. package/plugins/workflow-runner/src/stepExecutors/tool.js +0 -208
  415. package/plugins/workflow-runner/src/stepExecutors/tool.ts +0 -256
  416. package/plugins/workflow-runner/src/stepExecutors/verify.js +0 -58
  417. package/plugins/workflow-runner/src/stepExecutors/verify.ts +0 -68
  418. package/plugins/workflow-runner/src/stepPipeline.js +0 -277
  419. package/plugins/workflow-runner/src/stepPipeline.ts +0 -366
  420. package/plugins/workflow-runner/test/agent.test.ts +0 -587
  421. package/plugins/workflow-runner/test/agentLlmError.test.ts +0 -373
  422. package/plugins/workflow-runner/test/bashBlacklist.sweep.test.ts +0 -65
  423. package/plugins/workflow-runner/test/captureMisbind.test.ts +0 -99
  424. package/plugins/workflow-runner/test/cli.e2e.test.ts +0 -147
  425. package/plugins/workflow-runner/test/context-files.test.ts +0 -320
  426. package/plugins/workflow-runner/test/e2e.test.ts +0 -370
  427. package/plugins/workflow-runner/test/engineRobustness.test.ts +0 -226
  428. package/plugins/workflow-runner/test/expressions.test.ts +0 -397
  429. package/plugins/workflow-runner/test/foundation.test.ts +0 -126
  430. package/plugins/workflow-runner/test/graceful.test.ts +0 -139
  431. package/plugins/workflow-runner/test/greedyInputs.load.test.ts +0 -47
  432. package/plugins/workflow-runner/test/hostCapability.test.ts +0 -108
  433. package/plugins/workflow-runner/test/illustrate.load.test.ts +0 -183
  434. package/plugins/workflow-runner/test/jsonPayload.test.ts +0 -83
  435. package/plugins/workflow-runner/test/loader.test.ts +0 -617
  436. package/plugins/workflow-runner/test/maxTurnsRetryDelay.test.ts +0 -33
  437. package/plugins/workflow-runner/test/multimodal.test.ts +0 -342
  438. package/plugins/workflow-runner/test/output-schema.test.ts +0 -213
  439. package/plugins/workflow-runner/test/parseWorkflowArgs.test.ts +0 -87
  440. package/plugins/workflow-runner/test/pluginRunner.isolation.test.ts +0 -269
  441. package/plugins/workflow-runner/test/ppt.load.test.ts +0 -1220
  442. package/plugins/workflow-runner/test/ralphLoop.e2e.test.ts +0 -220
  443. package/plugins/workflow-runner/test/recovery.test.ts +0 -265
  444. package/plugins/workflow-runner/test/runner.test.ts +0 -911
  445. package/plugins/workflow-runner/test/runnerPreflight.test.ts +0 -379
  446. package/plugins/workflow-runner/test/skillTurnDone.test.ts +0 -71
  447. package/plugins/workflow-runner/test/stateLifecycle.test.ts +0 -150
  448. package/plugins/workflow-runner/test/stepPipeline.test.ts +0 -114
  449. package/plugins/workflow-runner/test/taskArbitration.test.ts +0 -338
  450. package/plugins/workflow-runner/test/taskLifecycle.test.ts +0 -270
  451. package/plugins/workflow-runner/test/toolArtifactOverwrite.test.ts +0 -221
  452. package/plugins/workflow-runner/test/toolCapabilityBoundary.test.ts +0 -29
  453. package/plugins/workflow-runner/test/toolRuntimeIntegration.test.ts +0 -168
  454. package/plugins/workflow-runner/test/webppt.load.test.ts +0 -250
  455. package/plugins/workflow-runner/test/word.load.test.ts +0 -208
  456. package/plugins/workflow-runner/test/workflows.smoke.test.ts +0 -76
  457. package/plugins/workflow-runner/test/yamlGuardConditions.test.ts +0 -207
  458. package/src/advisorState.js +0 -22
  459. package/src/cli/configWizard.js +0 -293
  460. package/src/tools/toolProgressBridge.js +0 -48
  461. package/workflows/adaptive-research.yaml +0 -50
  462. package/workflows/book-review-short.yaml +0 -125
  463. package/workflows/e2e-write-greet.yaml +0 -31
  464. package/workflows/illustrate.yaml +0 -227
  465. package/workflows/plan.yaml +0 -470
  466. package/workflows/ppt.yaml +0 -1670
  467. package/workflows/ralph-loop.yaml +0 -199
  468. package/workflows/schema.json +0 -176
  469. package/workflows/webppt.yaml +0 -496
  470. package/workflows/wechat-reading-shots.yaml +0 -315
  471. package/workflows/word.yaml +0 -720
  472. package/workflows/youtube-shorts.yaml +0 -202
@@ -1,5 +1,5 @@
1
+ import { BrowserControlCoordinator, } from './browserControl.js';
1
2
  import { getStreamPort, waitForStreamPort } from './browserDaemon.js';
2
- import { BrowserControlCoordinator } from './browserControl.js';
3
3
  class SessionBrowserStream {
4
4
  sessionName;
5
5
  socket = null;
@@ -8,26 +8,39 @@ class SessionBrowserStream {
8
8
  port = null;
9
9
  mode = 'browse';
10
10
  currentUrl = 'about:blank';
11
- status = { connected: false, screencasting: false };
11
+ status = {
12
+ connected: false,
13
+ screencasting: false,
14
+ };
15
+ reconnectTimer = null;
16
+ reconnectAttempt = 0;
17
+ disposed = false;
12
18
  picker;
13
19
  marked = [];
14
20
  control;
15
21
  constructor(sessionName) {
16
22
  this.sessionName = sessionName;
17
- this.control = new BrowserControlCoordinator({ conversationKey: `browser:${sessionName}` });
23
+ this.control = new BrowserControlCoordinator({
24
+ conversationKey: `browser:${sessionName}`,
25
+ });
18
26
  this.control.subscribe((state) => this.emit({ type: 'control_state', state }));
19
27
  this.control.subscribeInterventions((event) => this.emit({
20
- type: event.type === 'started' ? 'human_intervention_started' : 'human_intervention_resolved',
28
+ type: event.type === 'started'
29
+ ? 'human_intervention_started'
30
+ : 'human_intervention_resolved',
21
31
  event,
22
32
  }));
23
33
  }
24
34
  async connect() {
35
+ this.disposed = false;
25
36
  if (this.socket?.readyState === WebSocket.OPEN)
26
37
  return;
27
38
  if (this.connecting)
28
39
  return this.connecting;
29
40
  this.connecting = (async () => {
30
- this.port = (await getStreamPort(this.sessionName)) ?? (await waitForStreamPort(this.sessionName));
41
+ this.port =
42
+ (await getStreamPort(this.sessionName)) ??
43
+ (await waitForStreamPort(this.sessionName));
31
44
  await new Promise((resolve, reject) => {
32
45
  const socket = new WebSocket(`ws://127.0.0.1:${this.port}`);
33
46
  const timer = setTimeout(() => {
@@ -37,6 +50,7 @@ class SessionBrowserStream {
37
50
  socket.onopen = () => {
38
51
  clearTimeout(timer);
39
52
  this.socket = socket;
53
+ this.reconnectAttempt = 0;
40
54
  this.status = { ...this.status, connected: true };
41
55
  this.emit({ type: 'status', ...this.status });
42
56
  resolve();
@@ -48,16 +62,33 @@ class SessionBrowserStream {
48
62
  socket.onclose = () => {
49
63
  if (this.socket === socket)
50
64
  this.socket = null;
51
- this.status = { ...this.status, connected: false, screencasting: false };
65
+ this.status = {
66
+ ...this.status,
67
+ connected: false,
68
+ screencasting: false,
69
+ };
52
70
  this.emit({ type: 'status', ...this.status });
71
+ this.scheduleReconnect();
53
72
  };
54
73
  socket.onmessage = (event) => this.onMessage(event.data);
55
74
  });
56
- })().finally(() => { this.connecting = null; });
75
+ })().finally(() => {
76
+ this.connecting = null;
77
+ });
57
78
  return this.connecting;
58
79
  }
59
- disconnect() { this.socket?.close(); this.socket = null; }
60
- dispose() { this.disconnect(); this.listeners.clear(); }
80
+ disconnect() {
81
+ this.disposed = true;
82
+ if (this.reconnectTimer)
83
+ clearTimeout(this.reconnectTimer);
84
+ this.reconnectTimer = null;
85
+ this.socket?.close();
86
+ this.socket = null;
87
+ }
88
+ dispose() {
89
+ this.disconnect();
90
+ this.listeners.clear();
91
+ }
61
92
  send(frame) {
62
93
  if (this.socket?.readyState === WebSocket.OPEN)
63
94
  this.socket.send(JSON.stringify(frame));
@@ -66,22 +97,49 @@ class SessionBrowserStream {
66
97
  this.listeners.add(listener);
67
98
  return () => this.listeners.delete(listener);
68
99
  }
69
- getStatus() { return this.status; }
70
- getPort() { return this.port; }
71
- isConnected() { return this.socket?.readyState === WebSocket.OPEN; }
72
- setMode(mode) { this.mode = mode; this.emit({ type: 'mode', mode }); }
73
- getMode() { return this.mode; }
74
- getCurrentUrl() { return this.currentUrl; }
75
- setCurrentUrl(url) { this.currentUrl = url; this.emit({ type: 'url', url }); }
76
- setPicker(picker) { this.picker = picker; }
100
+ publish(event) {
101
+ this.emit(event);
102
+ }
103
+ getStatus() {
104
+ return this.status;
105
+ }
106
+ getPort() {
107
+ return this.port;
108
+ }
109
+ isConnected() {
110
+ return this.socket?.readyState === WebSocket.OPEN;
111
+ }
112
+ setMode(mode) {
113
+ this.mode = mode;
114
+ this.emit({ type: 'mode', mode });
115
+ }
116
+ getMode() {
117
+ return this.mode;
118
+ }
119
+ getCurrentUrl() {
120
+ return this.currentUrl;
121
+ }
122
+ setCurrentUrl(url) {
123
+ this.currentUrl = url;
124
+ this.emit({ type: 'url', url });
125
+ }
126
+ setPicker(picker) {
127
+ this.picker = picker;
128
+ }
77
129
  pickElement(x, y) {
78
130
  return this.picker ? this.picker(x, y) : Promise.resolve(null);
79
131
  }
80
132
  markElement(element) {
81
- this.marked = [...this.marked.filter((item) => item.ref !== element.ref), element];
133
+ this.marked = [
134
+ ...this.marked.filter((item) => item.ref !== element.ref),
135
+ element,
136
+ ];
82
137
  this.emit({ type: 'marked_elements', elements: this.marked });
83
138
  }
84
- clearMarkedElements() { this.marked = []; this.emit({ type: 'marked_elements', elements: [] }); }
139
+ clearMarkedElements() {
140
+ this.marked = [];
141
+ this.emit({ type: 'marked_elements', elements: [] });
142
+ }
85
143
  unmarkElement(ref) {
86
144
  const next = this.marked.filter((item) => item.ref !== ref);
87
145
  if (next.length !== this.marked.length) {
@@ -102,29 +160,67 @@ class SessionBrowserStream {
102
160
  // 只发语义终点(目标元素中心 + 动作类型);平滑移动交给面板 CSS 过渡,内核不做逐帧插值(避免 WS 洪泛 + 双层缓动打架)。
103
161
  this.emit({ type: 'agent_pointer', ...pointer });
104
162
  }
105
- getControlState() { return this.control.getState(); }
106
- subscribeControl(listener) { return this.control.subscribe(listener); }
107
- subscribeInterventions(listener) { return this.control.subscribeInterventions(listener); }
108
- beginAgentAction(input) { this.control.beginAgent(input); }
109
- finishAgentAction(runId) { this.control.finishAgent(runId); }
110
- requestHuman(reason, input) { return this.control.requestHuman(reason, input); }
111
- takeControl(input) { this.control.takeInteractive(input); }
112
- resumeControl(token) { return this.control.resume(token); }
113
- stopControl(token) { return this.control.stop(token); }
163
+ getControlState() {
164
+ return this.control.getState();
165
+ }
166
+ subscribeControl(listener) {
167
+ return this.control.subscribe(listener);
168
+ }
169
+ subscribeInterventions(listener) {
170
+ return this.control.subscribeInterventions(listener);
171
+ }
172
+ beginAgentAction(input) {
173
+ this.control.beginAgent(input);
174
+ }
175
+ finishAgentAction(runId) {
176
+ this.control.finishAgent(runId);
177
+ }
178
+ requestHuman(reason, input) {
179
+ return this.control.requestHuman(reason, input);
180
+ }
181
+ takeControl(input) {
182
+ this.control.takeInteractive(input);
183
+ }
184
+ resumeControl(token) {
185
+ return this.control.resume(token);
186
+ }
187
+ stopControl(token) {
188
+ return this.control.stop(token);
189
+ }
114
190
  resumeIntervention(credential) {
115
191
  return this.control.resumeIntervention(credential);
116
192
  }
117
193
  cancelIntervention(credential) {
118
194
  return this.control.cancelIntervention(credential);
119
195
  }
120
- waitForHuman(signal) { return this.control.waitForHuman(signal); }
196
+ waitForHuman(signal) {
197
+ return this.control.waitForHuman(signal);
198
+ }
121
199
  emit(event) {
122
- for (const listener of this.listeners)
123
- listener(event);
200
+ for (const listener of this.listeners) {
201
+ try {
202
+ listener(event);
203
+ }
204
+ catch {
205
+ // A disconnected/failed UI observer must not break browser control or
206
+ // prevent the remaining observers from receiving the event.
207
+ }
208
+ }
209
+ }
210
+ scheduleReconnect() {
211
+ if (this.disposed || !this.listeners.size || this.reconnectTimer)
212
+ return;
213
+ const delay = Math.min(10_000, 250 * 2 ** this.reconnectAttempt++);
214
+ this.reconnectTimer = setTimeout(() => {
215
+ this.reconnectTimer = null;
216
+ void this.connect().catch(() => this.scheduleReconnect());
217
+ }, delay);
124
218
  }
125
219
  onMessage(data) {
126
220
  try {
127
- const text = typeof data === 'string' ? data : Buffer.from(data).toString('utf8');
221
+ const text = typeof data === 'string'
222
+ ? data
223
+ : Buffer.from(data).toString('utf8');
128
224
  const event = JSON.parse(text);
129
225
  if (event.type === 'url' && typeof event.url === 'string')
130
226
  this.currentUrl = event.url;
@@ -133,7 +229,10 @@ class SessionBrowserStream {
133
229
  this.emit(event);
134
230
  }
135
231
  catch (error) {
136
- this.emit({ type: 'error', message: error instanceof Error ? error.message : String(error) });
232
+ this.emit({
233
+ type: 'error',
234
+ message: error instanceof Error ? error.message : String(error),
235
+ });
137
236
  }
138
237
  }
139
238
  }
@@ -1,6 +1,13 @@
1
+ import type {
2
+ HumanInterventionCredential,
3
+ HumanInterventionEvent,
4
+ } from '../../humanIntervention.js';
5
+ import {
6
+ BrowserControlCoordinator,
7
+ type BrowserControlState,
8
+ type BrowserHumanReason,
9
+ } from './browserControl.js';
1
10
  import { getStreamPort, waitForStreamPort } from './browserDaemon.js';
2
- import { BrowserControlCoordinator, type BrowserControlState, type BrowserHumanReason } from './browserControl.js';
3
- import type { HumanInterventionCredential, HumanInterventionEvent } from '../../humanIntervention.js';
4
11
 
5
12
  export type BrowserMode = 'browse' | 'assist' | 'auto';
6
13
  export type PickedBrowserElement = {
@@ -12,7 +19,10 @@ export type PickedBrowserElement = {
12
19
  boundingBox?: { x: number; y: number; width: number; height: number };
13
20
  attributes?: Record<string, string>;
14
21
  };
15
- export type MarkedBrowserElement = PickedBrowserElement & { label?: string; color?: string };
22
+ export type MarkedBrowserElement = PickedBrowserElement & {
23
+ label?: string;
24
+ color?: string;
25
+ };
16
26
  export type BrowserStreamEvent = Record<string, unknown> & { type: string };
17
27
 
18
28
  export interface BrowserStream {
@@ -21,6 +31,8 @@ export interface BrowserStream {
21
31
  send(frame: unknown): void;
22
32
  dispose(): void;
23
33
  subscribe(listener: (event: BrowserStreamEvent) => void): () => void;
34
+ /** Publish a local engine event to Desktop observers (downloads/CDP lifecycle). */
35
+ publish(event: BrowserStreamEvent): void;
24
36
  getStatus(): Record<string, unknown>;
25
37
  getPort(): number | null;
26
38
  isConnected(): boolean;
@@ -29,7 +41,9 @@ export interface BrowserStream {
29
41
  getCurrentUrl(): string;
30
42
  setCurrentUrl(url: string): void;
31
43
  pickElement(x: number, y: number): Promise<PickedBrowserElement | null>;
32
- setPicker(picker: (x: number, y: number) => Promise<PickedBrowserElement | null>): void;
44
+ setPicker(
45
+ picker: (x: number, y: number) => Promise<PickedBrowserElement | null>,
46
+ ): void;
33
47
  markElement(element: MarkedBrowserElement): void;
34
48
  clearMarkedElements(): void;
35
49
  /** Remove a single marked element by ref (does not affect other marks). */
@@ -41,7 +55,10 @@ export interface BrowserStream {
41
55
  * 候选只进对话的话,盯着浏览器的人看到的是 AI 忽然停住;推到面板上他可以一眼看出
42
56
  * AI 卡在哪,并点一下把正确的那个喂回去 —— 人的判断比模型再猜一轮便宜得多。
43
57
  */
44
- setAgentCandidates(candidates: Array<{ ref: string; line: string }>, selector?: string): void;
58
+ setAgentCandidates(
59
+ candidates: Array<{ ref: string; line: string }>,
60
+ selector?: string,
61
+ ): void;
45
62
  /**
46
63
  * 发布一次动作**做完之后**的生效结论(只在结论不是「变了」时调用)。
47
64
  *
@@ -55,14 +72,34 @@ export interface BrowserStream {
55
72
  maskedLines?: number;
56
73
  }): void;
57
74
  /** C4:发布 AI 指针语义终点给可见面板(宿主用 CSS 过渡平滑移动 + 渲染点击动效)。 */
58
- setAgentPointer(pointer: { x: number; y: number; visible: boolean; action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle'; elementRef?: string }): void;
75
+ setAgentPointer(pointer: {
76
+ x: number;
77
+ y: number;
78
+ visible: boolean;
79
+ action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle';
80
+ elementRef?: string;
81
+ }): void;
59
82
  getControlState(): BrowserControlState;
60
83
  /** 订阅控制权状态变化(立即以当前状态回调一次);供工具即时感知「用户接管 / 进入 waiting_human」以掐断在跑动作。 */
61
84
  subscribeControl(listener: (state: BrowserControlState) => void): () => void;
62
- subscribeInterventions(listener: (event: HumanInterventionEvent) => void): () => void;
63
- beginAgentAction(input: { runId: string; conversationKey?: string; action?: string }): void;
85
+ subscribeInterventions(
86
+ listener: (event: HumanInterventionEvent) => void,
87
+ ): () => void;
88
+ beginAgentAction(input: {
89
+ runId: string;
90
+ conversationKey?: string;
91
+ action?: string;
92
+ }): void;
64
93
  finishAgentAction(runId: string): void;
65
- requestHuman(reason: BrowserHumanReason, input?: { runId?: string; conversationKey?: string; ownerId?: string; summary?: string }): string;
94
+ requestHuman(
95
+ reason: BrowserHumanReason,
96
+ input?: {
97
+ runId?: string;
98
+ conversationKey?: string;
99
+ ownerId?: string;
100
+ summary?: string;
101
+ },
102
+ ): string;
66
103
  takeControl(input?: { runId?: string; conversationKey?: string }): void;
67
104
  resumeControl(token?: string): boolean;
68
105
  stopControl(token?: string): boolean;
@@ -78,25 +115,46 @@ class SessionBrowserStream implements BrowserStream {
78
115
  private port: number | null = null;
79
116
  private mode: BrowserMode = 'browse';
80
117
  private currentUrl = 'about:blank';
81
- private status: Record<string, unknown> = { connected: false, screencasting: false };
82
- private picker?: (x: number, y: number) => Promise<PickedBrowserElement | null>;
118
+ private status: Record<string, unknown> = {
119
+ connected: false,
120
+ screencasting: false,
121
+ };
122
+ private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
123
+ private reconnectAttempt = 0;
124
+ private disposed = false;
125
+ private picker?: (
126
+ x: number,
127
+ y: number,
128
+ ) => Promise<PickedBrowserElement | null>;
83
129
  private marked: MarkedBrowserElement[] = [];
84
130
  private control: BrowserControlCoordinator;
85
131
 
86
132
  constructor(private readonly sessionName: string) {
87
- this.control = new BrowserControlCoordinator({ conversationKey: `browser:${sessionName}` });
88
- this.control.subscribe((state) => this.emit({ type: 'control_state', state }));
89
- this.control.subscribeInterventions((event) => this.emit({
90
- type: event.type === 'started' ? 'human_intervention_started' : 'human_intervention_resolved',
91
- event,
92
- }));
133
+ this.control = new BrowserControlCoordinator({
134
+ conversationKey: `browser:${sessionName}`,
135
+ });
136
+ this.control.subscribe((state) =>
137
+ this.emit({ type: 'control_state', state }),
138
+ );
139
+ this.control.subscribeInterventions((event) =>
140
+ this.emit({
141
+ type:
142
+ event.type === 'started'
143
+ ? 'human_intervention_started'
144
+ : 'human_intervention_resolved',
145
+ event,
146
+ }),
147
+ );
93
148
  }
94
149
 
95
150
  async connect(): Promise<void> {
151
+ this.disposed = false;
96
152
  if (this.socket?.readyState === WebSocket.OPEN) return;
97
153
  if (this.connecting) return this.connecting;
98
154
  this.connecting = (async () => {
99
- this.port = (await getStreamPort(this.sessionName)) ?? (await waitForStreamPort(this.sessionName));
155
+ this.port =
156
+ (await getStreamPort(this.sessionName)) ??
157
+ (await waitForStreamPort(this.sessionName));
100
158
  await new Promise<void>((resolve, reject) => {
101
159
  const socket = new WebSocket(`ws://127.0.0.1:${this.port}`);
102
160
  const timer = setTimeout(() => {
@@ -106,6 +164,7 @@ class SessionBrowserStream implements BrowserStream {
106
164
  socket.onopen = () => {
107
165
  clearTimeout(timer);
108
166
  this.socket = socket;
167
+ this.reconnectAttempt = 0;
109
168
  this.status = { ...this.status, connected: true };
110
169
  this.emit({ type: 'status', ...this.status });
111
170
  resolve();
@@ -116,40 +175,86 @@ class SessionBrowserStream implements BrowserStream {
116
175
  };
117
176
  socket.onclose = () => {
118
177
  if (this.socket === socket) this.socket = null;
119
- this.status = { ...this.status, connected: false, screencasting: false };
178
+ this.status = {
179
+ ...this.status,
180
+ connected: false,
181
+ screencasting: false,
182
+ };
120
183
  this.emit({ type: 'status', ...this.status });
184
+ this.scheduleReconnect();
121
185
  };
122
186
  socket.onmessage = (event) => this.onMessage(event.data);
123
187
  });
124
- })().finally(() => { this.connecting = null; });
188
+ })().finally(() => {
189
+ this.connecting = null;
190
+ });
125
191
  return this.connecting;
126
192
  }
127
193
 
128
- disconnect(): void { this.socket?.close(); this.socket = null; }
129
- dispose(): void { this.disconnect(); this.listeners.clear(); }
194
+ disconnect(): void {
195
+ this.disposed = true;
196
+ if (this.reconnectTimer) clearTimeout(this.reconnectTimer);
197
+ this.reconnectTimer = null;
198
+ this.socket?.close();
199
+ this.socket = null;
200
+ }
201
+ dispose(): void {
202
+ this.disconnect();
203
+ this.listeners.clear();
204
+ }
130
205
  send(frame: unknown): void {
131
- if (this.socket?.readyState === WebSocket.OPEN) this.socket.send(JSON.stringify(frame));
206
+ if (this.socket?.readyState === WebSocket.OPEN)
207
+ this.socket.send(JSON.stringify(frame));
132
208
  }
133
209
  subscribe(listener: (event: BrowserStreamEvent) => void): () => void {
134
210
  this.listeners.add(listener);
135
211
  return () => this.listeners.delete(listener);
136
212
  }
137
- getStatus(): Record<string, unknown> { return this.status; }
138
- getPort(): number | null { return this.port; }
139
- isConnected(): boolean { return this.socket?.readyState === WebSocket.OPEN; }
140
- setMode(mode: BrowserMode): void { this.mode = mode; this.emit({ type: 'mode', mode }); }
141
- getMode(): BrowserMode { return this.mode; }
142
- getCurrentUrl(): string { return this.currentUrl; }
143
- setCurrentUrl(url: string): void { this.currentUrl = url; this.emit({ type: 'url', url }); }
144
- setPicker(picker: (x: number, y: number) => Promise<PickedBrowserElement | null>): void { this.picker = picker; }
213
+ publish(event: BrowserStreamEvent): void {
214
+ this.emit(event);
215
+ }
216
+ getStatus(): Record<string, unknown> {
217
+ return this.status;
218
+ }
219
+ getPort(): number | null {
220
+ return this.port;
221
+ }
222
+ isConnected(): boolean {
223
+ return this.socket?.readyState === WebSocket.OPEN;
224
+ }
225
+ setMode(mode: BrowserMode): void {
226
+ this.mode = mode;
227
+ this.emit({ type: 'mode', mode });
228
+ }
229
+ getMode(): BrowserMode {
230
+ return this.mode;
231
+ }
232
+ getCurrentUrl(): string {
233
+ return this.currentUrl;
234
+ }
235
+ setCurrentUrl(url: string): void {
236
+ this.currentUrl = url;
237
+ this.emit({ type: 'url', url });
238
+ }
239
+ setPicker(
240
+ picker: (x: number, y: number) => Promise<PickedBrowserElement | null>,
241
+ ): void {
242
+ this.picker = picker;
243
+ }
145
244
  pickElement(x: number, y: number): Promise<PickedBrowserElement | null> {
146
245
  return this.picker ? this.picker(x, y) : Promise.resolve(null);
147
246
  }
148
247
  markElement(element: MarkedBrowserElement): void {
149
- this.marked = [...this.marked.filter((item) => item.ref !== element.ref), element];
248
+ this.marked = [
249
+ ...this.marked.filter((item) => item.ref !== element.ref),
250
+ element,
251
+ ];
150
252
  this.emit({ type: 'marked_elements', elements: this.marked });
151
253
  }
152
- clearMarkedElements(): void { this.marked = []; this.emit({ type: 'marked_elements', elements: [] }); }
254
+ clearMarkedElements(): void {
255
+ this.marked = [];
256
+ this.emit({ type: 'marked_elements', elements: [] });
257
+ }
153
258
  unmarkElement(ref: string): void {
154
259
  const next = this.marked.filter((item) => item.ref !== ref);
155
260
  if (next.length !== this.marked.length) {
@@ -160,7 +265,10 @@ class SessionBrowserStream implements BrowserStream {
160
265
  setAgentActing(acting: boolean, action?: string, target?: string): void {
161
266
  this.emit({ type: 'agent_acting', acting, action, target });
162
267
  }
163
- setAgentCandidates(candidates: Array<{ ref: string; line: string }>, selector?: string): void {
268
+ setAgentCandidates(
269
+ candidates: Array<{ ref: string; line: string }>,
270
+ selector?: string,
271
+ ): void {
164
272
  this.emit({ type: 'agent_candidates', candidates, selector });
165
273
  }
166
274
  setAgentOutcome(outcome: {
@@ -171,39 +279,102 @@ class SessionBrowserStream implements BrowserStream {
171
279
  }): void {
172
280
  this.emit({ type: 'agent_outcome', ...outcome });
173
281
  }
174
- setAgentPointer(pointer: { x: number; y: number; visible: boolean; action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle'; elementRef?: string }): void {
282
+ setAgentPointer(pointer: {
283
+ x: number;
284
+ y: number;
285
+ visible: boolean;
286
+ action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle';
287
+ elementRef?: string;
288
+ }): void {
175
289
  // 只发语义终点(目标元素中心 + 动作类型);平滑移动交给面板 CSS 过渡,内核不做逐帧插值(避免 WS 洪泛 + 双层缓动打架)。
176
290
  this.emit({ type: 'agent_pointer', ...pointer });
177
291
  }
178
- getControlState(): BrowserControlState { return this.control.getState(); }
179
- subscribeControl(listener: (state: BrowserControlState) => void): () => void { return this.control.subscribe(listener); }
180
- subscribeInterventions(listener: (event: HumanInterventionEvent) => void): () => void { return this.control.subscribeInterventions(listener); }
181
- beginAgentAction(input: { runId: string; conversationKey?: string; action?: string }): void { this.control.beginAgent(input); }
182
- finishAgentAction(runId: string): void { this.control.finishAgent(runId); }
183
- requestHuman(reason: BrowserHumanReason, input?: { runId?: string; conversationKey?: string; ownerId?: string; summary?: string }): string { return this.control.requestHuman(reason, input); }
184
- takeControl(input?: { runId?: string; conversationKey?: string }): void { this.control.takeInteractive(input); }
185
- resumeControl(token?: string): boolean { return this.control.resume(token); }
186
- stopControl(token?: string): boolean { return this.control.stop(token); }
292
+ getControlState(): BrowserControlState {
293
+ return this.control.getState();
294
+ }
295
+ subscribeControl(listener: (state: BrowserControlState) => void): () => void {
296
+ return this.control.subscribe(listener);
297
+ }
298
+ subscribeInterventions(
299
+ listener: (event: HumanInterventionEvent) => void,
300
+ ): () => void {
301
+ return this.control.subscribeInterventions(listener);
302
+ }
303
+ beginAgentAction(input: {
304
+ runId: string;
305
+ conversationKey?: string;
306
+ action?: string;
307
+ }): void {
308
+ this.control.beginAgent(input);
309
+ }
310
+ finishAgentAction(runId: string): void {
311
+ this.control.finishAgent(runId);
312
+ }
313
+ requestHuman(
314
+ reason: BrowserHumanReason,
315
+ input?: {
316
+ runId?: string;
317
+ conversationKey?: string;
318
+ ownerId?: string;
319
+ summary?: string;
320
+ },
321
+ ): string {
322
+ return this.control.requestHuman(reason, input);
323
+ }
324
+ takeControl(input?: { runId?: string; conversationKey?: string }): void {
325
+ this.control.takeInteractive(input);
326
+ }
327
+ resumeControl(token?: string): boolean {
328
+ return this.control.resume(token);
329
+ }
330
+ stopControl(token?: string): boolean {
331
+ return this.control.stop(token);
332
+ }
187
333
  resumeIntervention(credential: HumanInterventionCredential): boolean {
188
334
  return this.control.resumeIntervention(credential);
189
335
  }
190
336
  cancelIntervention(credential: HumanInterventionCredential): boolean {
191
337
  return this.control.cancelIntervention(credential);
192
338
  }
193
- waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'> { return this.control.waitForHuman(signal); }
339
+ waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'> {
340
+ return this.control.waitForHuman(signal);
341
+ }
194
342
 
195
343
  private emit(event: BrowserStreamEvent): void {
196
- for (const listener of this.listeners) listener(event);
344
+ for (const listener of this.listeners) {
345
+ try {
346
+ listener(event);
347
+ } catch {
348
+ // A disconnected/failed UI observer must not break browser control or
349
+ // prevent the remaining observers from receiving the event.
350
+ }
351
+ }
352
+ }
353
+
354
+ private scheduleReconnect(): void {
355
+ if (this.disposed || !this.listeners.size || this.reconnectTimer) return;
356
+ const delay = Math.min(10_000, 250 * 2 ** this.reconnectAttempt++);
357
+ this.reconnectTimer = setTimeout(() => {
358
+ this.reconnectTimer = null;
359
+ void this.connect().catch(() => this.scheduleReconnect());
360
+ }, delay);
197
361
  }
198
362
  private onMessage(data: unknown): void {
199
363
  try {
200
- const text = typeof data === 'string' ? data : Buffer.from(data as ArrayBuffer).toString('utf8');
364
+ const text =
365
+ typeof data === 'string'
366
+ ? data
367
+ : Buffer.from(data as ArrayBuffer).toString('utf8');
201
368
  const event = JSON.parse(text) as BrowserStreamEvent;
202
- if (event.type === 'url' && typeof event.url === 'string') this.currentUrl = event.url;
369
+ if (event.type === 'url' && typeof event.url === 'string')
370
+ this.currentUrl = event.url;
203
371
  if (event.type === 'status') this.status = { ...this.status, ...event };
204
372
  this.emit(event);
205
373
  } catch (error) {
206
- this.emit({ type: 'error', message: error instanceof Error ? error.message : String(error) });
374
+ this.emit({
375
+ type: 'error',
376
+ message: error instanceof Error ? error.message : String(error),
377
+ });
207
378
  }
208
379
  }
209
380
  }
@@ -211,6 +382,9 @@ class SessionBrowserStream implements BrowserStream {
211
382
  const streams = new Map<string, SessionBrowserStream>();
212
383
  export function getBrowserStream(sessionName = 'minimal-agent'): BrowserStream {
213
384
  let stream = streams.get(sessionName);
214
- if (!stream) { stream = new SessionBrowserStream(sessionName); streams.set(sessionName, stream); }
385
+ if (!stream) {
386
+ stream = new SessionBrowserStream(sessionName);
387
+ streams.set(sessionName, stream);
388
+ }
215
389
  return stream;
216
390
  }