minimal-agent 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (573) hide show
  1. package/README.md +11 -3
  2. package/package.json +52 -5
  3. package/plugins/computer_use/.claude-plugin/plugin.json +6 -0
  4. package/plugins/computer_use/README.md +94 -0
  5. package/plugins/computer_use/app/_template.md +142 -0
  6. package/plugins/computer_use/app/chrome.md +165 -0
  7. package/plugins/computer_use/app/finder.md +76 -0
  8. package/plugins/computer_use/app/macos-system-settings.md +81 -0
  9. package/plugins/computer_use/app/microsoft-excel.md +80 -0
  10. package/plugins/computer_use/app/microsoft-powerpoint.md +80 -0
  11. package/plugins/computer_use/app/microsoft-word.md +79 -0
  12. package/plugins/computer_use/app/wechat.md +413 -0
  13. package/plugins/computer_use/app/weread.md +319 -0
  14. package/plugins/computer_use/commands/computeruse.md +25 -0
  15. package/plugins/computer_use/commands/cu.md +9 -0
  16. package/plugins/computer_use/plugin.js +26 -0
  17. package/plugins/computer_use/plugin.ts +30 -0
  18. package/plugins/computer_use/src/capture.js +90 -0
  19. package/plugins/computer_use/src/capture.ts +104 -0
  20. package/plugins/computer_use/src/commands.js +91 -0
  21. package/plugins/computer_use/src/commands.ts +103 -0
  22. package/plugins/computer_use/src/daemon.js +134 -0
  23. package/plugins/computer_use/src/daemon.ts +161 -0
  24. package/plugins/computer_use/src/manuals.js +135 -0
  25. package/plugins/computer_use/src/manuals.ts +159 -0
  26. package/plugins/computer_use/src/verify.js +168 -0
  27. package/plugins/computer_use/src/verify.ts +219 -0
  28. package/plugins/computer_use/test/commands.test.ts +73 -0
  29. package/plugins/cron-runner/plugin.ts +38 -0
  30. package/plugins/cron-runner/src/commands.ts +373 -0
  31. package/plugins/cron-runner/src/cronTime.ts +155 -0
  32. package/plugins/cron-runner/src/lock.ts +79 -0
  33. package/plugins/cron-runner/src/runs.ts +99 -0
  34. package/plugins/cron-runner/src/scheduleEngine.ts +76 -0
  35. package/plugins/cron-runner/src/scheduler.ts +376 -0
  36. package/plugins/cron-runner/src/schema.ts +139 -0
  37. package/plugins/cron-runner/src/service.ts +193 -0
  38. package/plugins/cron-runner/src/store.ts +333 -0
  39. package/plugins/cron-runner/src/types.ts +141 -0
  40. package/plugins/cron-runner/test/cronTime.test.ts +41 -0
  41. package/plugins/cron-runner/test/plugin-integration.test.ts +47 -0
  42. package/plugins/cron-runner/test/scheduleEngine.test.ts +41 -0
  43. package/plugins/cron-runner/test/service.test.ts +76 -0
  44. package/plugins/cron-runner/test/store-and-scheduler.test.ts +501 -0
  45. package/plugins/workflow-runner/plugin.js +56 -2
  46. package/plugins/workflow-runner/plugin.ts +113 -0
  47. package/plugins/workflow-runner/src/agentCommon.js +43 -4
  48. package/plugins/workflow-runner/src/agentCommon.ts +435 -0
  49. package/plugins/workflow-runner/src/controlFlow/branch.js +40 -0
  50. package/plugins/workflow-runner/src/controlFlow/branch.ts +46 -0
  51. package/plugins/workflow-runner/src/controlFlow/loop.js +211 -0
  52. package/plugins/workflow-runner/src/controlFlow/loop.ts +248 -0
  53. package/plugins/workflow-runner/src/controlFlow/pause.js +129 -0
  54. package/plugins/workflow-runner/src/controlFlow/pause.ts +141 -0
  55. package/plugins/workflow-runner/src/controlFlow/signal.js +21 -0
  56. package/plugins/workflow-runner/src/controlFlow/signal.ts +29 -0
  57. package/plugins/workflow-runner/src/controlFlow/types.js +18 -0
  58. package/plugins/workflow-runner/src/controlFlow/types.ts +63 -0
  59. package/plugins/workflow-runner/src/dslManifest.ts +127 -0
  60. package/plugins/workflow-runner/src/expressions.ts +622 -0
  61. package/plugins/workflow-runner/src/index.js +13 -4
  62. package/plugins/workflow-runner/src/index.ts +338 -0
  63. package/plugins/workflow-runner/src/jsonPayload.ts +84 -0
  64. package/plugins/workflow-runner/src/loader.js +23 -0
  65. package/plugins/workflow-runner/src/loader.ts +721 -0
  66. package/plugins/workflow-runner/src/multimodal.js +2 -1
  67. package/plugins/workflow-runner/src/multimodal.ts +64 -0
  68. package/plugins/workflow-runner/src/recovery.ts +149 -0
  69. package/plugins/workflow-runner/src/runner.js +75 -659
  70. package/plugins/workflow-runner/src/runner.ts +604 -0
  71. package/plugins/workflow-runner/src/stepExecutors/agent.ts +234 -0
  72. package/plugins/workflow-runner/src/stepExecutors/assert.ts +50 -0
  73. package/plugins/workflow-runner/src/stepExecutors/echo.js +31 -0
  74. package/plugins/workflow-runner/src/stepExecutors/echo.ts +38 -0
  75. package/plugins/workflow-runner/src/stepExecutors/llm.ts +133 -0
  76. package/plugins/workflow-runner/src/stepExecutors/set.js +49 -0
  77. package/plugins/workflow-runner/src/stepExecutors/set.ts +56 -0
  78. package/plugins/workflow-runner/src/stepExecutors/skill.js +5 -6
  79. package/plugins/workflow-runner/src/stepExecutors/skill.ts +140 -0
  80. package/plugins/workflow-runner/src/stepExecutors/tool.js +80 -8
  81. package/plugins/workflow-runner/src/stepExecutors/tool.ts +256 -0
  82. package/plugins/workflow-runner/src/stepExecutors/verify.ts +68 -0
  83. package/plugins/workflow-runner/src/stepPipeline.js +277 -0
  84. package/plugins/workflow-runner/src/stepPipeline.ts +366 -0
  85. package/plugins/workflow-runner/src/taskIdentity.ts +405 -0
  86. package/plugins/workflow-runner/src/toolRuntime.js +48 -0
  87. package/plugins/workflow-runner/src/toolRuntime.ts +64 -0
  88. package/plugins/workflow-runner/src/types.js +37 -1
  89. package/plugins/workflow-runner/src/types.ts +602 -0
  90. package/plugins/workflow-runner/src/workflowState.ts +733 -0
  91. package/plugins/workflow-runner/test/agent.test.ts +587 -0
  92. package/plugins/workflow-runner/test/agentLlmError.test.ts +373 -0
  93. package/plugins/workflow-runner/test/bashBlacklist.sweep.test.ts +65 -0
  94. package/plugins/workflow-runner/test/captureMisbind.test.ts +99 -0
  95. package/plugins/workflow-runner/test/cli.e2e.test.ts +147 -0
  96. package/plugins/workflow-runner/test/context-files.test.ts +320 -0
  97. package/plugins/workflow-runner/test/e2e.test.ts +370 -0
  98. package/plugins/workflow-runner/test/engineRobustness.test.ts +226 -0
  99. package/plugins/workflow-runner/test/expressions.test.ts +397 -0
  100. package/plugins/workflow-runner/test/foundation.test.ts +126 -0
  101. package/plugins/workflow-runner/test/graceful.test.ts +139 -0
  102. package/plugins/workflow-runner/test/greedyInputs.load.test.ts +47 -0
  103. package/plugins/workflow-runner/test/hostCapability.test.ts +108 -0
  104. package/plugins/workflow-runner/test/illustrate.load.test.ts +183 -0
  105. package/plugins/workflow-runner/test/jsonPayload.test.ts +83 -0
  106. package/plugins/workflow-runner/test/loader.test.ts +617 -0
  107. package/plugins/workflow-runner/test/maxTurnsRetryDelay.test.ts +33 -0
  108. package/plugins/workflow-runner/test/multimodal.test.ts +342 -0
  109. package/plugins/workflow-runner/test/output-schema.test.ts +213 -0
  110. package/plugins/workflow-runner/test/parseWorkflowArgs.test.ts +87 -0
  111. package/plugins/workflow-runner/test/pluginRunner.isolation.test.ts +269 -0
  112. package/plugins/workflow-runner/test/ppt.load.test.ts +1220 -0
  113. package/plugins/workflow-runner/test/ralphLoop.e2e.test.ts +220 -0
  114. package/plugins/workflow-runner/test/recovery.test.ts +265 -0
  115. package/plugins/workflow-runner/test/runner.test.ts +911 -0
  116. package/plugins/workflow-runner/test/runnerPreflight.test.ts +379 -0
  117. package/plugins/workflow-runner/test/skillTurnDone.test.ts +71 -0
  118. package/plugins/workflow-runner/test/stateLifecycle.test.ts +150 -0
  119. package/plugins/workflow-runner/test/stepPipeline.test.ts +114 -0
  120. package/plugins/workflow-runner/test/taskArbitration.test.ts +338 -0
  121. package/plugins/workflow-runner/test/taskIdentity.test.ts +237 -0
  122. package/plugins/workflow-runner/test/taskLifecycle.test.ts +270 -0
  123. package/plugins/workflow-runner/test/toolArtifactOverwrite.test.ts +221 -0
  124. package/plugins/workflow-runner/test/toolCapabilityBoundary.test.ts +29 -0
  125. package/plugins/workflow-runner/test/toolRuntimeIntegration.test.ts +168 -0
  126. package/plugins/workflow-runner/test/webppt.load.test.ts +250 -0
  127. package/plugins/workflow-runner/test/word.load.test.ts +208 -0
  128. package/plugins/workflow-runner/test/workflows.smoke.test.ts +76 -0
  129. package/plugins/workflow-runner/test/yamlGuardConditions.test.ts +207 -0
  130. package/skills/browser-automation/SKILL.md +9 -19
  131. package/skills/web-crawler/SKILL.md +9 -11
  132. package/src/askBridge.ts +249 -0
  133. package/src/bootstrap/cwdArg.js +10 -3
  134. package/src/bootstrap/cwdArg.ts +31 -0
  135. package/src/bootstrap/workingDir.ts +48 -0
  136. package/src/cli/args.js +70 -7
  137. package/src/cli/args.ts +146 -0
  138. package/src/cli/computerUse.js +185 -0
  139. package/src/cli/computerUse.ts +221 -0
  140. package/src/cli/configMenu.js +274 -202
  141. package/src/cli/configMenu.tsx +521 -0
  142. package/src/cli/memoryAccess.js +45 -0
  143. package/src/cli/memoryAccess.ts +53 -0
  144. package/src/cli/print.js +119 -5
  145. package/src/cli/print.ts +597 -0
  146. package/src/cli/quickModelSetup.js +60 -0
  147. package/src/cli/quickModelSetup.tsx +96 -0
  148. package/src/cli/streamJson.js +80 -4
  149. package/src/cli/streamJson.ts +433 -0
  150. package/src/cli/wizards/advisorWizard.js +1 -0
  151. package/src/cli/wizards/advisorWizard.tsx +195 -0
  152. package/src/cli/wizards/agentRuntimeWizard.js +84 -0
  153. package/src/cli/wizards/agentRuntimeWizard.tsx +134 -0
  154. package/src/cli/wizards/browserWizard.js +55 -0
  155. package/src/cli/wizards/browserWizard.tsx +77 -0
  156. package/src/cli/wizards/contextAdvisorWizard.js +84 -0
  157. package/src/cli/wizards/contextAdvisorWizard.tsx +124 -0
  158. package/src/cli/wizards/fieldForm.js +7 -4
  159. package/src/cli/wizards/fieldForm.tsx +250 -0
  160. package/src/cli/wizards/imageWizard.js +4 -5
  161. package/src/cli/wizards/imageWizard.tsx +96 -0
  162. package/src/cli/wizards/openRouterWizard.tsx +48 -0
  163. package/src/cli/wizards/providerPresets.js +1 -8
  164. package/src/cli/wizards/providerPresets.ts +106 -0
  165. package/src/cli/wizards/providerWizard.js +152 -205
  166. package/src/cli/wizards/providerWizard.tsx +266 -0
  167. package/src/cli/wizards/searchWizard.tsx +47 -0
  168. package/src/cli/wizards/ttsWizard.js +4 -3
  169. package/src/cli/wizards/ttsWizard.tsx +61 -0
  170. package/src/config/configDefaults.js +25 -0
  171. package/src/config/configDefaults.ts +33 -0
  172. package/src/config/configDiagnostics.js +117 -0
  173. package/src/config/configDiagnostics.ts +166 -0
  174. package/src/config/configFile.js +52 -32
  175. package/src/config/configFile.ts +633 -0
  176. package/src/config/configManagement.js +126 -0
  177. package/src/config/configManagement.ts +159 -0
  178. package/src/config/configService.js +66 -11
  179. package/src/config/configService.ts +554 -0
  180. package/src/config/numericLimits.js +61 -0
  181. package/src/config/numericLimits.ts +80 -0
  182. package/src/config.ts +109 -0
  183. package/src/context/agentHome.ts +11 -0
  184. package/src/context/archive.ts +144 -0
  185. package/src/context/budgetPlan.js +54 -0
  186. package/src/context/budgetPlan.ts +88 -0
  187. package/src/context/compact.js +179 -106
  188. package/src/context/compact.ts +584 -0
  189. package/src/context/contextBundle.js +119 -0
  190. package/src/context/contextBundle.ts +176 -0
  191. package/src/context/contextLock.ts +84 -0
  192. package/src/context/microCompactLite.ts +218 -0
  193. package/src/context/persistContext.ts +177 -0
  194. package/src/context/reactiveCompact.js +19 -4
  195. package/src/context/reactiveCompact.ts +240 -0
  196. package/src/context/recentDirs.ts +42 -0
  197. package/src/context/sessionPath.ts +206 -0
  198. package/src/context/sessionRegistry.js +8 -2
  199. package/src/context/sessionRegistry.ts +414 -0
  200. package/src/context/snipCompact.ts +163 -0
  201. package/src/context/tokenCounter.ts +106 -0
  202. package/src/context/workspaceIndex.js +2 -2
  203. package/src/context/workspaceIndex.ts +751 -0
  204. package/src/entry.compiled.tsx +20 -0
  205. package/src/harness/claimValidation.js +9 -0
  206. package/src/harness/claimValidation.ts +22 -0
  207. package/src/harness/evidenceLedger.js +92 -0
  208. package/src/harness/evidenceLedger.ts +96 -0
  209. package/src/harness/stallGuard.js +60 -0
  210. package/src/harness/stallGuard.ts +81 -0
  211. package/src/humanIntervention.js +129 -0
  212. package/src/humanIntervention.ts +216 -0
  213. package/src/infra/verification.ts +295 -0
  214. package/src/irreversible/rules.js +216 -0
  215. package/src/irreversible/rules.ts +261 -0
  216. package/src/llm/client.js +40 -3
  217. package/src/llm/client.ts +621 -0
  218. package/src/llm/promptCache.js +27 -0
  219. package/src/llm/promptCache.ts +41 -0
  220. package/src/llm/providers/anthropicFormat.js +41 -5
  221. package/src/llm/providers/anthropicFormat.ts +290 -0
  222. package/src/llm/providers/openaiFormat.js +186 -8
  223. package/src/llm/providers/openaiFormat.ts +428 -0
  224. package/src/loop.js +614 -63
  225. package/src/loop.ts +1454 -0
  226. package/src/main.js +67 -10
  227. package/src/main.tsx +344 -0
  228. package/src/memory/codec.js +204 -0
  229. package/src/memory/codec.ts +249 -0
  230. package/src/memory/constants.js +34 -0
  231. package/src/memory/constants.ts +50 -0
  232. package/src/memory/episodicMemory.js +684 -0
  233. package/src/memory/episodicMemory.ts +815 -0
  234. package/src/memory/fileStore.js +148 -0
  235. package/src/memory/fileStore.ts +184 -0
  236. package/src/memory/memoryService.js +570 -0
  237. package/src/memory/memoryService.ts +859 -0
  238. package/src/memory/migration.js +128 -0
  239. package/src/memory/migration.ts +160 -0
  240. package/src/memory/repository.js +789 -0
  241. package/src/memory/repository.ts +893 -0
  242. package/src/memory/retrieval.js +108 -0
  243. package/src/memory/retrieval.ts +145 -0
  244. package/src/memory/searchIndex.js +107 -0
  245. package/src/memory/searchIndex.ts +153 -0
  246. package/src/memory/topicView.js +82 -0
  247. package/src/memory/topicView.ts +95 -0
  248. package/src/memory/turnJournal.js +95 -0
  249. package/src/memory/turnJournal.ts +120 -0
  250. package/src/memory/types.js +2 -0
  251. package/src/memory/types.ts +186 -0
  252. package/src/messageQueue.ts +123 -0
  253. package/src/multimodal.js +128 -46
  254. package/src/multimodal.ts +349 -0
  255. package/src/plugin-sdk.js +21 -1
  256. package/src/plugin-sdk.ts +134 -0
  257. package/src/plugins/builtinPluginIds.js +47 -0
  258. package/src/plugins/builtinPluginIds.ts +51 -0
  259. package/src/plugins/builtinRegistry.ts +43 -0
  260. package/src/plugins/commandRouter.js +40 -26
  261. package/src/plugins/commandRouter.ts +261 -0
  262. package/src/plugins/hookEngine.ts +341 -0
  263. package/src/plugins/pluginApi.ts +133 -0
  264. package/src/plugins/pluginLifecycle.ts +34 -0
  265. package/src/plugins/pluginLoader.ts +147 -0
  266. package/src/plugins/pluginManifest.js +127 -0
  267. package/src/plugins/pluginManifest.ts +152 -0
  268. package/src/plugins/pluginRunner.js +47 -2
  269. package/src/plugins/pluginRunner.ts +207 -0
  270. package/src/plugins/transcript.ts +208 -0
  271. package/src/prompts/lint.js +169 -0
  272. package/src/prompts/lint.ts +220 -0
  273. package/src/prompts/pluginList.js +32 -16
  274. package/src/prompts/pluginList.ts +86 -0
  275. package/src/prompts/policyRegistry.js +318 -0
  276. package/src/prompts/policyRegistry.ts +358 -0
  277. package/src/prompts/projectInstructions.js +24 -0
  278. package/src/prompts/projectInstructions.ts +81 -0
  279. package/src/prompts/promptAssembly.js +311 -202
  280. package/src/prompts/promptAssembly.ts +510 -0
  281. package/src/prompts/skillList.js +106 -45
  282. package/src/prompts/skillList.ts +198 -0
  283. package/src/prompts/system.js +31 -6
  284. package/src/prompts/system.ts +81 -0
  285. package/src/runtime/agentRuntimeContext.js +53 -0
  286. package/src/runtime/agentRuntimeContext.ts +112 -0
  287. package/src/session/runAgentTurn.js +71 -12
  288. package/src/session/runAgentTurn.ts +181 -0
  289. package/src/session/runTurn.js +29 -0
  290. package/src/session/runTurn.ts +160 -0
  291. package/src/session/runtimeStore.js +149 -0
  292. package/src/session/runtimeStore.ts +197 -0
  293. package/src/session/sessionState.js +79 -3
  294. package/src/session/sessionState.ts +134 -0
  295. package/src/toolGate.ts +50 -0
  296. package/src/tools/advisor/advisor.js +76 -15
  297. package/src/tools/advisor/advisor.ts +512 -0
  298. package/src/tools/agent/agent.js +245 -29
  299. package/src/tools/agent/agent.ts +837 -0
  300. package/src/tools/agent/agentTaskService.js +60 -0
  301. package/src/tools/agent/agentTaskService.ts +113 -0
  302. package/src/tools/agent/agentTypes.js +20 -8
  303. package/src/tools/agent/agentTypes.ts +94 -0
  304. package/src/tools/agent/coordinator.ts +111 -0
  305. package/src/tools/agent/worktree.ts +296 -0
  306. package/src/tools/ask/ask.js +17 -0
  307. package/src/tools/ask/ask.ts +122 -0
  308. package/src/tools/bash/attachImage.js +223 -0
  309. package/src/tools/bash/attachImage.ts +281 -0
  310. package/src/tools/bash/bash.js +132 -14
  311. package/src/tools/bash/bash.ts +642 -0
  312. package/src/tools/bash/semantics.ts +103 -0
  313. package/src/tools/bash/warnings.js +17 -11
  314. package/src/tools/bash/warnings.ts +117 -0
  315. package/src/tools/capabilities.js +42 -0
  316. package/src/tools/capabilities.ts +69 -0
  317. package/src/tools/computeruse/accessibilityPermission.js +261 -0
  318. package/src/tools/computeruse/accessibilityPermission.ts +294 -0
  319. package/src/tools/computeruse/axSearch.js +96 -0
  320. package/src/tools/computeruse/axSearch.ts +120 -0
  321. package/src/tools/computeruse/axSnap.js +272 -0
  322. package/src/tools/computeruse/axSnap.ts +349 -0
  323. package/src/tools/computeruse/capabilityLabels.js +95 -0
  324. package/src/tools/computeruse/capabilityLabels.ts +115 -0
  325. package/src/tools/computeruse/computerUseService.js +2235 -0
  326. package/src/tools/computeruse/computerUseService.ts +2686 -0
  327. package/src/tools/computeruse/computeruse.js +2742 -0
  328. package/src/tools/computeruse/computeruse.ts +3265 -0
  329. package/src/tools/computeruse/diagnosticTrace.js +154 -0
  330. package/src/tools/computeruse/diagnosticTrace.ts +195 -0
  331. package/src/tools/computeruse/diagnostics.js +369 -0
  332. package/src/tools/computeruse/diagnostics.ts +453 -0
  333. package/src/tools/computeruse/grants.js +53 -0
  334. package/src/tools/computeruse/grants.ts +70 -0
  335. package/src/tools/computeruse/humanBoundary.js +159 -0
  336. package/src/tools/computeruse/humanBoundary.ts +199 -0
  337. package/src/tools/computeruse/intervention.js +77 -0
  338. package/src/tools/computeruse/intervention.ts +110 -0
  339. package/src/tools/computeruse/manualSchema.js +709 -0
  340. package/src/tools/computeruse/manualSchema.ts +916 -0
  341. package/src/tools/computeruse/observationImages.js +705 -0
  342. package/src/tools/computeruse/observationImages.ts +889 -0
  343. package/src/tools/computeruse/protocol.js +60 -0
  344. package/src/tools/computeruse/protocol.ts +515 -0
  345. package/src/tools/computeruse/telemetryLog.js +57 -0
  346. package/src/tools/computeruse/telemetryLog.ts +67 -0
  347. package/src/tools/computeruse/vision/appGuide.js +325 -0
  348. package/src/tools/computeruse/vision/appGuide.ts +482 -0
  349. package/src/tools/computeruse/vision/focus.js +291 -0
  350. package/src/tools/computeruse/vision/focus.ts +382 -0
  351. package/src/tools/computeruse/vision/frames.js +248 -0
  352. package/src/tools/computeruse/vision/frames.ts +329 -0
  353. package/src/tools/computeruse/vision/geometry.js +131 -0
  354. package/src/tools/computeruse/vision/geometry.ts +163 -0
  355. package/src/tools/computeruse/vision/imaging.js +201 -0
  356. package/src/tools/computeruse/vision/imaging.ts +281 -0
  357. package/src/tools/computeruse/vision/png.js +274 -0
  358. package/src/tools/computeruse/vision/png.ts +317 -0
  359. package/src/tools/computeruse/vision/raster.js +320 -0
  360. package/src/tools/computeruse/vision/raster.ts +408 -0
  361. package/src/tools/computeruse/vision/settle.js +131 -0
  362. package/src/tools/computeruse/vision/settle.ts +220 -0
  363. package/src/tools/computeruse/vision/telemetry.js +196 -0
  364. package/src/tools/computeruse/vision/telemetry.ts +241 -0
  365. package/src/tools/computeruse/visionOcr.js +115 -0
  366. package/src/tools/computeruse/visionOcr.ts +128 -0
  367. package/src/tools/computeruse/waitFor.js +237 -0
  368. package/src/tools/computeruse/waitFor.ts +291 -0
  369. package/src/tools/download/download.js +10 -6
  370. package/src/tools/download/download.ts +187 -0
  371. package/src/tools/edit/edit.js +30 -18
  372. package/src/tools/edit/edit.ts +349 -0
  373. package/src/tools/edit/multi-edit.js +25 -14
  374. package/src/tools/edit/multi-edit.ts +223 -0
  375. package/src/tools/effectiveToolSet.js +73 -0
  376. package/src/tools/effectiveToolSet.ts +91 -0
  377. package/src/tools/genimage/genimage.ts +257 -0
  378. package/src/tools/genspeech/genspeech.ts +83 -0
  379. package/src/tools/glob/glob.js +176 -29
  380. package/src/tools/glob/glob.ts +283 -0
  381. package/src/tools/grep/grep.js +61 -19
  382. package/src/tools/grep/grep.ts +239 -0
  383. package/src/tools/grep/rgPath.ts +206 -0
  384. package/src/tools/index.js +99 -5
  385. package/src/tools/index.ts +293 -0
  386. package/src/tools/lsp/config.js +113 -0
  387. package/src/tools/lsp/config.ts +113 -0
  388. package/src/tools/lsp/lsp.js +256 -0
  389. package/src/tools/lsp/lsp.ts +274 -0
  390. package/src/tools/lsp/service.js +303 -0
  391. package/src/tools/lsp/service.ts +296 -0
  392. package/src/tools/memory/memory.js +520 -0
  393. package/src/tools/memory/memory.ts +625 -0
  394. package/src/tools/memory/policy.js +53 -0
  395. package/src/tools/memory/policy.ts +72 -0
  396. package/src/tools/notebook/notebookEdit.js +114 -0
  397. package/src/tools/notebook/notebookEdit.ts +126 -0
  398. package/src/tools/powershell/powershell.js +210 -0
  399. package/src/tools/powershell/powershell.ts +239 -0
  400. package/src/tools/presentresult/presentresult.js +145 -0
  401. package/src/tools/presentresult/presentresult.ts +156 -0
  402. package/src/tools/read/read.js +371 -163
  403. package/src/tools/read/read.ts +558 -0
  404. package/src/tools/shared/diffSummary.js +44 -0
  405. package/src/tools/shared/diffSummary.ts +63 -0
  406. package/src/tools/shared/fileActivity.js +51 -0
  407. package/src/tools/shared/fileActivity.ts +66 -0
  408. package/src/tools/shared/fileMutationBus.js +13 -0
  409. package/src/tools/shared/fileMutationBus.ts +14 -0
  410. package/src/tools/shared/fileState.js +73 -65
  411. package/src/tools/shared/fileState.ts +145 -0
  412. package/src/tools/shared/fileUtils.js +81 -4
  413. package/src/tools/shared/fileUtils.ts +504 -0
  414. package/src/tools/shared/killTree.ts +50 -0
  415. package/src/tools/shared/permanentDeleteGuard.js +26 -0
  416. package/src/tools/shared/permanentDeleteGuard.ts +34 -0
  417. package/src/tools/shared/schemas.ts +20 -0
  418. package/src/tools/shared/toolOutputStore.js +97 -0
  419. package/src/tools/shared/toolOutputStore.ts +110 -0
  420. package/src/tools/skill/skill.js +86 -0
  421. package/src/tools/skill/skill.ts +94 -0
  422. package/src/tools/snip/snip.js +67 -0
  423. package/src/tools/snip/snip.ts +80 -0
  424. package/src/tools/tasks/backgroundTaskService.js +275 -0
  425. package/src/tools/tasks/backgroundTaskService.ts +369 -0
  426. package/src/tools/tasks/processEvidence.js +24 -0
  427. package/src/tools/tasks/processEvidence.ts +34 -0
  428. package/src/tools/tasks/taskManagement.js +163 -0
  429. package/src/tools/tasks/taskManagement.ts +180 -0
  430. package/src/tools/tasks/taskOutput.js +113 -0
  431. package/src/tools/tasks/taskOutput.ts +145 -0
  432. package/src/tools/tasks/taskStop.js +74 -0
  433. package/src/tools/tasks/taskStop.ts +99 -0
  434. package/src/tools/tasks/taskStore.js +46 -0
  435. package/src/tools/tasks/taskStore.ts +77 -0
  436. package/src/tools/terminal/terminalCapture.js +167 -0
  437. package/src/tools/terminal/terminalCapture.ts +179 -0
  438. package/src/tools/trash/trash.js +233 -0
  439. package/src/tools/trash/trash.ts +283 -0
  440. package/src/tools/types.js +1 -1
  441. package/src/tools/types.ts +24 -0
  442. package/src/tools/webbrowser/agentBrowserPath.ts +286 -0
  443. package/src/tools/webbrowser/browser.js +214 -11
  444. package/src/tools/webbrowser/browser.ts +293 -0
  445. package/src/tools/webbrowser/browserControl.js +153 -48
  446. package/src/tools/webbrowser/browserControl.ts +262 -0
  447. package/src/tools/webbrowser/browserDaemon.js +147 -39
  448. package/src/tools/webbrowser/browserDaemon.ts +270 -0
  449. package/src/tools/webbrowser/browserOutcome.js +273 -0
  450. package/src/tools/webbrowser/browserOutcome.ts +306 -0
  451. package/src/tools/webbrowser/browserRunRegistry.js +150 -0
  452. package/src/tools/webbrowser/browserRunRegistry.ts +179 -0
  453. package/src/tools/webbrowser/browserService.js +20 -6
  454. package/src/tools/webbrowser/browserService.ts +56 -0
  455. package/src/tools/webbrowser/browserStream.js +30 -1
  456. package/src/tools/webbrowser/browserStream.ts +216 -0
  457. package/src/tools/webbrowser/browserTabs.js +143 -0
  458. package/src/tools/webbrowser/browserTabs.ts +210 -0
  459. package/src/tools/webbrowser/webbrowser.js +831 -150
  460. package/src/tools/webbrowser/webbrowser.ts +1537 -0
  461. package/src/tools/webfetch/preapproved.ts +274 -0
  462. package/src/tools/webfetch/webfetch.js +24 -23
  463. package/src/tools/webfetch/webfetch.ts +383 -0
  464. package/src/tools/websearch/websearch.js +12 -11
  465. package/src/tools/websearch/websearch.ts +292 -0
  466. package/src/tools/worktree/sessionWorktree.js +128 -0
  467. package/src/tools/worktree/sessionWorktree.ts +158 -0
  468. package/src/tools/write/write.js +27 -44
  469. package/src/tools/write/write.ts +136 -0
  470. package/src/types.ts +960 -0
  471. package/src/ui/App.js +216 -12
  472. package/src/ui/App.tsx +879 -0
  473. package/src/ui/AskQuestion.tsx +300 -0
  474. package/src/ui/InputBox.js +77 -1
  475. package/src/ui/InputBox.tsx +567 -0
  476. package/src/ui/LiveArea.tsx +344 -0
  477. package/src/ui/MessageList.js +27 -6
  478. package/src/ui/MessageList.tsx +559 -0
  479. package/src/ui/ModelPicker.tsx +107 -0
  480. package/src/ui/QueuedMessagesBanner.tsx +51 -0
  481. package/src/ui/Root.js +6 -8
  482. package/src/ui/Root.tsx +100 -0
  483. package/src/ui/SessionPicker.tsx +169 -0
  484. package/src/ui/StatusLine.js +22 -1
  485. package/src/ui/StatusLine.tsx +145 -0
  486. package/src/ui/WorkspacePicker.tsx +84 -0
  487. package/src/ui/advisorCommand.js +54 -0
  488. package/src/ui/advisorCommand.ts +63 -0
  489. package/src/ui/computerCommand.js +19 -0
  490. package/src/ui/computerCommand.ts +35 -0
  491. package/src/ui/figures.ts +45 -0
  492. package/src/ui/hooks/askSlot.ts +112 -0
  493. package/src/ui/hooks/useChat.js +271 -7
  494. package/src/ui/hooks/useChat.ts +1578 -0
  495. package/src/ui/hooks/useInputHistory.ts +235 -0
  496. package/src/ui/hooks/usePasteHandler.ts +220 -0
  497. package/src/ui/hooks/useTerminalSize.ts +41 -0
  498. package/src/ui/hooks/useTextBuffer.ts +108 -0
  499. package/src/ui/hooks/useTokenUsage.ts +44 -0
  500. package/src/ui/liveViewport.ts +189 -0
  501. package/src/ui/markdown.ts +230 -0
  502. package/src/ui/textBuffer.ts +239 -0
  503. package/src/ui/theme.ts +35 -0
  504. package/src/utils/clipboardImage.js +358 -0
  505. package/src/utils/clipboardImage.ts +420 -0
  506. package/src/utils/debugLog.ts +50 -0
  507. package/src/utils/greenRoot.ts +35 -0
  508. package/src/utils/packageRoot.ts +40 -0
  509. package/src/utils/pathContains.js +28 -0
  510. package/src/utils/pathContains.ts +31 -0
  511. package/src/utils/resourcePaths.ts +71 -0
  512. package/src/utils/stripThink.ts +49 -0
  513. package/src/utils/zodToJson.ts +73 -0
  514. package/src/web/browserBridge.js +365 -66
  515. package/src/web/browserBridge.ts +672 -0
  516. package/src/web/protocol.ts +1804 -0
  517. package/vendor/computer-use/arm64-darwin/cu-helperd +0 -0
  518. package/workflows/ppt.yaml +13 -2
  519. package/workflows/wechat-reading-shots.yaml +315 -0
  520. package/workflows/word.yaml +11 -1
  521. package/skills/docx/scripts/office/helpers/__pycache__/__init__.cpython-314.pyc +0 -0
  522. package/skills/docx/scripts/office/helpers/__pycache__/merge_runs.cpython-314.pyc +0 -0
  523. package/skills/docx/scripts/office/helpers/__pycache__/simplify_redlines.cpython-314.pyc +0 -0
  524. package/skills/docx/scripts/office/validators/__pycache__/__init__.cpython-314.pyc +0 -0
  525. package/skills/docx/scripts/office/validators/__pycache__/base.cpython-314.pyc +0 -0
  526. package/skills/docx/scripts/office/validators/__pycache__/docx.cpython-314.pyc +0 -0
  527. package/skills/docx/scripts/office/validators/__pycache__/pptx.cpython-314.pyc +0 -0
  528. package/skills/docx/scripts/office/validators/__pycache__/redlining.cpython-314.pyc +0 -0
  529. package/skills/image-stats/scripts/__pycache__/image_stats.cpython-314.pyc +0 -0
  530. package/skills/image-stats/scripts/__pycache__/image_stats_test.cpython-314.pyc +0 -0
  531. package/skills/ui-design/scripts/__pycache__/assemble_material.cpython-314.pyc +0 -0
  532. package/skills/ui-design/scripts/__pycache__/assemble_material_test.cpython-314.pyc +0 -0
  533. package/skills/ui-design/scripts/__pycache__/chart_sanity_check.cpython-314.pyc +0 -0
  534. package/skills/ui-design/scripts/__pycache__/check_language_policy.cpython-314.pyc +0 -0
  535. package/skills/ui-design/scripts/__pycache__/check_ui_artifact.cpython-314.pyc +0 -0
  536. package/skills/ui-design/scripts/__pycache__/check_ui_artifact_pptx_test.cpython-314.pyc +0 -0
  537. package/skills/ui-design/scripts/__pycache__/deck_pages_check.cpython-314.pyc +0 -0
  538. package/skills/ui-design/scripts/__pycache__/deck_pages_check_test.cpython-314.pyc +0 -0
  539. package/skills/ui-design/scripts/__pycache__/deck_visual_style_check.cpython-314.pyc +0 -0
  540. package/skills/ui-design/scripts/__pycache__/deck_visual_style_check_test.cpython-314.pyc +0 -0
  541. package/skills/ui-design/scripts/__pycache__/design_budget_check.cpython-314.pyc +0 -0
  542. package/skills/ui-design/scripts/__pycache__/design_budget_check_test.cpython-314.pyc +0 -0
  543. package/skills/ui-design/scripts/__pycache__/design_repair_hints.cpython-314.pyc +0 -0
  544. package/skills/ui-design/scripts/__pycache__/design_repair_hints_test.cpython-314.pyc +0 -0
  545. package/skills/ui-design/scripts/__pycache__/design_style_check.cpython-314.pyc +0 -0
  546. package/skills/ui-design/scripts/__pycache__/design_style_check_test.cpython-314.pyc +0 -0
  547. package/skills/ui-design/scripts/__pycache__/draft_coverage_check.cpython-314.pyc +0 -0
  548. package/skills/ui-design/scripts/__pycache__/extract_source_assets.cpython-314.pyc +0 -0
  549. package/skills/ui-design/scripts/__pycache__/extract_source_assets_test.cpython-314.pyc +0 -0
  550. package/skills/ui-design/scripts/__pycache__/generated_image_review_collect.cpython-314.pyc +0 -0
  551. package/skills/ui-design/scripts/__pycache__/generated_image_review_collect_test.cpython-314.pyc +0 -0
  552. package/skills/ui-design/scripts/__pycache__/image_plan_lint.cpython-314.pyc +0 -0
  553. package/skills/ui-design/scripts/__pycache__/image_plan_lint_test.cpython-314.pyc +0 -0
  554. package/skills/ui-design/scripts/__pycache__/image_prompt_rewrite.cpython-314.pyc +0 -0
  555. package/skills/ui-design/scripts/__pycache__/image_prompt_rewrite_test.cpython-314.pyc +0 -0
  556. package/skills/ui-design/scripts/__pycache__/layout_balance.cpython-314.pyc +0 -0
  557. package/skills/ui-design/scripts/__pycache__/layout_balance_test.cpython-314.pyc +0 -0
  558. package/skills/ui-design/scripts/__pycache__/normalize_design_json.cpython-314.pyc +0 -0
  559. package/skills/ui-design/scripts/__pycache__/normalize_design_json_test.cpython-314.pyc +0 -0
  560. package/skills/ui-design/scripts/__pycache__/plan_source_layout.cpython-314.pyc +0 -0
  561. package/skills/ui-design/scripts/__pycache__/plan_source_layout_test.cpython-314.pyc +0 -0
  562. package/skills/ui-design/scripts/__pycache__/resolve_source_hint.cpython-314.pyc +0 -0
  563. package/skills/ui-design/scripts/__pycache__/resolve_source_hint_test.cpython-314.pyc +0 -0
  564. package/skills/ui-design/scripts/__pycache__/resolve_source_input.cpython-314.pyc +0 -0
  565. package/skills/ui-design/scripts/__pycache__/resolve_source_input_test.cpython-314.pyc +0 -0
  566. package/skills/ui-design/scripts/__pycache__/scaffold_ui_artifact.cpython-314.pyc +0 -0
  567. package/skills/ui-design/scripts/__pycache__/source_usage_check.cpython-314.pyc +0 -0
  568. package/skills/ui-design/scripts/__pycache__/source_usage_check_test.cpython-314.pyc +0 -0
  569. package/skills/ui-design/scripts/__pycache__/text_readability_pixel.cpython-314.pyc +0 -0
  570. package/skills/ui-design/scripts/__pycache__/visual_quality_gate.cpython-314.pyc +0 -0
  571. package/skills/ui-design/scripts/__pycache__/visual_quality_gate_test.cpython-314.pyc +0 -0
  572. /package/skills/ai-image/scripts/{generate_test.mjs → generate.selftest.mjs} +0 -0
  573. /package/skills/ai-image/scripts/{generate_from_plan_test.mjs → generate_from_plan.selftest.mjs} +0 -0
@@ -0,0 +1,56 @@
1
+ import type { AgentBrowserRun, AgentBrowserRunner } from './browser.js';
2
+
3
+ /**
4
+ * Process-wide action coordinator for the supervised browser. Desktop input,
5
+ * Agent tools, TUI and workflows share the same daemon, so CDP mutations must
6
+ * be serialized instead of racing through separate command adapters.
7
+ */
8
+ export class BrowserService {
9
+ private tails = new Map<string, Promise<void>>();
10
+
11
+ async exclusive<T>(input: {
12
+ sessionName: string;
13
+ bin: string;
14
+ runner: AgentBrowserRunner;
15
+ signal?: AbortSignal;
16
+ }, task: (run: (args: string[]) => Promise<AgentBrowserRun>) => Promise<T>): Promise<T> {
17
+ const previous = this.tails.get(input.sessionName) ?? Promise.resolve();
18
+ let release!: () => void;
19
+ const current = new Promise<void>((resolve) => { release = resolve; });
20
+ // 链上永远接一个已消化的 previous:任何一环 reject 都不能顺着队列传下去,
21
+ // 否则后来者在 `await previous` 处抛出。
22
+ const tail = previous.catch(() => undefined).then(() => current);
23
+ this.tails.set(input.sessionName, tail);
24
+
25
+ // ⚠️ `await previous` **必须在 try 之内**。它曾经在 try 之外:previous 一旦 reject,
26
+ // 函数在进入 try 之前就抛出 → `release()` 永不执行 → `current` 永不 resolve →
27
+ // 后续所有该 session 的动作永久排队。症状是死锁而不是异常,排查起来极难。
28
+ // 当前 current 只 resolve 不 reject,所以实际安全 —— 但那是一条没写下来的不变量,
29
+ // 谁往链上加一个 .then 都可能打破它。把 await 挪进来,安全性就不再依赖它。
30
+ try {
31
+ await previous.catch(() => undefined);
32
+ const run = async (args: string[]): Promise<AgentBrowserRun> => {
33
+ if (input.signal?.aborted) {
34
+ return { code: -1, stdout: '', stderr: 'Browser action cancelled' };
35
+ }
36
+ return input.runner(input.bin, args, input.signal);
37
+ };
38
+ return await task(run);
39
+ } finally {
40
+ release();
41
+ if (this.tails.get(input.sessionName) === tail) this.tails.delete(input.sessionName);
42
+ }
43
+ }
44
+
45
+ async run(input: {
46
+ sessionName: string;
47
+ bin: string;
48
+ args: string[];
49
+ runner: AgentBrowserRunner;
50
+ signal?: AbortSignal;
51
+ }): Promise<AgentBrowserRun> {
52
+ return this.exclusive(input, (run) => run(input.args));
53
+ }
54
+ }
55
+
56
+ export const browserService = new BrowserService();
@@ -11,10 +11,15 @@ class SessionBrowserStream {
11
11
  status = { connected: false, screencasting: false };
12
12
  picker;
13
13
  marked = [];
14
- control = new BrowserControlCoordinator();
14
+ control;
15
15
  constructor(sessionName) {
16
16
  this.sessionName = sessionName;
17
+ this.control = new BrowserControlCoordinator({ conversationKey: `browser:${sessionName}` });
17
18
  this.control.subscribe((state) => this.emit({ type: 'control_state', state }));
19
+ this.control.subscribeInterventions((event) => this.emit({
20
+ type: event.type === 'started' ? 'human_intervention_started' : 'human_intervention_resolved',
21
+ event,
22
+ }));
18
23
  }
19
24
  async connect() {
20
25
  if (this.socket?.readyState === WebSocket.OPEN)
@@ -77,17 +82,41 @@ class SessionBrowserStream {
77
82
  this.emit({ type: 'marked_elements', elements: this.marked });
78
83
  }
79
84
  clearMarkedElements() { this.marked = []; this.emit({ type: 'marked_elements', elements: [] }); }
85
+ unmarkElement(ref) {
86
+ const next = this.marked.filter((item) => item.ref !== ref);
87
+ if (next.length !== this.marked.length) {
88
+ this.marked = next;
89
+ this.emit({ type: 'marked_elements', elements: this.marked });
90
+ }
91
+ }
80
92
  setAgentActing(acting, action, target) {
81
93
  this.emit({ type: 'agent_acting', acting, action, target });
82
94
  }
95
+ setAgentCandidates(candidates, selector) {
96
+ this.emit({ type: 'agent_candidates', candidates, selector });
97
+ }
98
+ setAgentOutcome(outcome) {
99
+ this.emit({ type: 'agent_outcome', ...outcome });
100
+ }
101
+ setAgentPointer(pointer) {
102
+ // 只发语义终点(目标元素中心 + 动作类型);平滑移动交给面板 CSS 过渡,内核不做逐帧插值(避免 WS 洪泛 + 双层缓动打架)。
103
+ this.emit({ type: 'agent_pointer', ...pointer });
104
+ }
83
105
  getControlState() { return this.control.getState(); }
84
106
  subscribeControl(listener) { return this.control.subscribe(listener); }
107
+ subscribeInterventions(listener) { return this.control.subscribeInterventions(listener); }
85
108
  beginAgentAction(input) { this.control.beginAgent(input); }
86
109
  finishAgentAction(runId) { this.control.finishAgent(runId); }
87
110
  requestHuman(reason, input) { return this.control.requestHuman(reason, input); }
88
111
  takeControl(input) { this.control.takeInteractive(input); }
89
112
  resumeControl(token) { return this.control.resume(token); }
90
113
  stopControl(token) { return this.control.stop(token); }
114
+ resumeIntervention(credential) {
115
+ return this.control.resumeIntervention(credential);
116
+ }
117
+ cancelIntervention(credential) {
118
+ return this.control.cancelIntervention(credential);
119
+ }
91
120
  waitForHuman(signal) { return this.control.waitForHuman(signal); }
92
121
  emit(event) {
93
122
  for (const listener of this.listeners)
@@ -0,0 +1,216 @@
1
+ import { getStreamPort, waitForStreamPort } from './browserDaemon.js';
2
+ import { BrowserControlCoordinator, type BrowserControlState, type BrowserHumanReason } from './browserControl.js';
3
+ import type { HumanInterventionCredential, HumanInterventionEvent } from '../../humanIntervention.js';
4
+
5
+ export type BrowserMode = 'browse' | 'assist' | 'auto';
6
+ export type PickedBrowserElement = {
7
+ ref: string;
8
+ tag: string;
9
+ role?: string;
10
+ accessibleName?: string;
11
+ text?: string;
12
+ boundingBox?: { x: number; y: number; width: number; height: number };
13
+ attributes?: Record<string, string>;
14
+ };
15
+ export type MarkedBrowserElement = PickedBrowserElement & { label?: string; color?: string };
16
+ export type BrowserStreamEvent = Record<string, unknown> & { type: string };
17
+
18
+ export interface BrowserStream {
19
+ connect(): Promise<void>;
20
+ disconnect(): void;
21
+ send(frame: unknown): void;
22
+ dispose(): void;
23
+ subscribe(listener: (event: BrowserStreamEvent) => void): () => void;
24
+ getStatus(): Record<string, unknown>;
25
+ getPort(): number | null;
26
+ isConnected(): boolean;
27
+ setMode(mode: BrowserMode): void;
28
+ getMode(): BrowserMode;
29
+ getCurrentUrl(): string;
30
+ setCurrentUrl(url: string): void;
31
+ pickElement(x: number, y: number): Promise<PickedBrowserElement | null>;
32
+ setPicker(picker: (x: number, y: number) => Promise<PickedBrowserElement | null>): void;
33
+ markElement(element: MarkedBrowserElement): void;
34
+ clearMarkedElements(): void;
35
+ /** Remove a single marked element by ref (does not affect other marks). */
36
+ unmarkElement(ref: string): void;
37
+ setAgentActing(acting: boolean, action?: string, target?: string): void;
38
+ /**
39
+ * 把「这次没找到目标,最像的是这几个」推给可见面板。
40
+ *
41
+ * 候选只进对话的话,盯着浏览器的人看到的是 AI 忽然停住;推到面板上他可以一眼看出
42
+ * AI 卡在哪,并点一下把正确的那个喂回去 —— 人的判断比模型再猜一轮便宜得多。
43
+ */
44
+ setAgentCandidates(candidates: Array<{ ref: string; line: string }>, selector?: string): void;
45
+ /**
46
+ * 发布一次动作**做完之后**的生效结论(只在结论不是「变了」时调用)。
47
+ *
48
+ * 与 `setAgentActing` 分开是必须的:acting 在 `finally` 里被无条件置回 false,
49
+ * 把结论挂上去等于发完立刻抹掉。结论要活到下一次动作开始或页面切换。
50
+ */
51
+ setAgentOutcome(outcome: {
52
+ action: string;
53
+ effect: 'unchanged' | 'unknown';
54
+ selector?: string;
55
+ maskedLines?: number;
56
+ }): void;
57
+ /** C4:发布 AI 指针语义终点给可见面板(宿主用 CSS 过渡平滑移动 + 渲染点击动效)。 */
58
+ setAgentPointer(pointer: { x: number; y: number; visible: boolean; action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle'; elementRef?: string }): void;
59
+ getControlState(): BrowserControlState;
60
+ /** 订阅控制权状态变化(立即以当前状态回调一次);供工具即时感知「用户接管 / 进入 waiting_human」以掐断在跑动作。 */
61
+ subscribeControl(listener: (state: BrowserControlState) => void): () => void;
62
+ subscribeInterventions(listener: (event: HumanInterventionEvent) => void): () => void;
63
+ beginAgentAction(input: { runId: string; conversationKey?: string; action?: string }): void;
64
+ finishAgentAction(runId: string): void;
65
+ requestHuman(reason: BrowserHumanReason, input?: { runId?: string; conversationKey?: string; ownerId?: string; summary?: string }): string;
66
+ takeControl(input?: { runId?: string; conversationKey?: string }): void;
67
+ resumeControl(token?: string): boolean;
68
+ stopControl(token?: string): boolean;
69
+ resumeIntervention(credential: HumanInterventionCredential): boolean;
70
+ cancelIntervention(credential: HumanInterventionCredential): boolean;
71
+ waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'>;
72
+ }
73
+
74
+ class SessionBrowserStream implements BrowserStream {
75
+ private socket: WebSocket | null = null;
76
+ private connecting: Promise<void> | null = null;
77
+ private listeners = new Set<(event: BrowserStreamEvent) => void>();
78
+ private port: number | null = null;
79
+ private mode: BrowserMode = 'browse';
80
+ private currentUrl = 'about:blank';
81
+ private status: Record<string, unknown> = { connected: false, screencasting: false };
82
+ private picker?: (x: number, y: number) => Promise<PickedBrowserElement | null>;
83
+ private marked: MarkedBrowserElement[] = [];
84
+ private control: BrowserControlCoordinator;
85
+
86
+ constructor(private readonly sessionName: string) {
87
+ this.control = new BrowserControlCoordinator({ conversationKey: `browser:${sessionName}` });
88
+ this.control.subscribe((state) => this.emit({ type: 'control_state', state }));
89
+ this.control.subscribeInterventions((event) => this.emit({
90
+ type: event.type === 'started' ? 'human_intervention_started' : 'human_intervention_resolved',
91
+ event,
92
+ }));
93
+ }
94
+
95
+ async connect(): Promise<void> {
96
+ if (this.socket?.readyState === WebSocket.OPEN) return;
97
+ if (this.connecting) return this.connecting;
98
+ this.connecting = (async () => {
99
+ this.port = (await getStreamPort(this.sessionName)) ?? (await waitForStreamPort(this.sessionName));
100
+ await new Promise<void>((resolve, reject) => {
101
+ const socket = new WebSocket(`ws://127.0.0.1:${this.port}`);
102
+ const timer = setTimeout(() => {
103
+ socket.close();
104
+ reject(new Error('连接浏览器实时画面超时'));
105
+ }, 8_000);
106
+ socket.onopen = () => {
107
+ clearTimeout(timer);
108
+ this.socket = socket;
109
+ this.status = { ...this.status, connected: true };
110
+ this.emit({ type: 'status', ...this.status });
111
+ resolve();
112
+ };
113
+ socket.onerror = () => {
114
+ clearTimeout(timer);
115
+ reject(new Error('无法连接浏览器实时画面'));
116
+ };
117
+ socket.onclose = () => {
118
+ if (this.socket === socket) this.socket = null;
119
+ this.status = { ...this.status, connected: false, screencasting: false };
120
+ this.emit({ type: 'status', ...this.status });
121
+ };
122
+ socket.onmessage = (event) => this.onMessage(event.data);
123
+ });
124
+ })().finally(() => { this.connecting = null; });
125
+ return this.connecting;
126
+ }
127
+
128
+ disconnect(): void { this.socket?.close(); this.socket = null; }
129
+ dispose(): void { this.disconnect(); this.listeners.clear(); }
130
+ send(frame: unknown): void {
131
+ if (this.socket?.readyState === WebSocket.OPEN) this.socket.send(JSON.stringify(frame));
132
+ }
133
+ subscribe(listener: (event: BrowserStreamEvent) => void): () => void {
134
+ this.listeners.add(listener);
135
+ return () => this.listeners.delete(listener);
136
+ }
137
+ getStatus(): Record<string, unknown> { return this.status; }
138
+ getPort(): number | null { return this.port; }
139
+ isConnected(): boolean { return this.socket?.readyState === WebSocket.OPEN; }
140
+ setMode(mode: BrowserMode): void { this.mode = mode; this.emit({ type: 'mode', mode }); }
141
+ getMode(): BrowserMode { return this.mode; }
142
+ getCurrentUrl(): string { return this.currentUrl; }
143
+ setCurrentUrl(url: string): void { this.currentUrl = url; this.emit({ type: 'url', url }); }
144
+ setPicker(picker: (x: number, y: number) => Promise<PickedBrowserElement | null>): void { this.picker = picker; }
145
+ pickElement(x: number, y: number): Promise<PickedBrowserElement | null> {
146
+ return this.picker ? this.picker(x, y) : Promise.resolve(null);
147
+ }
148
+ markElement(element: MarkedBrowserElement): void {
149
+ this.marked = [...this.marked.filter((item) => item.ref !== element.ref), element];
150
+ this.emit({ type: 'marked_elements', elements: this.marked });
151
+ }
152
+ clearMarkedElements(): void { this.marked = []; this.emit({ type: 'marked_elements', elements: [] }); }
153
+ unmarkElement(ref: string): void {
154
+ const next = this.marked.filter((item) => item.ref !== ref);
155
+ if (next.length !== this.marked.length) {
156
+ this.marked = next;
157
+ this.emit({ type: 'marked_elements', elements: this.marked });
158
+ }
159
+ }
160
+ setAgentActing(acting: boolean, action?: string, target?: string): void {
161
+ this.emit({ type: 'agent_acting', acting, action, target });
162
+ }
163
+ setAgentCandidates(candidates: Array<{ ref: string; line: string }>, selector?: string): void {
164
+ this.emit({ type: 'agent_candidates', candidates, selector });
165
+ }
166
+ setAgentOutcome(outcome: {
167
+ action: string;
168
+ effect: 'unchanged' | 'unknown';
169
+ selector?: string;
170
+ maskedLines?: number;
171
+ }): void {
172
+ this.emit({ type: 'agent_outcome', ...outcome });
173
+ }
174
+ setAgentPointer(pointer: { x: number; y: number; visible: boolean; action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle'; elementRef?: string }): void {
175
+ // 只发语义终点(目标元素中心 + 动作类型);平滑移动交给面板 CSS 过渡,内核不做逐帧插值(避免 WS 洪泛 + 双层缓动打架)。
176
+ this.emit({ type: 'agent_pointer', ...pointer });
177
+ }
178
+ getControlState(): BrowserControlState { return this.control.getState(); }
179
+ subscribeControl(listener: (state: BrowserControlState) => void): () => void { return this.control.subscribe(listener); }
180
+ subscribeInterventions(listener: (event: HumanInterventionEvent) => void): () => void { return this.control.subscribeInterventions(listener); }
181
+ beginAgentAction(input: { runId: string; conversationKey?: string; action?: string }): void { this.control.beginAgent(input); }
182
+ finishAgentAction(runId: string): void { this.control.finishAgent(runId); }
183
+ requestHuman(reason: BrowserHumanReason, input?: { runId?: string; conversationKey?: string; ownerId?: string; summary?: string }): string { return this.control.requestHuman(reason, input); }
184
+ takeControl(input?: { runId?: string; conversationKey?: string }): void { this.control.takeInteractive(input); }
185
+ resumeControl(token?: string): boolean { return this.control.resume(token); }
186
+ stopControl(token?: string): boolean { return this.control.stop(token); }
187
+ resumeIntervention(credential: HumanInterventionCredential): boolean {
188
+ return this.control.resumeIntervention(credential);
189
+ }
190
+ cancelIntervention(credential: HumanInterventionCredential): boolean {
191
+ return this.control.cancelIntervention(credential);
192
+ }
193
+ waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'> { return this.control.waitForHuman(signal); }
194
+
195
+ private emit(event: BrowserStreamEvent): void {
196
+ for (const listener of this.listeners) listener(event);
197
+ }
198
+ private onMessage(data: unknown): void {
199
+ try {
200
+ const text = typeof data === 'string' ? data : Buffer.from(data as ArrayBuffer).toString('utf8');
201
+ const event = JSON.parse(text) as BrowserStreamEvent;
202
+ if (event.type === 'url' && typeof event.url === 'string') this.currentUrl = event.url;
203
+ if (event.type === 'status') this.status = { ...this.status, ...event };
204
+ this.emit(event);
205
+ } catch (error) {
206
+ this.emit({ type: 'error', message: error instanceof Error ? error.message : String(error) });
207
+ }
208
+ }
209
+ }
210
+
211
+ const streams = new Map<string, SessionBrowserStream>();
212
+ export function getBrowserStream(sessionName = 'minimal-agent'): BrowserStream {
213
+ let stream = streams.get(sessionName);
214
+ if (!stream) { stream = new SessionBrowserStream(sessionName); streams.set(sessionName, stream); }
215
+ return stream;
216
+ }
@@ -0,0 +1,143 @@
1
+ import { browserService } from './browserService.js';
2
+ import { browserRunRegistry, } from './browserRunRegistry.js';
3
+ export function parseTabList(text) {
4
+ try {
5
+ const parsed = JSON.parse(text);
6
+ return (parsed.data?.tabs ?? []).flatMap((tab) => {
7
+ const id = typeof tab.tabId === 'string' && tab.tabId
8
+ ? tab.tabId
9
+ : typeof tab.id === 'string' && tab.id
10
+ ? tab.id
11
+ : Number.isInteger(tab.index)
12
+ ? String(tab.index)
13
+ : '';
14
+ return id ? [{
15
+ id,
16
+ active: Boolean(tab.active),
17
+ ...(typeof tab.url === 'string' ? { url: tab.url } : {}),
18
+ ...(typeof tab.title === 'string' ? { title: tab.title } : {}),
19
+ ...(typeof tab.type === 'string' ? { type: tab.type } : {}),
20
+ }] : [];
21
+ });
22
+ }
23
+ catch {
24
+ return [];
25
+ }
26
+ }
27
+ export class BrowserTabError extends Error {
28
+ code;
29
+ constructor(code, message) {
30
+ super(message);
31
+ this.code = code;
32
+ this.name = 'BrowserTabError';
33
+ }
34
+ }
35
+ function parseListedTabs(result) {
36
+ if (result.code !== 0) {
37
+ throw new BrowserTabError('browser_unavailable', result.stderr || result.stdout || '无法读取浏览器标签');
38
+ }
39
+ const tabs = parseTabList(result.stdout);
40
+ let reportedCount = 0;
41
+ try {
42
+ const parsed = JSON.parse(result.stdout);
43
+ reportedCount = Array.isArray(parsed.data?.tabs) ? parsed.data.tabs.length : 0;
44
+ }
45
+ catch {
46
+ throw new BrowserTabError('browser_protocol_mismatch', '浏览器标签列表不是有效 JSON');
47
+ }
48
+ if (reportedCount > 0 && tabs.length === 0) {
49
+ throw new BrowserTabError('browser_protocol_mismatch', '浏览器返回了无法识别的标签格式,请更新 Runtime 或 agent-browser');
50
+ }
51
+ return tabs;
52
+ }
53
+ export async function listBrowserTabs(input) {
54
+ const result = await browserService.run({
55
+ sessionName: input.sessionName,
56
+ bin: input.bin,
57
+ args: [...input.globals, 'tab', 'list', '--json'],
58
+ runner: input.runner,
59
+ signal: input.signal,
60
+ });
61
+ return parseListedTabs(result);
62
+ }
63
+ function assertSwitched(switched) {
64
+ if (switched.code !== 0) {
65
+ throw new BrowserTabError('tab_not_found', switched.stderr || switched.stdout || '无法切换到会话浏览器标签');
66
+ }
67
+ }
68
+ export async function createConversationTab(input, url = 'about:blank') {
69
+ return browserService.exclusive({
70
+ sessionName: input.sessionName,
71
+ bin: input.bin,
72
+ runner: input.runner,
73
+ signal: input.signal,
74
+ }, async (run) => {
75
+ const before = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
76
+ const beforeIds = new Set(before.map((tab) => tab.id));
77
+ const created = await run([...input.globals, 'tab', 'new', url]);
78
+ if (created.code !== 0) {
79
+ throw new BrowserTabError('tab_bootstrap_failed', created.stderr || created.stdout || '无法创建会话浏览器标签');
80
+ }
81
+ const after = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
82
+ const added = after.filter((tab) => !beforeIds.has(tab.id));
83
+ const tab = added.find((candidate) => candidate.active) ?? added.at(-1);
84
+ if (!tab) {
85
+ throw new BrowserTabError('tab_bootstrap_failed', '新标签创建成功,但 Runtime 未返回其标签 ID');
86
+ }
87
+ try {
88
+ browserRunRegistry.claimTab(input.context, tab.id);
89
+ }
90
+ catch (error) {
91
+ throw new BrowserTabError('tab_owned_by_other_conversation', error instanceof Error ? error.message : String(error));
92
+ }
93
+ assertSwitched(await run([...input.globals, 'tab', tab.id]));
94
+ return tab.id;
95
+ });
96
+ }
97
+ export async function ensureConversationTab(input) {
98
+ return browserService.exclusive({
99
+ sessionName: input.sessionName,
100
+ bin: input.bin,
101
+ runner: input.runner,
102
+ signal: input.signal,
103
+ }, async (run) => {
104
+ const tabs = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
105
+ let tabId = input.context.activeTabId;
106
+ if (!tabId || !tabs.some((tab) => tab.id === tabId)) {
107
+ const claimed = browserRunRegistry.claimedTabs(input.context.conversationKey);
108
+ tabId = claimed.find((candidate) => tabs.some((tab) => tab.id === candidate));
109
+ }
110
+ if (!tabId) {
111
+ const hasClaimedTabs = browserRunRegistry.claimedTabs(input.context.conversationKey).length > 0;
112
+ const reusableBlank = !hasClaimedTabs
113
+ ? tabs.find((tab) => tab.url === 'about:blank' && browserRunRegistry.ownerOf(tab.id) === undefined)
114
+ : undefined;
115
+ if (reusableBlank) {
116
+ tabId = reusableBlank.id;
117
+ }
118
+ else {
119
+ const beforeIds = new Set(tabs.map((tab) => tab.id));
120
+ const created = await run([...input.globals, 'tab', 'new', 'about:blank']);
121
+ if (created.code !== 0) {
122
+ throw new BrowserTabError('tab_bootstrap_failed', created.stderr || created.stdout || '无法创建会话浏览器标签');
123
+ }
124
+ const after = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
125
+ const added = after.filter((tab) => !beforeIds.has(tab.id));
126
+ tabId = (added.find((candidate) => candidate.active) ?? added.at(-1))?.id;
127
+ if (!tabId) {
128
+ throw new BrowserTabError('tab_bootstrap_failed', '新标签创建成功,但 Runtime 未返回其标签 ID');
129
+ }
130
+ }
131
+ }
132
+ try {
133
+ browserRunRegistry.claimTab(input.context, tabId);
134
+ }
135
+ catch (error) {
136
+ throw new BrowserTabError('tab_owned_by_other_conversation', error instanceof Error ? error.message : String(error));
137
+ }
138
+ assertSwitched(await run([...input.globals, 'tab', tabId]));
139
+ return tabId;
140
+ });
141
+ }
142
+ /** Compatibility alias for callers compiled against the previous helper name. */
143
+ export const claimConversationTab = ensureConversationTab;
@@ -0,0 +1,210 @@
1
+ import type { AgentBrowserRun, AgentBrowserRunner } from './browser.js';
2
+ import { browserService } from './browserService.js';
3
+ import {
4
+ browserRunRegistry,
5
+ type BrowserRunContext,
6
+ } from './browserRunRegistry.js';
7
+
8
+ export interface BrowserTabRecord {
9
+ id: string;
10
+ active: boolean;
11
+ url?: string;
12
+ title?: string;
13
+ type?: string;
14
+ }
15
+
16
+ export function parseTabList(text: string): BrowserTabRecord[] {
17
+ try {
18
+ const parsed = JSON.parse(text) as {
19
+ data?: {
20
+ tabs?: Array<{
21
+ tabId?: string;
22
+ id?: string;
23
+ index?: number;
24
+ active?: boolean;
25
+ url?: string;
26
+ title?: string;
27
+ type?: string;
28
+ }>;
29
+ };
30
+ };
31
+ return (parsed.data?.tabs ?? []).flatMap((tab) => {
32
+ const id = typeof tab.tabId === 'string' && tab.tabId
33
+ ? tab.tabId
34
+ : typeof tab.id === 'string' && tab.id
35
+ ? tab.id
36
+ : Number.isInteger(tab.index)
37
+ ? String(tab.index)
38
+ : '';
39
+ return id ? [{
40
+ id,
41
+ active: Boolean(tab.active),
42
+ ...(typeof tab.url === 'string' ? { url: tab.url } : {}),
43
+ ...(typeof tab.title === 'string' ? { title: tab.title } : {}),
44
+ ...(typeof tab.type === 'string' ? { type: tab.type } : {}),
45
+ }] : [];
46
+ });
47
+ } catch {
48
+ return [];
49
+ }
50
+ }
51
+
52
+ export class BrowserTabError extends Error {
53
+ constructor(
54
+ public readonly code:
55
+ | 'browser_unavailable'
56
+ | 'tab_bootstrap_failed'
57
+ | 'tab_not_found'
58
+ | 'tab_owned_by_other_conversation'
59
+ | 'browser_protocol_mismatch',
60
+ message: string,
61
+ ) {
62
+ super(message);
63
+ this.name = 'BrowserTabError';
64
+ }
65
+ }
66
+
67
+ type BrowserTabInput = {
68
+ context: BrowserRunContext;
69
+ sessionName: string;
70
+ bin: string;
71
+ globals: string[];
72
+ runner: AgentBrowserRunner;
73
+ signal?: AbortSignal;
74
+ };
75
+
76
+ function parseListedTabs(result: AgentBrowserRun): BrowserTabRecord[] {
77
+ if (result.code !== 0) {
78
+ throw new BrowserTabError(
79
+ 'browser_unavailable',
80
+ result.stderr || result.stdout || '无法读取浏览器标签',
81
+ );
82
+ }
83
+ const tabs = parseTabList(result.stdout);
84
+ let reportedCount = 0;
85
+ try {
86
+ const parsed = JSON.parse(result.stdout) as { data?: { tabs?: unknown[] } };
87
+ reportedCount = Array.isArray(parsed.data?.tabs) ? parsed.data.tabs.length : 0;
88
+ } catch {
89
+ throw new BrowserTabError('browser_protocol_mismatch', '浏览器标签列表不是有效 JSON');
90
+ }
91
+ if (reportedCount > 0 && tabs.length === 0) {
92
+ throw new BrowserTabError(
93
+ 'browser_protocol_mismatch',
94
+ '浏览器返回了无法识别的标签格式,请更新 Runtime 或 agent-browser',
95
+ );
96
+ }
97
+ return tabs;
98
+ }
99
+
100
+ export async function listBrowserTabs(input: Omit<BrowserTabInput, 'context'>): Promise<BrowserTabRecord[]> {
101
+ const result = await browserService.run({
102
+ sessionName: input.sessionName,
103
+ bin: input.bin,
104
+ args: [...input.globals, 'tab', 'list', '--json'],
105
+ runner: input.runner,
106
+ signal: input.signal,
107
+ });
108
+ return parseListedTabs(result);
109
+ }
110
+
111
+ function assertSwitched(switched: AgentBrowserRun): void {
112
+ if (switched.code !== 0) {
113
+ throw new BrowserTabError(
114
+ 'tab_not_found',
115
+ switched.stderr || switched.stdout || '无法切换到会话浏览器标签',
116
+ );
117
+ }
118
+ }
119
+
120
+ export async function createConversationTab(
121
+ input: BrowserTabInput,
122
+ url = 'about:blank',
123
+ ): Promise<string> {
124
+ return browserService.exclusive({
125
+ sessionName: input.sessionName,
126
+ bin: input.bin,
127
+ runner: input.runner,
128
+ signal: input.signal,
129
+ }, async (run) => {
130
+ const before = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
131
+ const beforeIds = new Set(before.map((tab) => tab.id));
132
+ const created = await run([...input.globals, 'tab', 'new', url]);
133
+ if (created.code !== 0) {
134
+ throw new BrowserTabError(
135
+ 'tab_bootstrap_failed',
136
+ created.stderr || created.stdout || '无法创建会话浏览器标签',
137
+ );
138
+ }
139
+ const after = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
140
+ const added = after.filter((tab) => !beforeIds.has(tab.id));
141
+ const tab = added.find((candidate) => candidate.active) ?? added.at(-1);
142
+ if (!tab) {
143
+ throw new BrowserTabError('tab_bootstrap_failed', '新标签创建成功,但 Runtime 未返回其标签 ID');
144
+ }
145
+ try {
146
+ browserRunRegistry.claimTab(input.context, tab.id);
147
+ } catch (error) {
148
+ throw new BrowserTabError(
149
+ 'tab_owned_by_other_conversation',
150
+ error instanceof Error ? error.message : String(error),
151
+ );
152
+ }
153
+ assertSwitched(await run([...input.globals, 'tab', tab.id]));
154
+ return tab.id;
155
+ });
156
+ }
157
+
158
+ export async function ensureConversationTab(input: BrowserTabInput): Promise<string> {
159
+ return browserService.exclusive({
160
+ sessionName: input.sessionName,
161
+ bin: input.bin,
162
+ runner: input.runner,
163
+ signal: input.signal,
164
+ }, async (run) => {
165
+ const tabs = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
166
+ let tabId = input.context.activeTabId;
167
+ if (!tabId || !tabs.some((tab) => tab.id === tabId)) {
168
+ const claimed = browserRunRegistry.claimedTabs(input.context.conversationKey);
169
+ tabId = claimed.find((candidate) => tabs.some((tab) => tab.id === candidate));
170
+ }
171
+ if (!tabId) {
172
+ const hasClaimedTabs = browserRunRegistry.claimedTabs(input.context.conversationKey).length > 0;
173
+ const reusableBlank = !hasClaimedTabs
174
+ ? tabs.find((tab) => tab.url === 'about:blank' && browserRunRegistry.ownerOf(tab.id) === undefined)
175
+ : undefined;
176
+ if (reusableBlank) {
177
+ tabId = reusableBlank.id;
178
+ } else {
179
+ const beforeIds = new Set(tabs.map((tab) => tab.id));
180
+ const created = await run([...input.globals, 'tab', 'new', 'about:blank']);
181
+ if (created.code !== 0) {
182
+ throw new BrowserTabError(
183
+ 'tab_bootstrap_failed',
184
+ created.stderr || created.stdout || '无法创建会话浏览器标签',
185
+ );
186
+ }
187
+ const after = parseListedTabs(await run([...input.globals, 'tab', 'list', '--json']));
188
+ const added = after.filter((tab) => !beforeIds.has(tab.id));
189
+ tabId = (added.find((candidate) => candidate.active) ?? added.at(-1))?.id;
190
+ if (!tabId) {
191
+ throw new BrowserTabError('tab_bootstrap_failed', '新标签创建成功,但 Runtime 未返回其标签 ID');
192
+ }
193
+ }
194
+ }
195
+
196
+ try {
197
+ browserRunRegistry.claimTab(input.context, tabId);
198
+ } catch (error) {
199
+ throw new BrowserTabError(
200
+ 'tab_owned_by_other_conversation',
201
+ error instanceof Error ? error.message : String(error),
202
+ );
203
+ }
204
+ assertSwitched(await run([...input.globals, 'tab', tabId]));
205
+ return tabId;
206
+ });
207
+ }
208
+
209
+ /** Compatibility alias for callers compiled against the previous helper name. */
210
+ export const claimConversationTab = ensureConversationTab;