@vellumai/assistant 0.6.3 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (667) hide show
  1. package/ARCHITECTURE.md +273 -10
  2. package/Dockerfile +2 -3
  3. package/bun.lock +5 -13
  4. package/docs/backup-troubleshooting.md +52 -0
  5. package/docs/browser-use-architecture-phase2.md +174 -0
  6. package/docs/stt-provider-onboarding.md +120 -0
  7. package/knip.json +12 -2
  8. package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
  9. package/node_modules/@vellumai/ces-contracts/package.json +3 -3
  10. package/openapi.yaml +982 -72
  11. package/package.json +4 -6
  12. package/scripts/generate-openapi.ts +0 -1
  13. package/scripts/test.sh +73 -18
  14. package/src/__tests__/agent-image-optimize.test.ts +28 -0
  15. package/src/__tests__/agent-loop.test.ts +123 -0
  16. package/src/__tests__/anthropic-provider.test.ts +263 -10
  17. package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
  18. package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
  19. package/src/__tests__/browser-fill-credential.test.ts +11 -0
  20. package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
  21. package/src/__tests__/browser-skill-endstate.test.ts +31 -7
  22. package/src/__tests__/btw-routes.test.ts +7 -0
  23. package/src/__tests__/call-controller.test.ts +581 -20
  24. package/src/__tests__/catalog-files.test.ts +138 -0
  25. package/src/__tests__/channel-invite-transport.test.ts +2 -2
  26. package/src/__tests__/channel-readiness-routes.test.ts +16 -20
  27. package/src/__tests__/channel-readiness-service.test.ts +12 -7
  28. package/src/__tests__/checker.test.ts +157 -10
  29. package/src/__tests__/clawhub-files.test.ts +347 -0
  30. package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
  31. package/src/__tests__/config-analysis.test.ts +100 -0
  32. package/src/__tests__/config-schema.test.ts +1013 -66
  33. package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
  34. package/src/__tests__/config-watcher.test.ts +43 -8
  35. package/src/__tests__/contact-store-user-file.test.ts +512 -0
  36. package/src/__tests__/contacts-write.test.ts +197 -0
  37. package/src/__tests__/context-window-manager.test.ts +88 -0
  38. package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
  39. package/src/__tests__/conversation-agent-loop-overflow.test.ts +1 -0
  40. package/src/__tests__/conversation-agent-loop.test.ts +98 -2
  41. package/src/__tests__/conversation-confirmation-signals.test.ts +135 -0
  42. package/src/__tests__/conversation-error.test.ts +70 -0
  43. package/src/__tests__/conversation-history-web-search.test.ts +11 -4
  44. package/src/__tests__/conversation-init.benchmark.test.ts +6 -1
  45. package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
  46. package/src/__tests__/conversation-list-source.test.ts +145 -0
  47. package/src/__tests__/conversation-pre-run-repair.test.ts +2 -0
  48. package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
  49. package/src/__tests__/conversation-queue.test.ts +901 -60
  50. package/src/__tests__/conversation-routes-disk-view.test.ts +270 -0
  51. package/src/__tests__/conversation-runtime-assembly.test.ts +55 -0
  52. package/src/__tests__/conversation-skill-tools.test.ts +7 -4
  53. package/src/__tests__/conversation-slash-commands.test.ts +33 -0
  54. package/src/__tests__/conversation-slash-queue.test.ts +89 -18
  55. package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
  56. package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
  57. package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
  58. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
  59. package/src/__tests__/credential-health-service.test.ts +352 -0
  60. package/src/__tests__/credential-security-invariants.test.ts +5 -3
  61. package/src/__tests__/credential-vault-unit.test.ts +379 -3
  62. package/src/__tests__/credentials-cli.test.ts +40 -16
  63. package/src/__tests__/cross-provider-web-search.test.ts +146 -35
  64. package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
  65. package/src/__tests__/device-id.test.ts +112 -0
  66. package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
  67. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
  68. package/src/__tests__/email-html-renderer.test.ts +71 -0
  69. package/src/__tests__/email-invite-adapter.test.ts +36 -32
  70. package/src/__tests__/emit-event-signal.test.ts +71 -0
  71. package/src/__tests__/extension-id-sync-guard.test.ts +75 -8
  72. package/src/__tests__/fixtures/mock-chrome-extension.ts +11 -0
  73. package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
  74. package/src/__tests__/gateway-only-guard.test.ts +0 -1
  75. package/src/__tests__/gemini-provider.test.ts +64 -0
  76. package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
  77. package/src/__tests__/gmail-archive-fallback.test.ts +193 -0
  78. package/src/__tests__/gmail-archive-gate.test.ts +246 -0
  79. package/src/__tests__/gmail-preferences.test.ts +117 -0
  80. package/src/__tests__/headless-browser-interactions.test.ts +43 -0
  81. package/src/__tests__/headless-browser-mode.test.ts +614 -0
  82. package/src/__tests__/headless-browser-navigate.test.ts +142 -5
  83. package/src/__tests__/headless-browser-read-tools.test.ts +11 -0
  84. package/src/__tests__/headless-browser-snapshot.test.ts +10 -0
  85. package/src/__tests__/heartbeat-service.test.ts +70 -17
  86. package/src/__tests__/home-state-routes.test.ts +162 -0
  87. package/src/__tests__/host-bash-proxy.test.ts +0 -5
  88. package/src/__tests__/host-browser-e2e-cloud.test.ts +138 -4
  89. package/src/__tests__/host-browser-e2e-self-hosted.test.ts +4 -4
  90. package/src/__tests__/host-browser-ws-events-e2e.test.ts +103 -0
  91. package/src/__tests__/host-cu-proxy.test.ts +0 -5
  92. package/src/__tests__/identity-intro-cache.test.ts +40 -10
  93. package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
  94. package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
  95. package/src/__tests__/llm-context-normalization.test.ts +488 -0
  96. package/src/__tests__/llm-context-route-provider.test.ts +86 -5
  97. package/src/__tests__/llm-usage-store.test.ts +363 -0
  98. package/src/__tests__/media-stream-output.test.ts +555 -0
  99. package/src/__tests__/media-stream-parser.test.ts +374 -0
  100. package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
  101. package/src/__tests__/media-stream-stt-session.test.ts +588 -0
  102. package/src/__tests__/media-turn-detector.test.ts +440 -0
  103. package/src/__tests__/message-queue.test.ts +125 -0
  104. package/src/__tests__/migration-export-http.test.ts +6 -6
  105. package/src/__tests__/migration-import-commit-http.test.ts +8 -6
  106. package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
  107. package/src/__tests__/migration-validate-http.test.ts +3 -3
  108. package/src/__tests__/mock-gateway-ipc.ts +151 -0
  109. package/src/__tests__/model-intents.test.ts +2 -2
  110. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  111. package/src/__tests__/oauth-cli.test.ts +2 -0
  112. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +1 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +2 -0
  115. package/src/__tests__/oauth-store.test.ts +85 -0
  116. package/src/__tests__/oauth2-gateway-transport.test.ts +249 -6
  117. package/src/__tests__/onboarding-template-contract.test.ts +6 -13
  118. package/src/__tests__/openai-provider.test.ts +176 -0
  119. package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
  120. package/src/__tests__/openai-responses-provider.test.ts +1105 -0
  121. package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
  122. package/src/__tests__/outlook-unsubscribe.test.ts +31 -2
  123. package/src/__tests__/persona-resolver.test.ts +251 -0
  124. package/src/__tests__/platform-bash-auto-approve.test.ts +4 -0
  125. package/src/__tests__/platform.test.ts +92 -1
  126. package/src/__tests__/post-turn-tool-result-truncation.test.ts +47 -0
  127. package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
  128. package/src/__tests__/pricing.test.ts +174 -0
  129. package/src/__tests__/qdrant-manager.test.ts +29 -8
  130. package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
  131. package/src/__tests__/relationship-state-contract.test.ts +175 -0
  132. package/src/__tests__/relay-server.test.ts +423 -5
  133. package/src/__tests__/search-skills-unified.test.ts +118 -0
  134. package/src/__tests__/secret-scanner-executor.test.ts +4 -0
  135. package/src/__tests__/secure-keys.test.ts +107 -0
  136. package/src/__tests__/send-endpoint-busy.test.ts +5 -1
  137. package/src/__tests__/sequence-store.test.ts +1 -1
  138. package/src/__tests__/server-history-render.test.ts +49 -0
  139. package/src/__tests__/settings-routes.test.ts +201 -0
  140. package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
  141. package/src/__tests__/skills-file-content-endpoint.test.ts +276 -145
  142. package/src/__tests__/skills-files-catalog-fallback.test.ts +381 -93
  143. package/src/__tests__/skills.test.ts +5 -2
  144. package/src/__tests__/skillssh-files.test.ts +446 -0
  145. package/src/__tests__/slack-block-formatting.test.ts +110 -0
  146. package/src/__tests__/slack-channel-config.test.ts +564 -1
  147. package/src/__tests__/stt-catalog-parity.test.ts +282 -0
  148. package/src/__tests__/stt-stream-session.test.ts +535 -0
  149. package/src/__tests__/system-prompt.test.ts +112 -26
  150. package/src/__tests__/telephony-stt-routing.test.ts +329 -0
  151. package/src/__tests__/terminal-tools.test.ts +18 -7
  152. package/src/__tests__/test-preload.ts +18 -0
  153. package/src/__tests__/test-support/browser-skill-harness.ts +4 -1
  154. package/src/__tests__/tool-executor-lifecycle-events.test.ts +9 -5
  155. package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
  156. package/src/__tests__/tool-executor.test.ts +33 -24
  157. package/src/__tests__/tool-result-truncation.test.ts +36 -0
  158. package/src/__tests__/trust-store.test.ts +7 -1
  159. package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
  160. package/src/__tests__/tts-catalog-parity.test.ts +345 -0
  161. package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
  162. package/src/__tests__/twilio-routes.test.ts +376 -0
  163. package/src/__tests__/unicode.test.ts +293 -0
  164. package/src/__tests__/update-bulletin-format.test.ts +59 -0
  165. package/src/__tests__/update-bulletin.test.ts +206 -5
  166. package/src/__tests__/usage-routes.test.ts +25 -4
  167. package/src/__tests__/user-reference.test.ts +46 -61
  168. package/src/__tests__/verification-control-plane-policy.test.ts +4 -0
  169. package/src/__tests__/voice-config-update.test.ts +403 -0
  170. package/src/__tests__/voice-quality.test.ts +434 -19
  171. package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
  172. package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
  173. package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
  174. package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
  175. package/src/__tests__/workspace-migration-meets.test.ts +244 -0
  176. package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
  177. package/src/__tests__/workspace-policy.test.ts +2 -0
  178. package/src/agent/image-optimize.ts +24 -12
  179. package/src/agent/loop.ts +43 -3
  180. package/src/backup/__tests__/backup-key.test.ts +152 -0
  181. package/src/backup/__tests__/backup-worker.test.ts +767 -0
  182. package/src/backup/__tests__/list-snapshots.test.ts +87 -0
  183. package/src/backup/__tests__/local-writer.test.ts +218 -0
  184. package/src/backup/__tests__/offsite-writer.test.ts +641 -0
  185. package/src/backup/__tests__/paths.test.ts +300 -0
  186. package/src/backup/__tests__/restore.test.ts +498 -0
  187. package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
  188. package/src/backup/__tests__/stream-crypt.test.ts +228 -0
  189. package/src/backup/backup-key.ts +137 -0
  190. package/src/backup/backup-worker.ts +459 -0
  191. package/src/backup/list-snapshots.ts +147 -0
  192. package/src/backup/local-writer.ts +133 -0
  193. package/src/backup/offsite-writer.ts +222 -0
  194. package/src/backup/paths.ts +226 -0
  195. package/src/backup/restore.ts +322 -0
  196. package/src/backup/snapshot-lock.ts +431 -0
  197. package/src/backup/stream-crypt.ts +263 -0
  198. package/src/bundler/package-resolver.ts +4 -0
  199. package/src/calls/audio-store.ts +11 -5
  200. package/src/calls/call-controller.ts +226 -71
  201. package/src/calls/call-domain.ts +9 -0
  202. package/src/calls/call-speech-output.ts +190 -0
  203. package/src/calls/call-transport.ts +77 -0
  204. package/src/calls/media-stream-audio-transcode.ts +173 -0
  205. package/src/calls/media-stream-output.ts +660 -0
  206. package/src/calls/media-stream-parser.ts +300 -0
  207. package/src/calls/media-stream-protocol.ts +166 -0
  208. package/src/calls/media-stream-server.ts +592 -0
  209. package/src/calls/media-stream-stt-session.ts +460 -0
  210. package/src/calls/media-turn-detector.ts +230 -0
  211. package/src/calls/relay-server.ts +90 -75
  212. package/src/calls/resolve-call-tts-provider.ts +136 -0
  213. package/src/calls/telephony-stt-routing.ts +145 -0
  214. package/src/calls/tts-call-strategy.ts +161 -0
  215. package/src/calls/tts-text-sanitizer.ts +32 -16
  216. package/src/calls/twilio-routes.ts +281 -17
  217. package/src/calls/voice-quality.ts +78 -35
  218. package/src/calls/voice-session-bridge.ts +8 -1
  219. package/src/channels/types.ts +16 -0
  220. package/src/cli/__tests__/run-assistant-command.ts +11 -1
  221. package/src/cli/commands/__tests__/backup.test.ts +1165 -0
  222. package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
  223. package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
  224. package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
  225. package/src/cli/commands/__tests__/email-download.test.ts +16 -1
  226. package/src/cli/commands/__tests__/email-list.test.ts +22 -4
  227. package/src/cli/commands/__tests__/email-register.test.ts +4 -4
  228. package/src/cli/commands/__tests__/email-send.test.ts +37 -4
  229. package/src/cli/commands/__tests__/email-status.test.ts +5 -1
  230. package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
  231. package/src/cli/commands/backup.ts +993 -0
  232. package/src/cli/commands/conversations.ts +77 -0
  233. package/src/cli/commands/credentials.ts +0 -1
  234. package/src/cli/commands/domain.ts +210 -0
  235. package/src/cli/commands/email.ts +255 -3
  236. package/src/cli/commands/oauth/__tests__/connect.test.ts +12 -0
  237. package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +1 -0
  238. package/src/cli/commands/oauth/__tests__/providers-register.test.ts +1 -0
  239. package/src/cli/commands/oauth/__tests__/providers-update.test.ts +1 -0
  240. package/src/cli/commands/oauth/mode.ts +12 -3
  241. package/src/cli/commands/oauth/providers.ts +15 -0
  242. package/src/cli/commands/oauth/shared.ts +2 -1
  243. package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +4 -9
  244. package/src/cli/commands/platform/__tests__/connect.test.ts +6 -0
  245. package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -1
  246. package/src/cli/commands/platform/__tests__/status.test.ts +6 -0
  247. package/src/cli/program.ts +30 -4
  248. package/src/config/__tests__/backup-schema.test.ts +134 -0
  249. package/src/config/assistant-feature-flags.ts +61 -62
  250. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +37 -1
  251. package/src/config/bundled-skills/browser/SKILL.md +30 -5
  252. package/src/config/bundled-skills/browser/TOOLS.json +123 -0
  253. package/src/config/bundled-skills/browser/tools/browser-attach.ts +12 -0
  254. package/src/config/bundled-skills/browser/tools/browser-detach.ts +12 -0
  255. package/src/config/bundled-skills/browser/tools/browser-status.ts +12 -0
  256. package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +17 -0
  257. package/src/config/bundled-skills/contacts/SKILL.md +2 -2
  258. package/src/config/bundled-skills/gmail/SKILL.md +53 -7
  259. package/src/config/bundled-skills/gmail/TOOLS.json +33 -3
  260. package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +116 -9
  261. package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +138 -11
  262. package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +59 -0
  263. package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +82 -0
  264. package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +113 -17
  265. package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -2
  266. package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
  267. package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
  268. package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
  269. package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
  270. package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
  271. package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
  272. package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
  273. package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
  274. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  275. package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +2 -2
  276. package/src/config/bundled-skills/outlook/SKILL.md +2 -2
  277. package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +2 -2
  278. package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
  279. package/src/config/bundled-skills/phone-calls/references/CONFIG.md +27 -18
  280. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
  281. package/src/config/bundled-skills/settings/TOOLS.json +3 -3
  282. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
  283. package/src/config/bundled-skills/slack/SKILL.md +1 -0
  284. package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
  285. package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
  286. package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
  287. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
  288. package/src/config/bundled-tool-registry.ts +8 -0
  289. package/src/config/env-registry.ts +24 -0
  290. package/src/config/env.ts +34 -10
  291. package/src/config/feature-flag-registry.json +46 -14
  292. package/src/config/loader.ts +26 -12
  293. package/src/config/schema.ts +35 -10
  294. package/src/config/schemas/__tests__/stt.test.ts +43 -0
  295. package/src/config/schemas/analysis.ts +51 -0
  296. package/src/config/schemas/backup.ts +72 -0
  297. package/src/config/schemas/calls.ts +1 -26
  298. package/src/config/schemas/elevenlabs.ts +0 -59
  299. package/src/config/schemas/filing.ts +47 -7
  300. package/src/config/schemas/heartbeat.ts +27 -5
  301. package/src/config/schemas/host-browser.ts +47 -1
  302. package/src/config/schemas/inference.ts +1 -1
  303. package/src/config/schemas/memory-lifecycle.ts +14 -2
  304. package/src/config/schemas/services.ts +44 -0
  305. package/src/config/schemas/stt.ts +59 -0
  306. package/src/config/schemas/tts.ts +230 -0
  307. package/src/config/schemas/updates.ts +14 -0
  308. package/src/config/skills.ts +4 -0
  309. package/src/config/types.ts +4 -0
  310. package/src/contacts/contact-store.ts +56 -11
  311. package/src/contacts/contacts-write.ts +38 -1
  312. package/src/context/post-turn-tool-result-truncation.ts +3 -2
  313. package/src/context/tool-result-truncation.ts +2 -1
  314. package/src/context/window-manager.ts +45 -12
  315. package/src/credential-execution/executable-discovery.ts +12 -2
  316. package/src/credential-execution/process-manager.ts +33 -2
  317. package/src/credential-health/credential-health-service.ts +366 -0
  318. package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
  319. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
  320. package/src/daemon/__tests__/conversation-tool-setup.test.ts +17 -8
  321. package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
  322. package/src/daemon/config-watcher.ts +99 -5
  323. package/src/daemon/conversation-agent-loop-handlers.ts +6 -0
  324. package/src/daemon/conversation-agent-loop.ts +101 -24
  325. package/src/daemon/conversation-error.ts +11 -0
  326. package/src/daemon/conversation-history.ts +40 -6
  327. package/src/daemon/conversation-launch.ts +220 -0
  328. package/src/daemon/conversation-lifecycle.ts +59 -9
  329. package/src/daemon/conversation-messaging.ts +37 -3
  330. package/src/daemon/conversation-notifiers.ts +5 -0
  331. package/src/daemon/conversation-process.ts +581 -19
  332. package/src/daemon/conversation-queue-manager.ts +24 -0
  333. package/src/daemon/conversation-runtime-assembly.ts +11 -1
  334. package/src/daemon/conversation-slash.ts +36 -0
  335. package/src/daemon/conversation-surfaces.ts +94 -4
  336. package/src/daemon/conversation-tool-setup.ts +25 -0
  337. package/src/daemon/conversation-usage.ts +7 -4
  338. package/src/daemon/conversation.ts +86 -28
  339. package/src/daemon/handlers/config-slack-channel.ts +269 -94
  340. package/src/daemon/handlers/conversations.ts +4 -1
  341. package/src/daemon/handlers/shared.ts +22 -0
  342. package/src/daemon/handlers/skills.ts +321 -77
  343. package/src/daemon/host-browser-proxy.ts +2 -1
  344. package/src/daemon/lifecycle.ts +122 -25
  345. package/src/daemon/message-protocol.ts +6 -0
  346. package/src/daemon/message-types/conversations.ts +34 -1
  347. package/src/daemon/message-types/home.ts +40 -0
  348. package/src/daemon/message-types/meet.ts +143 -0
  349. package/src/daemon/message-types/messages.ts +14 -0
  350. package/src/daemon/message-types/schedules.ts +34 -2
  351. package/src/daemon/message-types/skills.ts +16 -0
  352. package/src/daemon/message-types/surfaces.ts +2 -0
  353. package/src/daemon/server.ts +347 -2
  354. package/src/daemon/shutdown-handlers.ts +32 -4
  355. package/src/daemon/shutdown-registry.ts +40 -0
  356. package/src/daemon/tool-side-effects.ts +9 -0
  357. package/src/email/html-renderer.ts +76 -0
  358. package/src/heartbeat/heartbeat-service.ts +93 -7
  359. package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
  360. package/src/home/__tests__/emit-feed-event.test.ts +169 -0
  361. package/src/home/__tests__/feed-scheduler.test.ts +194 -0
  362. package/src/home/__tests__/feed-types.test.ts +275 -0
  363. package/src/home/__tests__/feed-writer.test.ts +688 -0
  364. package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
  365. package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
  366. package/src/home/__tests__/progress-formula.test.ts +213 -0
  367. package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
  368. package/src/home/__tests__/rollup-producer.test.ts +398 -0
  369. package/src/home/assistant-feed-authoring.ts +124 -0
  370. package/src/home/emit-feed-event.ts +158 -0
  371. package/src/home/feed-scheduler.ts +247 -0
  372. package/src/home/feed-types.ts +181 -0
  373. package/src/home/feed-writer.ts +469 -0
  374. package/src/home/platform-gmail-digest.ts +163 -0
  375. package/src/home/progress-formula.ts +86 -0
  376. package/src/home/relationship-state-writer.ts +824 -0
  377. package/src/home/relationship-state.ts +143 -0
  378. package/src/home/rollup-producer.ts +384 -0
  379. package/src/hooks/runner.ts +7 -0
  380. package/src/inbound/platform-callback-registration.ts +12 -3
  381. package/src/inbound/public-ingress-urls.ts +12 -0
  382. package/src/instrument.ts +1 -1
  383. package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
  384. package/src/ipc/cli-client.ts +151 -0
  385. package/src/ipc/cli-server.ts +234 -0
  386. package/src/ipc/gateway-client.ts +180 -0
  387. package/src/ipc/routes/index.ts +5 -0
  388. package/src/ipc/routes/wake-conversation.ts +19 -0
  389. package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
  390. package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
  391. package/src/memory/__tests__/conversation-analyze-job.test.ts +232 -0
  392. package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
  393. package/src/memory/app-store.ts +1 -1
  394. package/src/memory/attachments-store.ts +70 -0
  395. package/src/memory/auto-analysis-enqueue.ts +127 -0
  396. package/src/memory/auto-analysis-guard.ts +27 -0
  397. package/src/memory/cleanup-schedule-state.ts +37 -0
  398. package/src/memory/conversation-analyze-job.ts +73 -0
  399. package/src/memory/conversation-crud.ts +99 -0
  400. package/src/memory/conversation-disk-view.ts +7 -0
  401. package/src/memory/conversation-group-migration.ts +34 -2
  402. package/src/memory/conversation-queries.ts +6 -5
  403. package/src/memory/db-init.ts +6 -0
  404. package/src/memory/db-maintenance.ts +108 -0
  405. package/src/memory/db.ts +1 -0
  406. package/src/memory/graph/conversation-graph-memory.ts +15 -0
  407. package/src/memory/graph/extraction.test.ts +23 -0
  408. package/src/memory/graph/extraction.ts +8 -0
  409. package/src/memory/graph/retriever.ts +27 -18
  410. package/src/memory/graph/scoring.test.ts +186 -0
  411. package/src/memory/graph/scoring.ts +31 -1
  412. package/src/memory/graph/tools.ts +1 -1
  413. package/src/memory/group-crud.ts +6 -1
  414. package/src/memory/indexer.ts +95 -16
  415. package/src/memory/job-handlers/cleanup.ts +11 -8
  416. package/src/memory/job-handlers/conversation-starters.ts +16 -10
  417. package/src/memory/jobs-store.ts +64 -4
  418. package/src/memory/jobs-worker.ts +22 -9
  419. package/src/memory/llm-usage-store.ts +92 -56
  420. package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
  421. package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
  422. package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
  423. package/src/memory/migrations/index.ts +6 -0
  424. package/src/memory/migrations/registry.ts +8 -0
  425. package/src/memory/qdrant-manager.ts +43 -16
  426. package/src/memory/schema/conversations.ts +2 -0
  427. package/src/memory/schema/oauth.ts +3 -0
  428. package/src/memory/usage-buckets.ts +396 -0
  429. package/src/messaging/providers/gmail/client.ts +57 -6
  430. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
  431. package/src/messaging/providers/slack/adapter.ts +143 -38
  432. package/src/messaging/providers/slack/client.ts +16 -0
  433. package/src/messaging/providers/slack/types.ts +4 -0
  434. package/src/notifications/decision-engine.ts +3 -3
  435. package/src/notifications/signal.ts +5 -0
  436. package/src/oauth/__tests__/identity-verifier.test.ts +1 -0
  437. package/src/oauth/byo-connection.test.ts +18 -1
  438. package/src/oauth/byo-connection.ts +3 -1
  439. package/src/oauth/connect-orchestrator.ts +2 -0
  440. package/src/oauth/connection-resolver.ts +6 -2
  441. package/src/oauth/connection.ts +2 -0
  442. package/src/oauth/oauth-store.ts +9 -0
  443. package/src/oauth/platform-connection.test.ts +98 -0
  444. package/src/oauth/platform-connection.ts +52 -31
  445. package/src/oauth/seed-providers.ts +7 -0
  446. package/src/permissions/checker.ts +16 -6
  447. package/src/permissions/defaults.ts +49 -1
  448. package/src/permissions/trust-store.ts +3 -3
  449. package/src/permissions/workspace-policy.ts +3 -0
  450. package/src/platform/client.test.ts +10 -0
  451. package/src/platform/sync-identity.ts +129 -0
  452. package/src/prompts/persona-resolver.ts +126 -2
  453. package/src/prompts/system-prompt.ts +59 -18
  454. package/src/prompts/templates/BOOTSTRAP.md +5 -5
  455. package/src/prompts/templates/SOUL.md +3 -1
  456. package/src/prompts/templates/UPDATES.md +12 -0
  457. package/src/prompts/templates/channels/slack.md +20 -0
  458. package/src/prompts/update-bulletin-format.ts +26 -9
  459. package/src/prompts/update-bulletin.ts +34 -23
  460. package/src/prompts/user-reference.ts +20 -17
  461. package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
  462. package/src/providers/anthropic/client.ts +157 -61
  463. package/src/providers/fireworks/client.ts +2 -2
  464. package/src/providers/gemini/client.ts +9 -1
  465. package/src/providers/model-catalog.ts +6 -0
  466. package/src/providers/model-intents.ts +4 -4
  467. package/src/providers/ollama/client.ts +2 -2
  468. package/src/providers/openai/chat-completions-provider.ts +474 -0
  469. package/src/providers/openai/client.ts +25 -440
  470. package/src/providers/openai/responses-provider.ts +502 -0
  471. package/src/providers/openrouter/client.ts +101 -4
  472. package/src/providers/provider-secret-catalog.ts +139 -0
  473. package/src/providers/registry.ts +2 -2
  474. package/src/providers/retry.ts +14 -3
  475. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
  476. package/src/providers/speech-to-text/__tests__/resolve.test.ts +828 -0
  477. package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
  478. package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
  479. package/src/providers/speech-to-text/deepgram.test.ts +332 -0
  480. package/src/providers/speech-to-text/deepgram.ts +115 -0
  481. package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
  482. package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
  483. package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
  484. package/src/providers/speech-to-text/google-gemini.ts +101 -0
  485. package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
  486. package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
  487. package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
  488. package/src/providers/speech-to-text/openai-whisper.ts +63 -33
  489. package/src/providers/speech-to-text/provider-catalog.ts +306 -0
  490. package/src/providers/speech-to-text/resolve.ts +386 -6
  491. package/src/providers/types.ts +9 -0
  492. package/src/runtime/AGENTS.md +43 -1
  493. package/src/runtime/__tests__/agent-wake.test.ts +831 -0
  494. package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
  495. package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
  496. package/src/runtime/agent-wake.ts +512 -0
  497. package/src/runtime/auth/__tests__/route-policy.test.ts +40 -0
  498. package/src/runtime/auth/route-policy.ts +30 -5
  499. package/src/runtime/auth/token-service.ts +56 -1
  500. package/src/runtime/btw-sidechain.ts +2 -0
  501. package/src/runtime/capability-tokens.ts +10 -10
  502. package/src/runtime/channel-invite-transport.ts +1 -1
  503. package/src/runtime/channel-invite-transports/email.ts +14 -6
  504. package/src/runtime/channel-readiness-service.ts +12 -22
  505. package/src/runtime/chrome-extension-registry.ts +38 -2
  506. package/src/runtime/http-server.ts +395 -10
  507. package/src/runtime/http-types.ts +6 -2
  508. package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +36 -0
  509. package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
  510. package/src/runtime/migrations/migration-transport.ts +1 -0
  511. package/src/runtime/migrations/migration-wizard.ts +1 -0
  512. package/src/runtime/migrations/vbundle-import-analyzer.ts +77 -1
  513. package/src/runtime/migrations/vbundle-importer.ts +34 -0
  514. package/src/runtime/pending-interactions.ts +0 -11
  515. package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
  516. package/src/runtime/routes/__tests__/home-feed-routes.test.ts +507 -0
  517. package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +208 -0
  518. package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
  519. package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
  520. package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
  521. package/src/runtime/routes/app-management-routes.ts +12 -18
  522. package/src/runtime/routes/attachment-routes.test.ts +9 -3
  523. package/src/runtime/routes/attachment-routes.ts +216 -17
  524. package/src/runtime/routes/backup-routes.ts +519 -0
  525. package/src/runtime/routes/browser-extension-pair-routes.ts +82 -23
  526. package/src/runtime/routes/btw-routes.ts +8 -6
  527. package/src/runtime/routes/contact-routes.test.ts +298 -0
  528. package/src/runtime/routes/contact-routes.ts +132 -5
  529. package/src/runtime/routes/conversation-analysis-routes.ts +22 -142
  530. package/src/runtime/routes/conversation-management-routes.ts +115 -0
  531. package/src/runtime/routes/conversation-routes.ts +367 -146
  532. package/src/runtime/routes/filing-routes.ts +93 -0
  533. package/src/runtime/routes/home-feed-routes.ts +334 -0
  534. package/src/runtime/routes/home-state-routes.ts +138 -0
  535. package/src/runtime/routes/host-browser-routes.ts +3 -14
  536. package/src/runtime/routes/identity-intro-cache.ts +7 -3
  537. package/src/runtime/routes/identity-routes.ts +3 -17
  538. package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
  539. package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
  540. package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
  541. package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
  542. package/src/runtime/routes/integrations/slack/channel.ts +11 -3
  543. package/src/runtime/routes/integrations/slack/share.ts +45 -7
  544. package/src/runtime/routes/llm-context-normalization.ts +303 -0
  545. package/src/runtime/routes/memory-item-routes.test.ts +3 -2
  546. package/src/runtime/routes/migration-routes.ts +40 -5
  547. package/src/runtime/routes/settings-routes.ts +22 -5
  548. package/src/runtime/routes/skills-routes.ts +76 -7
  549. package/src/runtime/routes/stt-routes.ts +233 -0
  550. package/src/runtime/routes/surface-action-routes.ts +41 -2
  551. package/src/runtime/routes/tts-routes.ts +108 -24
  552. package/src/runtime/routes/usage-routes.ts +30 -2
  553. package/src/runtime/routes/user-route-dispatcher.ts +50 -5
  554. package/src/runtime/routes/user-routes.ts +13 -1
  555. package/src/runtime/routes/work-items-routes.ts +8 -1
  556. package/src/runtime/runtime-mode.ts +33 -0
  557. package/src/runtime/services/__tests__/analyze-conversation.test.ts +444 -0
  558. package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
  559. package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
  560. package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
  561. package/src/runtime/services/analyze-conversation.ts +344 -0
  562. package/src/runtime/services/analyze-deps-singleton.ts +32 -0
  563. package/src/runtime/services/auto-analysis-prompt.ts +55 -0
  564. package/src/runtime/skill-route-registry.ts +49 -0
  565. package/src/runtime/slack-block-formatting.ts +437 -10
  566. package/src/schedule/scheduler.ts +50 -0
  567. package/src/security/oauth2.ts +26 -4
  568. package/src/security/secure-keys.ts +25 -2
  569. package/src/security/token-manager.ts +8 -0
  570. package/src/sequence/engine.ts +23 -0
  571. package/src/sequence/types.ts +1 -1
  572. package/src/skills/catalog-files.ts +64 -2
  573. package/src/skills/category-inference.ts +122 -0
  574. package/src/skills/clawhub-files.ts +213 -0
  575. package/src/skills/clawhub.ts +84 -23
  576. package/src/skills/skill-file-provider.ts +40 -0
  577. package/src/skills/skillssh-files.ts +395 -0
  578. package/src/skills/skillssh-registry.ts +4 -4
  579. package/src/stt/__tests__/daemon-batch-transcriber.test.ts +392 -0
  580. package/src/stt/__tests__/types.test.ts +89 -0
  581. package/src/stt/daemon-batch-transcriber.ts +195 -0
  582. package/src/stt/stt-stream-session.ts +499 -0
  583. package/src/stt/types.ts +330 -0
  584. package/src/stt/wav-encoder.test.ts +373 -0
  585. package/src/stt/wav-encoder.ts +175 -0
  586. package/src/subagent/manager.ts +38 -14
  587. package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
  588. package/src/tools/browser/__tests__/browser-status.test.ts +123 -0
  589. package/src/tools/browser/browser-execution.ts +1163 -23
  590. package/src/tools/browser/browser-manager.ts +45 -0
  591. package/src/tools/browser/browser-mode-constants.ts +12 -0
  592. package/src/tools/browser/browser-mode.ts +92 -0
  593. package/src/tools/browser/browser-status-constants.ts +33 -0
  594. package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +393 -0
  595. package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +29 -0
  596. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1648 -32
  597. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +264 -0
  598. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +183 -17
  599. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +254 -21
  600. package/src/tools/browser/cdp-client/errors.ts +15 -0
  601. package/src/tools/browser/cdp-client/extension-cdp-client.ts +39 -16
  602. package/src/tools/browser/cdp-client/factory.ts +797 -87
  603. package/src/tools/browser/cdp-client/index.ts +16 -2
  604. package/src/tools/browser/cdp-client/types.ts +68 -0
  605. package/src/tools/credentials/vault.ts +35 -6
  606. package/src/tools/network/web-fetch.ts +5 -2
  607. package/src/tools/network/web-search.ts +5 -2
  608. package/src/tools/shared/shell-output.ts +3 -1
  609. package/src/tools/side-effects.ts +2 -0
  610. package/src/tools/skills/sandbox-runner.ts +3 -2
  611. package/src/tools/terminal/safe-env.ts +10 -2
  612. package/src/tools/terminal/shell.ts +15 -4
  613. package/src/tools/tool-manifest.ts +21 -0
  614. package/src/tools/types.ts +17 -0
  615. package/src/tools/ui-surface/definitions.ts +6 -1
  616. package/src/tts/__tests__/provider-adapters.test.ts +834 -0
  617. package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
  618. package/src/tts/__tests__/provider-catalog.test.ts +183 -0
  619. package/src/tts/__tests__/provider-registry.test.ts +90 -0
  620. package/src/tts/provider-catalog.ts +201 -0
  621. package/src/tts/provider-registry.ts +73 -0
  622. package/src/tts/providers/deepgram-provider.ts +219 -0
  623. package/src/tts/providers/elevenlabs-provider.ts +211 -0
  624. package/src/tts/providers/fish-audio-provider.ts +183 -0
  625. package/src/tts/providers/index.ts +42 -0
  626. package/src/tts/providers/register-builtins.ts +130 -0
  627. package/src/tts/synthesize-text.ts +110 -0
  628. package/src/tts/tts-config-resolver.ts +78 -0
  629. package/src/tts/types.ts +153 -0
  630. package/src/types/onboarding-context.ts +7 -0
  631. package/src/util/abort-reasons.ts +58 -0
  632. package/src/util/device-id.ts +32 -16
  633. package/src/util/errors.ts +9 -1
  634. package/src/util/platform.ts +54 -10
  635. package/src/util/pricing.ts +66 -3
  636. package/src/util/spawn.ts +1 -1
  637. package/src/util/truncate.ts +4 -2
  638. package/src/util/unicode.ts +201 -0
  639. package/src/version.ts +19 -24
  640. package/src/watcher/engine.ts +23 -0
  641. package/src/watcher/watcher-store.ts +31 -0
  642. package/src/workspace/migrations/003-seed-device-id.ts +9 -3
  643. package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
  644. package/src/workspace/migrations/029-seed-pkb.ts +1 -1
  645. package/src/workspace/migrations/031-drop-user-md.ts +317 -0
  646. package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
  647. package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
  648. package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
  649. package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
  650. package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
  651. package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
  652. package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
  653. package/src/workspace/migrations/registry.ts +16 -0
  654. package/src/workspace/top-level-renderer.ts +13 -1
  655. package/src/workspace/turn-commit.ts +31 -0
  656. package/src/__tests__/email-cli.test.ts +0 -297
  657. package/src/__tests__/email-service-config-fallback.test.ts +0 -102
  658. package/src/cli/commands/browser-relay.ts +0 -466
  659. package/src/email/guardrails.ts +0 -221
  660. package/src/email/provider.ts +0 -117
  661. package/src/email/providers/agentmail.ts +0 -361
  662. package/src/email/providers/index.ts +0 -65
  663. package/src/email/service.ts +0 -384
  664. package/src/email/types.ts +0 -126
  665. package/src/prompts/templates/USER.md +0 -13
  666. package/src/providers/speech-to-text/types.ts +0 -17
  667. package/src/runtime/routes/browser-cdp-routes.ts +0 -229
@@ -1,6 +1,8 @@
1
+ import { getConfig } from "../../config/loader.js";
1
2
  import type { ImageContent } from "../../providers/types.js";
2
3
  import { getLogger } from "../../util/logger.js";
3
4
  import { truncate } from "../../util/truncate.js";
5
+ import { safeStringSlice } from "../../util/unicode.js";
4
6
  import { credentialBroker } from "../credentials/broker.js";
5
7
  import {
6
8
  isPrivateOrLocalHost,
@@ -17,12 +19,22 @@ import {
17
19
  } from "./auth-detector.js";
18
20
  import type { RouteHandler } from "./browser-manager.js";
19
21
  import { browserManager } from "./browser-manager.js";
22
+ import { type BrowserMode, normalizeBrowserMode } from "./browser-mode.js";
23
+ import { BROWSER_MODE } from "./browser-mode-constants.js";
20
24
  import {
21
25
  ensureScreencast,
22
26
  getSender,
23
27
  stopAllScreencasts,
24
28
  stopBrowserScreencast,
25
29
  } from "./browser-screencast.js";
30
+ import {
31
+ BROWSER_STATUS_INPUT_FIELD,
32
+ BROWSER_STATUS_MODE,
33
+ BROWSER_STATUS_MODES,
34
+ type BrowserStatusMode,
35
+ CDP_INSPECT_STATUS_DISCOVERY_CODE,
36
+ EXTENSION_STATUS_ERROR_MARKER,
37
+ } from "./browser-status-constants.js";
26
38
  import {
27
39
  formatAxSnapshot,
28
40
  transformAxTree,
@@ -45,8 +57,18 @@ import {
45
57
  waitForSelector as cdpWaitForSelector,
46
58
  waitForText as cdpWaitForText,
47
59
  } from "./cdp-client/cdp-dom-helpers.js";
48
- import { getCdpClient } from "./cdp-client/factory.js";
49
- import type { CdpClient } from "./cdp-client/types.js";
60
+ import { CdpError } from "./cdp-client/errors.js";
61
+ import {
62
+ buildCandidateList,
63
+ getCdpClient,
64
+ isDesktopAutoCooldownActive,
65
+ } from "./cdp-client/factory.js";
66
+ import type {
67
+ AttemptDiagnostic,
68
+ CdpClient,
69
+ CdpClientKind,
70
+ } from "./cdp-client/types.js";
71
+ import { checkBrowserRuntime } from "./runtime-check.js";
50
72
 
51
73
  const log = getLogger("headless-browser");
52
74
 
@@ -60,6 +82,37 @@ export const MAX_WAIT_MS = 30_000;
60
82
 
61
83
  export const MAX_EXTRACT_LENGTH = 50_000;
62
84
 
85
+ type StatusCheckMode = BrowserStatusMode;
86
+
87
+ const MODE_TRADEOFFS: Record<StatusCheckMode, string[]> = {
88
+ [BROWSER_STATUS_MODE.EXTENSION]: [
89
+ "Controls the user's existing Chrome profile and tabs.",
90
+ "Requires the Vellum extension to be paired and actively connected.",
91
+ "Best when the user wants the assistant to operate in their real browser session.",
92
+ ],
93
+ [BROWSER_STATUS_MODE.CDP_INSPECT]: [
94
+ "Controls an existing Chrome instance launched with --remote-debugging-port.",
95
+ "Does not require the extension to be connected.",
96
+ "Requires remote debugging to stay enabled on localhost, which is more manual to maintain.",
97
+ ],
98
+ [BROWSER_STATUS_MODE.LOCAL]: [
99
+ "Runs a dedicated Playwright-managed Chromium profile.",
100
+ "Most isolated and reliable fallback when extension/CDP inspect are unavailable.",
101
+ "Does not use the user's existing browser profile, so sessions/cookies may differ.",
102
+ ],
103
+ };
104
+
105
+ interface BrowserStatusModeResult {
106
+ mode: StatusCheckMode;
107
+ available: boolean;
108
+ verified: "active_probe" | "preflight";
109
+ autoCandidate: boolean;
110
+ summary: string;
111
+ userActions: string[];
112
+ tradeoffs: string[];
113
+ details: Record<string, unknown>;
114
+ }
115
+
63
116
  /**
64
117
  * IIFE evaluated inside the page via `Runtime.evaluate` to auto-dismiss
65
118
  * common blocker modals (regulatory notices, cookie banners) that
@@ -99,6 +152,329 @@ export const EXTRACT_LINKS_EXPRESSION = `
99
152
  })()
100
153
  `;
101
154
 
155
+ // ── browser_mode parsing ─────────────────────────────────────────────
156
+
157
+ /**
158
+ * Parse the `browser_mode` field from a tool input map. Returns either
159
+ * a normalized {@link BrowserMode} or a pre-formatted error string
160
+ * suitable for returning directly in a tool response.
161
+ *
162
+ * When the value is absent, undefined, or empty the default `"auto"`
163
+ * is returned. Invalid values produce a descriptive error listing
164
+ * accepted values and aliases.
165
+ */
166
+ export function parseBrowserMode(
167
+ input: Record<string, unknown>,
168
+ ): { ok: true; mode: BrowserMode } | { ok: false; error: string } {
169
+ const raw = input.browser_mode;
170
+ const result = normalizeBrowserMode(raw);
171
+ if ("error" in result) {
172
+ return { ok: false, error: `Error: ${result.error}` };
173
+ }
174
+ return { ok: true, mode: result.mode };
175
+ }
176
+
177
+ // ── Mode-selection failure formatter ─────────────────────────────────
178
+
179
+ /**
180
+ * Remediation hints keyed by (candidateKind, discoveryCode | errorCode).
181
+ * Discovery codes come from DevToolsDiscoveryError; error codes come
182
+ * from CdpError. The formatter walks these in priority order: exact
183
+ * (kind, discoveryCode) first, then (kind, errorCode), then a generic
184
+ * per-kind fallback.
185
+ */
186
+ const REMEDIATION_HINTS: Record<string, string[]> = {
187
+ // Extension backend
188
+ "extension:transport_error": [
189
+ "Ensure the Vellum browser extension is installed and enabled.",
190
+ "Check that the extension WebSocket connection is active (extension popup → status).",
191
+ "Try reconnecting the extension or reloading the extension service worker.",
192
+ ],
193
+ // cdp-inspect backend — discovery-level failures
194
+ "cdp-inspect:unreachable": [
195
+ "Ensure Chrome/Chromium is running with --remote-debugging-port=9222.",
196
+ "Verify no firewall or antivirus is blocking localhost:9222.",
197
+ "Try: /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222",
198
+ ],
199
+ "cdp-inspect:non_chrome": [
200
+ "The process listening on the configured port is not Chrome/Chromium.",
201
+ "Check if another application (dev server, proxy) is using port 9222.",
202
+ "Ensure Chrome is launched with --remote-debugging-port=9222.",
203
+ ],
204
+ "cdp-inspect:timeout": [
205
+ "Chrome DevTools endpoint did not respond within the probe timeout.",
206
+ "Ensure Chrome is running and listening on the configured port.",
207
+ "Try increasing hostBrowser.cdpInspect.probeTimeoutMs in config.",
208
+ ],
209
+ "cdp-inspect:no_targets": [
210
+ "Chrome is reachable but has no open page targets.",
211
+ "Open at least one browser tab, then retry.",
212
+ ],
213
+ "cdp-inspect:non_loopback": [
214
+ "CDP inspect only allows loopback hosts (localhost, 127.0.0.1, ::1).",
215
+ "Update hostBrowser.cdpInspect.host in config to a loopback address.",
216
+ ],
217
+ "cdp-inspect:transport_error": [
218
+ "CDP endpoint unreachable. Ensure Chrome is running with --remote-debugging-port.",
219
+ "Verify the configured host:port matches Chrome's DevTools listener.",
220
+ "Consider using browser_mode: 'extension' or 'local' as an alternative.",
221
+ ],
222
+ // Local/Playwright backend
223
+ "local:transport_error": [
224
+ "The local Playwright-managed browser failed to start or connect.",
225
+ "Check that the Playwright browser binary is downloaded (bun run install).",
226
+ "Try closing any stale Chromium processes and retrying.",
227
+ ],
228
+ };
229
+
230
+ /**
231
+ * Build a human-readable, tool-response-ready error string from a
232
+ * pinned-mode failure. Includes:
233
+ * - the requested mode
234
+ * - ordered attempted modes with exact failure reasons
235
+ * - a remediation checklist tailored by backend and failure code
236
+ *
237
+ * Exported for testing.
238
+ */
239
+ export function formatModeSelectionFailure(
240
+ requestedMode: BrowserMode,
241
+ error: CdpError,
242
+ ): string {
243
+ const lines: string[] = [];
244
+ lines.push(`Error: Browser mode "${requestedMode}" failed.`);
245
+ lines.push("");
246
+
247
+ const diagnostics: readonly AttemptDiagnostic[] =
248
+ error.attemptDiagnostics ?? [];
249
+
250
+ if (diagnostics.length > 0) {
251
+ lines.push("Attempted backends:");
252
+ for (const diag of diagnostics) {
253
+ const status =
254
+ diag.stage === "success" ? "OK" : `FAILED at ${diag.stage}`;
255
+ lines.push(` - ${diag.candidateKind}: ${status}`);
256
+ if (diag.errorMessage) {
257
+ lines.push(` Reason: ${diag.errorMessage}`);
258
+ }
259
+ if (diag.discoveryCode) {
260
+ lines.push(` Discovery code: ${diag.discoveryCode}`);
261
+ }
262
+ }
263
+ lines.push("");
264
+ }
265
+
266
+ // Collect remediation hints
267
+ const hints = collectRemediationHints(diagnostics, error);
268
+ if (hints.length > 0) {
269
+ lines.push("Remediation:");
270
+ for (const hint of hints) {
271
+ lines.push(` - ${hint}`);
272
+ }
273
+ }
274
+
275
+ return lines.join("\n");
276
+ }
277
+
278
+ /**
279
+ * Gather remediation hints based on attempt diagnostics and the error.
280
+ * Walks each diagnostic and looks up hints by (kind, discoveryCode),
281
+ * then (kind, errorCode), then generic kind-level fallback.
282
+ */
283
+ function collectRemediationHints(
284
+ diagnostics: readonly AttemptDiagnostic[],
285
+ error: CdpError,
286
+ ): string[] {
287
+ const seen = new Set<string>();
288
+ const hints: string[] = [];
289
+
290
+ const addHints = (key: string) => {
291
+ const list = REMEDIATION_HINTS[key];
292
+ if (!list) return;
293
+ for (const hint of list) {
294
+ if (!seen.has(hint)) {
295
+ seen.add(hint);
296
+ hints.push(hint);
297
+ }
298
+ }
299
+ };
300
+
301
+ for (const diag of diagnostics) {
302
+ if (diag.stage === "success") continue;
303
+ if (diag.discoveryCode) {
304
+ addHints(`${diag.candidateKind}:${diag.discoveryCode}`);
305
+ }
306
+ if (diag.errorCode) {
307
+ addHints(`${diag.candidateKind}:${diag.errorCode}`);
308
+ }
309
+ }
310
+
311
+ // Fallback: if no diagnostics but we have a top-level error, use
312
+ // the error code with a generic candidate kind derived from the mode.
313
+ if (diagnostics.length === 0 && error.code) {
314
+ // Try to infer the candidate kind from the error message
315
+ for (const kind of BROWSER_STATUS_MODES) {
316
+ if (error.message.toLowerCase().includes(kind)) {
317
+ addHints(`${kind}:${error.code}`);
318
+ }
319
+ }
320
+ }
321
+
322
+ return hints;
323
+ }
324
+
325
+ /**
326
+ * Parse browser_mode from input and acquire a CdpClient. Returns
327
+ * either a `{ cdp, browserMode }` pair on success or a pre-formatted
328
+ * `{ errorResult }` on failure (invalid mode or pinned-mode
329
+ * precondition not met).
330
+ *
331
+ * This is the single integration point for all CDP-backed tool
332
+ * functions. Using it ensures every tool:
333
+ * - normalizes aliases (`cdp-debugger` -> `cdp-inspect`, etc.)
334
+ * - passes the mode preference to the factory
335
+ * - surfaces a remediation-rich error on pinned-mode failures
336
+ *
337
+ * Per-conversation stickiness: when the incoming `browser_mode` is
338
+ * `"auto"` and the conversation has already resolved to a backend
339
+ * kind on a prior call, the factory is pinned to that kind instead
340
+ * of re-running the auto priority list. This prevents
341
+ * `browser_navigate` (e.g. pinned to `local`) and `browser_screenshot`
342
+ * (default auto) in the same conversation from landing on different
343
+ * Chrome instances. Explicit non-auto modes override and update the
344
+ * memo; teardown via browser_close / browser_detach clears it.
345
+ *
346
+ * The returned client is wrapped so its first successful `send()`
347
+ * writes the resolved kind back to the conversation memo.
348
+ */
349
+ export function acquireCdpClientWithMode(
350
+ input: Record<string, unknown>,
351
+ context: ToolContext,
352
+ ):
353
+ | {
354
+ cdp: ReturnType<typeof getCdpClient>;
355
+ browserMode: BrowserMode;
356
+ errorResult?: never;
357
+ }
358
+ | { cdp?: never; browserMode?: never; errorResult: ToolExecutionResult } {
359
+ const modeResult = parseBrowserMode(input);
360
+ if (!modeResult.ok) {
361
+ return {
362
+ errorResult: { content: modeResult.error, isError: true },
363
+ };
364
+ }
365
+ const browserMode = modeResult.mode;
366
+
367
+ const rememberedKind = browserManager.getPreferredBackendKind(
368
+ context.conversationId,
369
+ );
370
+ const effectiveMode: BrowserMode =
371
+ browserMode === "auto" && rememberedKind !== null
372
+ ? rememberedKind
373
+ : browserMode;
374
+
375
+ try {
376
+ const raw = getCdpClient(context, { mode: effectiveMode });
377
+ const cdp = wrapWithKindMemo(raw, context.conversationId);
378
+ return { cdp, browserMode };
379
+ } catch (err) {
380
+ // Sticky-mode fallback: the caller requested "auto" but we pinned to
381
+ // a remembered backend kind that has since become unavailable. Drop
382
+ // the stale memo and retry with fresh auto selection so a dead
383
+ // sticky preference doesn't surface as a hard failure.
384
+ if (browserMode === "auto" && effectiveMode !== "auto") {
385
+ browserManager.clearPreferredBackendKind(context.conversationId);
386
+ try {
387
+ const raw = getCdpClient(context, { mode: "auto" });
388
+ const cdp = wrapWithKindMemo(raw, context.conversationId);
389
+ return { cdp, browserMode };
390
+ } catch (retryErr) {
391
+ if (retryErr instanceof CdpError) {
392
+ return {
393
+ errorResult: {
394
+ content: formatModeSelectionFailure("auto", retryErr),
395
+ isError: true,
396
+ },
397
+ };
398
+ }
399
+ throw retryErr;
400
+ }
401
+ }
402
+ if (err instanceof CdpError && browserMode !== "auto") {
403
+ return {
404
+ errorResult: {
405
+ content: formatModeSelectionFailure(browserMode, err),
406
+ isError: true,
407
+ },
408
+ };
409
+ }
410
+ throw err;
411
+ }
412
+ }
413
+
414
+ /**
415
+ * Wrap a {@link ScopedCdpClient} so the first successful `send()`
416
+ * records the resolved backend kind in the conversation's
417
+ * `preferredBackendKinds` memo. Subsequent sends are no-ops for the
418
+ * memo; dispose() delegates to the underlying client.
419
+ */
420
+ function wrapWithKindMemo(
421
+ inner: ReturnType<typeof getCdpClient>,
422
+ conversationId: string,
423
+ ): ReturnType<typeof getCdpClient> {
424
+ let recorded = false;
425
+ return {
426
+ get kind() {
427
+ return inner.kind;
428
+ },
429
+ conversationId: inner.conversationId,
430
+ async send<T = unknown>(
431
+ method: string,
432
+ params?: Record<string, unknown>,
433
+ signal?: AbortSignal,
434
+ ): Promise<T> {
435
+ const result = await inner.send<T>(method, params, signal);
436
+ if (!recorded) {
437
+ browserManager.setPreferredBackendKind(conversationId, inner.kind);
438
+ recorded = true;
439
+ }
440
+ return result;
441
+ },
442
+ dispose(): void {
443
+ inner.dispose();
444
+ },
445
+ };
446
+ }
447
+
448
+ // ── CDP error diagnostics helper ─────────────────────────────────────
449
+
450
+ /**
451
+ * Check whether a caught error is a {@link CdpError} carrying
452
+ * {@link AttemptDiagnostic attempt diagnostics} from the factory's
453
+ * failover walk. When the browser_mode is pinned (not "auto") and
454
+ * diagnostics are present, format the error with the full remediation
455
+ * checklist via {@link formatModeSelectionFailure}. Otherwise return
456
+ * `null` so the caller falls through to its generic error message.
457
+ *
458
+ * This handles the case where pinned-mode unavailability is surfaced
459
+ * on the first `cdp.send()` (via `sendWithFailover`) rather than
460
+ * during client construction (which `acquireCdpClientWithMode` already
461
+ * covers).
462
+ */
463
+ function formatCdpSendDiagnostics(
464
+ err: unknown,
465
+ browserMode: BrowserMode,
466
+ ): string | null {
467
+ if (
468
+ err instanceof CdpError &&
469
+ browserMode !== "auto" &&
470
+ err.code === "transport_error" &&
471
+ err.attemptDiagnostics
472
+ ) {
473
+ return formatModeSelectionFailure(browserMode, err);
474
+ }
475
+ return null;
476
+ }
477
+
102
478
  // ── Shared element resolution ────────────────────────────────────────
103
479
 
104
480
  /**
@@ -174,6 +550,8 @@ export async function executeBrowserNavigate(
174
550
  return { content: "Error: operation was cancelled", isError: true };
175
551
  }
176
552
 
553
+ // Pre-flight URL validation runs before CDP acquisition so we fail
554
+ // fast on obviously invalid URLs without opening a browser session.
177
555
  const parsedUrl = parseUrl(input.url);
178
556
  if (!parsedUrl) {
179
557
  return {
@@ -212,7 +590,10 @@ export async function executeBrowserNavigate(
212
590
  }
213
591
  }
214
592
 
215
- const cdp = getCdpClient(context);
593
+ // URL validation passed — acquire the CDP client.
594
+ const acquired = acquireCdpClientWithMode(input, context);
595
+ if (acquired.errorResult) return acquired.errorResult;
596
+ const { cdp, browserMode } = acquired;
216
597
 
217
598
  // Screencast + handoff are Playwright-backed and only meaningful
218
599
  // for the local sacrificial-profile path. On the extension path the
@@ -223,10 +604,12 @@ export async function executeBrowserNavigate(
223
604
  await ensureScreencast(context.conversationId);
224
605
  }
225
606
 
226
- // SSRF route interception is a Playwright-specific affordance used on
227
- // the local path to block redirect-time requests to private networks.
228
- // On the extension path we rely on the pre-CDP URL validation above;
229
- // see phase3-cdp-migration.md PR 7 for the rationale.
607
+ // SSRF route interception uses the Playwright page.route() API to
608
+ // block redirect-time requests to private networks. This only works
609
+ // on the local path where Playwright manages the browser; on the
610
+ // extension/cdp-inspect paths, CDP navigates a different browser so
611
+ // the Playwright route handler would be a no-op. The post-navigation
612
+ // final URL check below provides defense-in-depth for all paths.
230
613
  let routeHandler: RouteHandler | null = null;
231
614
  let blockedUrl: string | null = null;
232
615
 
@@ -338,6 +721,53 @@ export async function executeBrowserNavigate(
338
721
  { timeoutMs: NAVIGATE_TIMEOUT_MS },
339
722
  context.signal,
340
723
  );
724
+
725
+ // Defense-in-depth: check the final URL after navigation completes.
726
+ // This catches redirect-based SSRF even when Playwright route
727
+ // interception is unavailable (e.g. extension-backed sessions where
728
+ // the CDP transport is separate from the Playwright page).
729
+ if (!allowPrivateNetwork) {
730
+ const finalParsed = parseUrl(finalUrl);
731
+ if (
732
+ finalParsed &&
733
+ (isPrivateOrLocalHost(finalParsed.hostname) ||
734
+ (
735
+ await resolveRequestAddress(
736
+ finalParsed.hostname,
737
+ resolveHostAddresses,
738
+ false,
739
+ )
740
+ ).blockedAddress)
741
+ ) {
742
+ // Navigate the page away from the private target to prevent
743
+ // follow-up tool calls (e.g. browser_snapshot) from reading
744
+ // the already-loaded private content.
745
+ try {
746
+ await navigateAndWait(
747
+ cdp,
748
+ "about:blank",
749
+ { timeoutMs: 3_000 },
750
+ context.signal,
751
+ );
752
+ } catch {
753
+ // Best-effort — if the reset fails, the CDP session will be
754
+ // disposed in the finally block anyway.
755
+ }
756
+ // Clean up the route handler before returning to avoid leaking
757
+ // a stale interception handler on the session page.
758
+ if (routeHandler) {
759
+ const page = await browserManager.getOrCreateSessionPage(
760
+ context.conversationId,
761
+ );
762
+ await page.unroute("**/*", routeHandler);
763
+ routeHandler = null;
764
+ }
765
+ return {
766
+ content: `Error: Navigation blocked. Final URL resolved to a local/private network target (${sanitizeUrlForOutput(finalParsed)}). Set allow_private_network=true if you explicitly need it.`,
767
+ isError: true,
768
+ };
769
+ }
770
+ }
341
771
  if (navigationTimedOut) {
342
772
  // If the page URL never changed from before navigation, the page
343
773
  // never actually loaded - re-throw instead of reporting success.
@@ -353,7 +783,7 @@ export async function executeBrowserNavigate(
353
783
  }
354
784
 
355
785
  // Remove the Playwright route handler now that navigation is
356
- // complete (local path only).
786
+ // complete (local path only — route interception is gated above).
357
787
  if (routeHandler) {
358
788
  const page = await browserManager.getOrCreateSessionPage(
359
789
  context.conversationId,
@@ -563,6 +993,11 @@ export async function executeBrowserNavigate(
563
993
  };
564
994
  }
565
995
 
996
+ const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
997
+ if (diagnosticMessage) {
998
+ return { content: diagnosticMessage, isError: true };
999
+ }
1000
+
566
1001
  const msg = err instanceof Error ? err.message : String(err);
567
1002
  log.error({ err, url: safeRequestedUrl }, "Navigation failed");
568
1003
  return { content: `Error: Navigation failed: ${msg}`, isError: true };
@@ -577,7 +1012,10 @@ export async function executeBrowserSnapshot(
577
1012
  _input: Record<string, unknown>,
578
1013
  context: ToolContext,
579
1014
  ): Promise<ToolExecutionResult> {
580
- const cdp = getCdpClient(context);
1015
+ const acquired = acquireCdpClientWithMode(_input, context);
1016
+ if (acquired.errorResult) return acquired.errorResult;
1017
+ const { cdp, browserMode } = acquired;
1018
+
581
1019
  try {
582
1020
  const currentUrl = await getCurrentUrl(cdp, context.signal);
583
1021
  const title = await getPageTitle(cdp, context.signal);
@@ -608,6 +1046,10 @@ export async function executeBrowserSnapshot(
608
1046
  isError: false,
609
1047
  };
610
1048
  } catch (err) {
1049
+ const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
1050
+ if (diagnosticMessage) {
1051
+ return { content: diagnosticMessage, isError: true };
1052
+ }
611
1053
  const msg = err instanceof Error ? err.message : String(err);
612
1054
  log.error({ err }, "Snapshot failed");
613
1055
  return { content: `Error: Snapshot failed: ${msg}`, isError: true };
@@ -622,9 +1064,11 @@ export async function executeBrowserScreenshot(
622
1064
  input: Record<string, unknown>,
623
1065
  context: ToolContext,
624
1066
  ): Promise<ToolExecutionResult> {
1067
+ const acquired = acquireCdpClientWithMode(input, context);
1068
+ if (acquired.errorResult) return acquired.errorResult;
1069
+ const { cdp, browserMode } = acquired;
625
1070
  const fullPage = input.full_page === true;
626
1071
 
627
- const cdp = getCdpClient(context);
628
1072
  try {
629
1073
  const buffer = await captureScreenshotJpeg(
630
1074
  cdp,
@@ -650,6 +1094,10 @@ export async function executeBrowserScreenshot(
650
1094
  contentBlocks: [imageBlock],
651
1095
  };
652
1096
  } catch (err) {
1097
+ const diagnosticMessage = formatCdpSendDiagnostics(err, browserMode);
1098
+ if (diagnosticMessage) {
1099
+ return { content: diagnosticMessage, isError: true };
1100
+ }
653
1101
  const msg = err instanceof Error ? err.message : String(err);
654
1102
  log.error({ err }, "Screenshot failed");
655
1103
  return { content: `Error: Screenshot failed: ${msg}`, isError: true };
@@ -658,13 +1106,118 @@ export async function executeBrowserScreenshot(
658
1106
  }
659
1107
  }
660
1108
 
1109
+ // ── browser_attach ───────────────────────────────────────────────────
1110
+
1111
+ export async function executeBrowserAttach(
1112
+ _input: Record<string, unknown>,
1113
+ context: ToolContext,
1114
+ ): Promise<ToolExecutionResult> {
1115
+ const acquired = acquireCdpClientWithMode(_input, context);
1116
+ if (acquired.errorResult) return acquired.errorResult;
1117
+ const cdp = acquired.cdp;
1118
+ try {
1119
+ if (cdp.kind === "extension") {
1120
+ // Extension path: explicitly attach the debugger via a synthetic
1121
+ // Vellum.attach command so the debugging session is established
1122
+ // before any navigation or interaction.
1123
+ const result = await cdp.send<{ attached?: boolean; target?: unknown }>(
1124
+ "Vellum.attach",
1125
+ {},
1126
+ context.signal,
1127
+ );
1128
+ log.debug(
1129
+ { conversationId: context.conversationId, result },
1130
+ "Browser debugger attached (extension)",
1131
+ );
1132
+ return {
1133
+ content: "Browser debugger attached.",
1134
+ isError: false,
1135
+ };
1136
+ }
1137
+
1138
+ // Non-extension backends (local / cdp-inspect): explicit attach is
1139
+ // not required — the backend manages its own connection lifecycle.
1140
+ // Return a deterministic no-op success.
1141
+ return {
1142
+ content:
1143
+ "Browser session ready. (Explicit attach is not required on this backend.)",
1144
+ isError: false,
1145
+ };
1146
+ } catch (err) {
1147
+ const diagnosticMessage = formatCdpSendDiagnostics(
1148
+ err,
1149
+ acquired.browserMode,
1150
+ );
1151
+ if (diagnosticMessage) {
1152
+ return { content: diagnosticMessage, isError: true };
1153
+ }
1154
+ const msg = err instanceof Error ? err.message : String(err);
1155
+ log.error({ err }, "Attach failed");
1156
+ return { content: `Error: Attach failed: ${msg}`, isError: true };
1157
+ } finally {
1158
+ cdp.dispose();
1159
+ }
1160
+ }
1161
+
1162
+ // ── browser_detach ──────────────────────────────────────────────────
1163
+
1164
+ export async function executeBrowserDetach(
1165
+ _input: Record<string, unknown>,
1166
+ context: ToolContext,
1167
+ ): Promise<ToolExecutionResult> {
1168
+ const acquired = acquireCdpClientWithMode(_input, context);
1169
+ if (acquired.errorResult) return acquired.errorResult;
1170
+ const cdp = acquired.cdp;
1171
+ try {
1172
+ if (cdp.kind === "extension") {
1173
+ // Extension path: explicitly detach the debugger via a synthetic
1174
+ // Vellum.detach command so the Chrome debugging banner clears.
1175
+ const result = await cdp.send<{ detached?: boolean; target?: unknown }>(
1176
+ "Vellum.detach",
1177
+ {},
1178
+ context.signal,
1179
+ );
1180
+ log.debug(
1181
+ { conversationId: context.conversationId, result },
1182
+ "Browser debugger detached (extension)",
1183
+ );
1184
+ }
1185
+
1186
+ return {
1187
+ content: "Browser debugger detached and snapshot state cleared.",
1188
+ isError: false,
1189
+ };
1190
+ } catch (err) {
1191
+ const diagnosticMessage = formatCdpSendDiagnostics(
1192
+ err,
1193
+ acquired.browserMode,
1194
+ );
1195
+ if (diagnosticMessage) {
1196
+ return { content: diagnosticMessage, isError: true };
1197
+ }
1198
+ const msg = err instanceof Error ? err.message : String(err);
1199
+ log.error({ err }, "Detach failed");
1200
+ return { content: `Error: Detach failed: ${msg}`, isError: true };
1201
+ } finally {
1202
+ // Always reset conversation-scoped browser state, even if the
1203
+ // Vellum.detach round-trip failed (target gone, transport dropped).
1204
+ // browser_detach is the user's recovery path — leaving a stale
1205
+ // sticky backend or snapshot map behind would defeat its purpose.
1206
+ browserManager.clearSnapshotBackendNodeMap(context.conversationId);
1207
+ browserManager.clearPreferredBackendKind(context.conversationId);
1208
+ cdp.dispose();
1209
+ }
1210
+ }
1211
+
661
1212
  // ── browser_close ────────────────────────────────────────────────────
662
1213
 
663
1214
  export async function executeBrowserClose(
664
1215
  input: Record<string, unknown>,
665
1216
  context: ToolContext,
666
1217
  ): Promise<ToolExecutionResult> {
667
- const cdp = getCdpClient(context);
1218
+ const acquired = acquireCdpClientWithMode(input, context);
1219
+ if (acquired.errorResult) return acquired.errorResult;
1220
+ const cdp = acquired.cdp;
668
1221
  try {
669
1222
  if (cdp.kind === "local") {
670
1223
  // Local/sacrificial-profile path: tear down the Playwright page,
@@ -690,15 +1243,29 @@ export async function executeBrowserClose(
690
1243
  }
691
1244
 
692
1245
  // Extension path: the user owns their Chrome tab — we must not
693
- // close it. Only drop the cached snapshot state so stale eids
694
- // from prior snapshots cannot be resolved by later tool calls.
1246
+ // close it. Detach the debugger (so the Chrome debugging banner
1247
+ // clears promptly) and drop the cached snapshot state so stale
1248
+ // eids from prior snapshots cannot be resolved by later tool calls.
1249
+ try {
1250
+ await cdp.send("Vellum.detach", {}, context.signal);
1251
+ } catch {
1252
+ // Tolerate detach failures (already detached, tab closed, etc.)
1253
+ }
695
1254
  browserManager.clearSnapshotBackendNodeMap(context.conversationId);
1255
+ browserManager.clearPreferredBackendKind(context.conversationId);
696
1256
  return {
697
1257
  content:
698
1258
  "Browser session cleared. (Your Chrome tab was not closed — close it yourself if desired.)",
699
1259
  isError: false,
700
1260
  };
701
1261
  } catch (err) {
1262
+ const diagnosticMessage = formatCdpSendDiagnostics(
1263
+ err,
1264
+ acquired.browserMode,
1265
+ );
1266
+ if (diagnosticMessage) {
1267
+ return { content: diagnosticMessage, isError: true };
1268
+ }
702
1269
  const msg = err instanceof Error ? err.message : String(err);
703
1270
  log.error({ err }, "Close failed");
704
1271
  return { content: `Error: Close failed: ${msg}`, isError: true };
@@ -716,7 +1283,9 @@ export async function executeBrowserClick(
716
1283
  const { resolved, error } = resolveElement(context.conversationId, input);
717
1284
  if (error) return { content: error, isError: true };
718
1285
 
719
- const cdp = getCdpClient(context);
1286
+ const acquired = acquireCdpClientWithMode(input, context);
1287
+ if (acquired.errorResult) return acquired.errorResult;
1288
+ const cdp = acquired.cdp;
720
1289
  try {
721
1290
  let backendNodeId: number;
722
1291
  if (resolved!.kind === "backend") {
@@ -744,6 +1313,13 @@ export async function executeBrowserClick(
744
1313
  : resolved!.selector;
745
1314
  return { content: `Clicked element: ${desc}`, isError: false };
746
1315
  } catch (err) {
1316
+ const diagnosticMessage = formatCdpSendDiagnostics(
1317
+ err,
1318
+ acquired.browserMode,
1319
+ );
1320
+ if (diagnosticMessage) {
1321
+ return { content: diagnosticMessage, isError: true };
1322
+ }
747
1323
  const msg = err instanceof Error ? err.message : String(err);
748
1324
  log.error({ err }, "Click failed");
749
1325
  return { content: `Error: Click failed: ${msg}`, isError: true };
@@ -828,7 +1404,9 @@ export async function executeBrowserType(
828
1404
  ? `element_id "${resolved!.eid}"`
829
1405
  : resolved!.selector;
830
1406
 
831
- const cdp = getCdpClient(context);
1407
+ const acquired = acquireCdpClientWithMode(input, context);
1408
+ if (acquired.errorResult) return acquired.errorResult;
1409
+ const cdp = acquired.cdp;
832
1410
  try {
833
1411
  let backendNodeId: number;
834
1412
  if (resolved!.kind === "backend") {
@@ -857,6 +1435,13 @@ export async function executeBrowserType(
857
1435
  if (pressEnter) lines.push("(pressed Enter after typing)");
858
1436
  return { content: lines.join("\n"), isError: false };
859
1437
  } catch (err) {
1438
+ const diagnosticMessage = formatCdpSendDiagnostics(
1439
+ err,
1440
+ acquired.browserMode,
1441
+ );
1442
+ if (diagnosticMessage) {
1443
+ return { content: diagnosticMessage, isError: true };
1444
+ }
860
1445
  const msg = err instanceof Error ? err.message : String(err);
861
1446
  log.error({ err, target: targetDescription }, "Type failed");
862
1447
  return { content: `Error: Type failed: ${msg}`, isError: true };
@@ -896,7 +1481,9 @@ export async function executeBrowserPressKey(
896
1481
  : resolved!.selector;
897
1482
  }
898
1483
 
899
- const cdp = getCdpClient(context);
1484
+ const acquired = acquireCdpClientWithMode(input, context);
1485
+ if (acquired.errorResult) return acquired.errorResult;
1486
+ const cdp = acquired.cdp;
900
1487
  try {
901
1488
  if (resolved) {
902
1489
  let backendNodeId: number;
@@ -921,6 +1508,13 @@ export async function executeBrowserPressKey(
921
1508
  await dispatchKeyPress(cdp, key, context.signal);
922
1509
  return { content: `Pressed "${key}"`, isError: false };
923
1510
  } catch (err) {
1511
+ const diagnosticMessage = formatCdpSendDiagnostics(
1512
+ err,
1513
+ acquired.browserMode,
1514
+ );
1515
+ if (diagnosticMessage) {
1516
+ return { content: diagnosticMessage, isError: true };
1517
+ }
924
1518
  const msg = err instanceof Error ? err.message : String(err);
925
1519
  log.error({ err, key }, "Press key failed");
926
1520
  return { content: `Error: Press key failed: ${msg}`, isError: true };
@@ -964,7 +1558,9 @@ export async function executeBrowserScroll(
964
1558
  break;
965
1559
  }
966
1560
 
967
- const cdp = getCdpClient(context);
1561
+ const acquired = acquireCdpClientWithMode(input, context);
1562
+ if (acquired.errorResult) return acquired.errorResult;
1563
+ const cdp = acquired.cdp;
968
1564
  try {
969
1565
  // Fetch viewport dimensions so we can dispatch the wheel event at
970
1566
  // the viewport center — scrolling from (0, 0) misses sticky
@@ -985,6 +1581,13 @@ export async function executeBrowserScroll(
985
1581
 
986
1582
  return { content: `Scrolled ${direction} by ${amount}px`, isError: false };
987
1583
  } catch (err) {
1584
+ const diagnosticMessage = formatCdpSendDiagnostics(
1585
+ err,
1586
+ acquired.browserMode,
1587
+ );
1588
+ if (diagnosticMessage) {
1589
+ return { content: diagnosticMessage, isError: true };
1590
+ }
988
1591
  const msg = err instanceof Error ? err.message : String(err);
989
1592
  log.error({ err, direction }, "Scroll failed");
990
1593
  return { content: `Error: Scroll failed: ${msg}`, isError: true };
@@ -1018,7 +1621,9 @@ export async function executeBrowserSelectOption(
1018
1621
  ? `element_id "${resolved!.eid}"`
1019
1622
  : resolved!.selector;
1020
1623
 
1021
- const cdp = getCdpClient(context);
1624
+ const acquired = acquireCdpClientWithMode(input, context);
1625
+ if (acquired.errorResult) return acquired.errorResult;
1626
+ const cdp = acquired.cdp;
1022
1627
  try {
1023
1628
  let backendNodeId: number;
1024
1629
  if (resolved!.kind === "backend") {
@@ -1108,6 +1713,13 @@ export async function executeBrowserSelectOption(
1108
1713
  isError: false,
1109
1714
  };
1110
1715
  } catch (err) {
1716
+ const diagnosticMessage = formatCdpSendDiagnostics(
1717
+ err,
1718
+ acquired.browserMode,
1719
+ );
1720
+ if (diagnosticMessage) {
1721
+ return { content: diagnosticMessage, isError: true };
1722
+ }
1111
1723
  const msg = err instanceof Error ? err.message : String(err);
1112
1724
  log.error({ err, target: targetDescription }, "Select option failed");
1113
1725
  return { content: `Error: Select option failed: ${msg}`, isError: true };
@@ -1125,7 +1737,9 @@ export async function executeBrowserHover(
1125
1737
  const { resolved, error } = resolveElement(context.conversationId, input);
1126
1738
  if (error) return { content: error, isError: true };
1127
1739
 
1128
- const cdp = getCdpClient(context);
1740
+ const acquired = acquireCdpClientWithMode(input, context);
1741
+ if (acquired.errorResult) return acquired.errorResult;
1742
+ const cdp = acquired.cdp;
1129
1743
  try {
1130
1744
  let backendNodeId: number;
1131
1745
  if (resolved!.kind === "backend") {
@@ -1150,6 +1764,13 @@ export async function executeBrowserHover(
1150
1764
  : resolved!.selector;
1151
1765
  return { content: `Hovered element: ${desc}`, isError: false };
1152
1766
  } catch (err) {
1767
+ const diagnosticMessage = formatCdpSendDiagnostics(
1768
+ err,
1769
+ acquired.browserMode,
1770
+ );
1771
+ if (diagnosticMessage) {
1772
+ return { content: diagnosticMessage, isError: true };
1773
+ }
1153
1774
  const msg = err instanceof Error ? err.message : String(err);
1154
1775
  log.error({ err }, "Hover failed");
1155
1776
  return { content: `Error: Hover failed: ${msg}`, isError: true };
@@ -1195,6 +1816,13 @@ export async function executeBrowserWaitFor(
1195
1816
  ? Math.min(input.timeout, MAX_WAIT_MS)
1196
1817
  : MAX_WAIT_MS;
1197
1818
 
1819
+ // Validate browser_mode even on the duration path so invalid values
1820
+ // are rejected consistently regardless of which wait mode is used.
1821
+ const modeResult = parseBrowserMode(input);
1822
+ if (!modeResult.ok) {
1823
+ return { content: modeResult.error, isError: true };
1824
+ }
1825
+
1198
1826
  // Duration mode has no CDP interaction — handle without acquiring
1199
1827
  // a CdpClient so the common "sleep" path stays transport-agnostic.
1200
1828
  if (duration != null) {
@@ -1203,7 +1831,9 @@ export async function executeBrowserWaitFor(
1203
1831
  return { content: `Waited ${waitMs}ms.`, isError: false };
1204
1832
  }
1205
1833
 
1206
- const cdp = getCdpClient(context);
1834
+ const acquired = acquireCdpClientWithMode(input, context);
1835
+ if (acquired.errorResult) return acquired.errorResult;
1836
+ const cdp = acquired.cdp;
1207
1837
  try {
1208
1838
  if (selector) {
1209
1839
  // browser_wait_for selector mode is "did this node appear at
@@ -1227,6 +1857,13 @@ export async function executeBrowserWaitFor(
1227
1857
  isError: false,
1228
1858
  };
1229
1859
  } catch (err) {
1860
+ const diagnosticMessage = formatCdpSendDiagnostics(
1861
+ err,
1862
+ acquired.browserMode,
1863
+ );
1864
+ if (diagnosticMessage) {
1865
+ return { content: diagnosticMessage, isError: true };
1866
+ }
1230
1867
  const msg = err instanceof Error ? err.message : String(err);
1231
1868
  log.error({ err }, "Wait failed");
1232
1869
  return { content: `Error: Wait failed: ${msg}`, isError: true };
@@ -1243,7 +1880,9 @@ export async function executeBrowserExtract(
1243
1880
  ): Promise<ToolExecutionResult> {
1244
1881
  const includeLinks = input.include_links === true;
1245
1882
 
1246
- const cdp = getCdpClient(context);
1883
+ const acquired = acquireCdpClientWithMode(input, context);
1884
+ if (acquired.errorResult) return acquired.errorResult;
1885
+ const cdp = acquired.cdp;
1247
1886
  try {
1248
1887
  const currentUrl = await getCurrentUrl(cdp, context.signal);
1249
1888
  const title = await getPageTitle(cdp, context.signal);
@@ -1257,7 +1896,8 @@ export async function executeBrowserExtract(
1257
1896
 
1258
1897
  if (textContent.length > MAX_EXTRACT_LENGTH) {
1259
1898
  textContent =
1260
- textContent.slice(0, MAX_EXTRACT_LENGTH) + "\n... (truncated)";
1899
+ safeStringSlice(textContent, 0, MAX_EXTRACT_LENGTH) +
1900
+ "\n... (truncated)";
1261
1901
  }
1262
1902
 
1263
1903
  const lines: string[] = [
@@ -1283,6 +1923,13 @@ export async function executeBrowserExtract(
1283
1923
 
1284
1924
  return { content: lines.join("\n"), isError: false };
1285
1925
  } catch (err) {
1926
+ const diagnosticMessage = formatCdpSendDiagnostics(
1927
+ err,
1928
+ acquired.browserMode,
1929
+ );
1930
+ if (diagnosticMessage) {
1931
+ return { content: diagnosticMessage, isError: true };
1932
+ }
1286
1933
  const msg = err instanceof Error ? err.message : String(err);
1287
1934
  log.error({ err }, "Extract failed");
1288
1935
  return { content: `Error: Extract failed: ${msg}`, isError: true };
@@ -1316,7 +1963,9 @@ export async function executeBrowserFillCredential(
1316
1963
  ? `element_id "${resolved!.eid}"`
1317
1964
  : resolved!.selector;
1318
1965
 
1319
- const cdp = getCdpClient(context);
1966
+ const acquired = acquireCdpClientWithMode(input, context);
1967
+ if (acquired.errorResult) return acquired.errorResult;
1968
+ const cdp = acquired.cdp;
1320
1969
  try {
1321
1970
  let backendNodeId: number;
1322
1971
  if (resolved!.kind === "backend") {
@@ -1405,6 +2054,13 @@ export async function executeBrowserFillCredential(
1405
2054
  isError: false,
1406
2055
  };
1407
2056
  } catch (err) {
2057
+ const diagnosticMessage = formatCdpSendDiagnostics(
2058
+ err,
2059
+ acquired.browserMode,
2060
+ );
2061
+ if (diagnosticMessage) {
2062
+ return { content: diagnosticMessage, isError: true };
2063
+ }
1408
2064
  const msg = err instanceof Error ? err.message : String(err);
1409
2065
  log.error({ err }, "Fill credential failed");
1410
2066
  return { content: `Error: Fill credential failed: ${msg}`, isError: true };
@@ -1412,3 +2068,487 @@ export async function executeBrowserFillCredential(
1412
2068
  cdp.dispose();
1413
2069
  }
1414
2070
  }
2071
+
2072
+ function dedupeStrings(values: string[]): string[] {
2073
+ const seen = new Set<string>();
2074
+ const out: string[] = [];
2075
+ for (const value of values) {
2076
+ if (!value) continue;
2077
+ if (seen.has(value)) continue;
2078
+ seen.add(value);
2079
+ out.push(value);
2080
+ }
2081
+ return out;
2082
+ }
2083
+
2084
+ function modeTradeoffs(mode: StatusCheckMode): string[] {
2085
+ return MODE_TRADEOFFS[mode];
2086
+ }
2087
+
2088
+ function extensionSetupActions(): string[] {
2089
+ return [
2090
+ "Install and enable the Vellum Relay Chrome extension.",
2091
+ "Open the extension popup and click Pair with local assistant.",
2092
+ "Keep the extension connected to the assistant relay.",
2093
+ ];
2094
+ }
2095
+
2096
+ function cdpInspectSetupActions(): string[] {
2097
+ return [
2098
+ "Launch Chrome with --remote-debugging-port=9222 (or your configured port).",
2099
+ "Keep Chrome running while browser tools are in use.",
2100
+ "Ensure the configured host is loopback (localhost / 127.0.0.1 / ::1).",
2101
+ ];
2102
+ }
2103
+
2104
+ function localSetupActions(): string[] {
2105
+ return [
2106
+ "Install assistant dependencies with bun install in assistant/.",
2107
+ "Install Chromium for Playwright: bunx playwright install chromium.",
2108
+ ];
2109
+ }
2110
+
2111
+ function extractDiscoveryCodes(error: CdpError): string[] {
2112
+ const diagnostics = error.attemptDiagnostics ?? [];
2113
+ const codes: string[] = [];
2114
+ for (const diag of diagnostics) {
2115
+ if (diag.discoveryCode) codes.push(diag.discoveryCode);
2116
+ }
2117
+ return dedupeStrings(codes);
2118
+ }
2119
+
2120
+ function containsTokenCaseInsensitive(text: string, token: string): boolean {
2121
+ return text.toLowerCase().includes(token.toLowerCase());
2122
+ }
2123
+
2124
+ function probeFailureActions(mode: StatusCheckMode, error: CdpError): string[] {
2125
+ const actions: string[] = [];
2126
+ const message = error.message.toLowerCase();
2127
+ const discoveryCodes = extractDiscoveryCodes(error).map((c) =>
2128
+ c.toLowerCase(),
2129
+ );
2130
+
2131
+ if (mode === BROWSER_STATUS_MODE.EXTENSION) {
2132
+ actions.push(...extensionSetupActions());
2133
+ if (
2134
+ containsTokenCaseInsensitive(
2135
+ message,
2136
+ EXTENSION_STATUS_ERROR_MARKER.UNAUTHORIZED_ORIGIN,
2137
+ )
2138
+ ) {
2139
+ actions.push(
2140
+ "Ensure this extension ID is present in meta/browser-extension/chrome-extension-allowlist.json and restart the assistant.",
2141
+ );
2142
+ }
2143
+ if (
2144
+ containsTokenCaseInsensitive(
2145
+ message,
2146
+ EXTENSION_STATUS_ERROR_MARKER.NATIVE_MESSAGING_HOST,
2147
+ )
2148
+ ) {
2149
+ actions.push(
2150
+ "Reinstall the native messaging host manifest and confirm it allows this extension ID.",
2151
+ );
2152
+ }
2153
+ if (
2154
+ containsTokenCaseInsensitive(
2155
+ message,
2156
+ EXTENSION_STATUS_ERROR_MARKER.HTTP_401,
2157
+ )
2158
+ ) {
2159
+ actions.push(
2160
+ "Re-pair the extension so it refreshes its local relay credential.",
2161
+ );
2162
+ }
2163
+ }
2164
+
2165
+ if (mode === BROWSER_STATUS_MODE.CDP_INSPECT) {
2166
+ actions.push(...cdpInspectSetupActions());
2167
+ if (discoveryCodes.includes(CDP_INSPECT_STATUS_DISCOVERY_CODE.NO_TARGETS)) {
2168
+ actions.push("Open at least one normal web page tab and retry.");
2169
+ }
2170
+ if (
2171
+ discoveryCodes.includes(
2172
+ CDP_INSPECT_STATUS_DISCOVERY_CODE.INVALID_RESPONSE,
2173
+ ) ||
2174
+ discoveryCodes.includes(
2175
+ CDP_INSPECT_STATUS_DISCOVERY_CODE.WS_FALLBACK_FAILED,
2176
+ )
2177
+ ) {
2178
+ actions.push(
2179
+ "Verify nothing else is bound to the configured CDP port and exposing non-DevTools responses.",
2180
+ );
2181
+ }
2182
+ }
2183
+
2184
+ if (mode === BROWSER_STATUS_MODE.LOCAL) {
2185
+ actions.push(...localSetupActions());
2186
+ }
2187
+
2188
+ return dedupeStrings(actions);
2189
+ }
2190
+
2191
+ async function probePinnedBrowserMode(
2192
+ mode: StatusCheckMode,
2193
+ context: ToolContext,
2194
+ ): Promise<
2195
+ | {
2196
+ ok: true;
2197
+ backendKind: CdpClientKind;
2198
+ }
2199
+ | {
2200
+ ok: false;
2201
+ error: CdpError;
2202
+ diagnostic: string;
2203
+ }
2204
+ > {
2205
+ let cdp: ReturnType<typeof getCdpClient> | null = null;
2206
+ try {
2207
+ cdp = getCdpClient(context, { mode });
2208
+ await cdp.send(
2209
+ "Runtime.evaluate",
2210
+ {
2211
+ expression: "document.readyState",
2212
+ returnByValue: true,
2213
+ },
2214
+ context.signal,
2215
+ );
2216
+ return { ok: true, backendKind: cdp.kind };
2217
+ } catch (err) {
2218
+ if (err instanceof CdpError) {
2219
+ return {
2220
+ ok: false,
2221
+ error: err,
2222
+ diagnostic: formatModeSelectionFailure(mode, err),
2223
+ };
2224
+ }
2225
+ const wrapped = new CdpError(
2226
+ "transport_error",
2227
+ err instanceof Error ? err.message : String(err),
2228
+ { underlying: err },
2229
+ );
2230
+ return {
2231
+ ok: false,
2232
+ error: wrapped,
2233
+ diagnostic: formatModeSelectionFailure(mode, wrapped),
2234
+ };
2235
+ } finally {
2236
+ cdp?.dispose();
2237
+ }
2238
+ }
2239
+
2240
+ async function checkExtensionModeStatus(
2241
+ context: ToolContext,
2242
+ autoCandidate: boolean,
2243
+ ): Promise<BrowserStatusModeResult> {
2244
+ const proxyBound = Boolean(context.hostBrowserProxy);
2245
+ const proxyConnected = context.hostBrowserProxy?.isAvailable() ?? false;
2246
+
2247
+ if (!proxyBound) {
2248
+ return {
2249
+ mode: BROWSER_STATUS_MODE.EXTENSION,
2250
+ available: false,
2251
+ verified: "preflight",
2252
+ autoCandidate,
2253
+ summary:
2254
+ "Extension mode is unavailable: no host browser proxy is bound to this conversation.",
2255
+ userActions: extensionSetupActions(),
2256
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
2257
+ details: {
2258
+ proxyBound,
2259
+ proxyConnected,
2260
+ },
2261
+ };
2262
+ }
2263
+
2264
+ if (!proxyConnected) {
2265
+ return {
2266
+ mode: BROWSER_STATUS_MODE.EXTENSION,
2267
+ available: false,
2268
+ verified: "preflight",
2269
+ autoCandidate,
2270
+ summary:
2271
+ "Extension mode is unavailable: the extension transport is currently disconnected.",
2272
+ userActions: extensionSetupActions(),
2273
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
2274
+ details: {
2275
+ proxyBound,
2276
+ proxyConnected,
2277
+ },
2278
+ };
2279
+ }
2280
+
2281
+ const probe = await probePinnedBrowserMode(
2282
+ BROWSER_STATUS_MODE.EXTENSION,
2283
+ context,
2284
+ );
2285
+ if (probe.ok) {
2286
+ return {
2287
+ mode: BROWSER_STATUS_MODE.EXTENSION,
2288
+ available: true,
2289
+ verified: "active_probe",
2290
+ autoCandidate,
2291
+ summary: "Extension mode is ready and responded to an active CDP probe.",
2292
+ userActions: [],
2293
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
2294
+ details: {
2295
+ proxyBound,
2296
+ proxyConnected,
2297
+ backendKind: probe.backendKind,
2298
+ },
2299
+ };
2300
+ }
2301
+
2302
+ return {
2303
+ mode: BROWSER_STATUS_MODE.EXTENSION,
2304
+ available: false,
2305
+ verified: "active_probe",
2306
+ autoCandidate,
2307
+ summary: `Extension mode probe failed: ${probe.error.message}`,
2308
+ userActions: probeFailureActions(
2309
+ BROWSER_STATUS_MODE.EXTENSION,
2310
+ probe.error,
2311
+ ),
2312
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
2313
+ details: {
2314
+ proxyBound,
2315
+ proxyConnected,
2316
+ errorCode: probe.error.code,
2317
+ diagnostic: probe.diagnostic,
2318
+ attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
2319
+ },
2320
+ };
2321
+ }
2322
+
2323
+ async function checkCdpInspectModeStatus(
2324
+ context: ToolContext,
2325
+ autoCandidate: boolean,
2326
+ ): Promise<BrowserStatusModeResult> {
2327
+ const cdpInspectConfig = getConfig().hostBrowser.cdpInspect;
2328
+ const desktopAutoEnabled =
2329
+ context.transportInterface === "macos" &&
2330
+ cdpInspectConfig.desktopAuto.enabled;
2331
+ const cooldownActive =
2332
+ desktopAutoEnabled &&
2333
+ isDesktopAutoCooldownActive(cdpInspectConfig.desktopAuto.cooldownMs);
2334
+
2335
+ const probe = await probePinnedBrowserMode(
2336
+ BROWSER_STATUS_MODE.CDP_INSPECT,
2337
+ context,
2338
+ );
2339
+ if (probe.ok) {
2340
+ return {
2341
+ mode: BROWSER_STATUS_MODE.CDP_INSPECT,
2342
+ available: true,
2343
+ verified: "active_probe",
2344
+ autoCandidate,
2345
+ summary:
2346
+ "CDP inspect mode is ready and responded to an active CDP probe.",
2347
+ userActions: [],
2348
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.CDP_INSPECT),
2349
+ details: {
2350
+ backendKind: probe.backendKind,
2351
+ configEnabled: cdpInspectConfig.enabled,
2352
+ configHost: cdpInspectConfig.host,
2353
+ configPort: cdpInspectConfig.port,
2354
+ desktopAutoEnabled,
2355
+ desktopAutoCooldownActive: cooldownActive,
2356
+ },
2357
+ };
2358
+ }
2359
+
2360
+ return {
2361
+ mode: BROWSER_STATUS_MODE.CDP_INSPECT,
2362
+ available: false,
2363
+ verified: "active_probe",
2364
+ autoCandidate,
2365
+ summary: `CDP inspect probe failed: ${probe.error.message}`,
2366
+ userActions: probeFailureActions(
2367
+ BROWSER_STATUS_MODE.CDP_INSPECT,
2368
+ probe.error,
2369
+ ),
2370
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.CDP_INSPECT),
2371
+ details: {
2372
+ errorCode: probe.error.code,
2373
+ discoveryCodes: extractDiscoveryCodes(probe.error),
2374
+ diagnostic: probe.diagnostic,
2375
+ attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
2376
+ configEnabled: cdpInspectConfig.enabled,
2377
+ configHost: cdpInspectConfig.host,
2378
+ configPort: cdpInspectConfig.port,
2379
+ desktopAutoEnabled,
2380
+ desktopAutoCooldownActive: cooldownActive,
2381
+ },
2382
+ };
2383
+ }
2384
+
2385
+ async function checkLocalModeStatus(
2386
+ context: ToolContext,
2387
+ autoCandidate: boolean,
2388
+ checkLocalLaunch: boolean,
2389
+ ): Promise<BrowserStatusModeResult> {
2390
+ const runtime = await checkBrowserRuntime();
2391
+ if (!runtime.playwrightAvailable || !runtime.chromiumInstalled) {
2392
+ return {
2393
+ mode: BROWSER_STATUS_MODE.LOCAL,
2394
+ available: false,
2395
+ verified: "preflight",
2396
+ autoCandidate,
2397
+ summary:
2398
+ runtime.error ??
2399
+ "Local mode preflight failed: Playwright Chromium runtime is not ready.",
2400
+ userActions: localSetupActions(),
2401
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
2402
+ details: {
2403
+ runtime,
2404
+ launchProbeRequested: checkLocalLaunch,
2405
+ },
2406
+ };
2407
+ }
2408
+
2409
+ if (!checkLocalLaunch) {
2410
+ return {
2411
+ mode: BROWSER_STATUS_MODE.LOCAL,
2412
+ available: true,
2413
+ verified: "preflight",
2414
+ autoCandidate,
2415
+ summary:
2416
+ "Local mode preflight passed (Playwright + Chromium are present). Launch probe was skipped.",
2417
+ userActions: [],
2418
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
2419
+ details: {
2420
+ runtime,
2421
+ launchProbeRequested: checkLocalLaunch,
2422
+ },
2423
+ };
2424
+ }
2425
+
2426
+ const probe = await probePinnedBrowserMode(
2427
+ BROWSER_STATUS_MODE.LOCAL,
2428
+ context,
2429
+ );
2430
+ if (probe.ok) {
2431
+ return {
2432
+ mode: BROWSER_STATUS_MODE.LOCAL,
2433
+ available: true,
2434
+ verified: "active_probe",
2435
+ autoCandidate,
2436
+ summary: "Local mode is ready and responded to an active CDP probe.",
2437
+ userActions: [],
2438
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
2439
+ details: {
2440
+ runtime,
2441
+ launchProbeRequested: checkLocalLaunch,
2442
+ backendKind: probe.backendKind,
2443
+ },
2444
+ };
2445
+ }
2446
+
2447
+ return {
2448
+ mode: BROWSER_STATUS_MODE.LOCAL,
2449
+ available: false,
2450
+ verified: "active_probe",
2451
+ autoCandidate,
2452
+ summary: `Local mode probe failed: ${probe.error.message}`,
2453
+ userActions: probeFailureActions(BROWSER_STATUS_MODE.LOCAL, probe.error),
2454
+ tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.LOCAL),
2455
+ details: {
2456
+ runtime,
2457
+ launchProbeRequested: checkLocalLaunch,
2458
+ errorCode: probe.error.code,
2459
+ diagnostic: probe.diagnostic,
2460
+ attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
2461
+ },
2462
+ };
2463
+ }
2464
+
2465
+ // ── browser_status ────────────────────────────────────────────────────
2466
+
2467
+ export async function executeBrowserStatus(
2468
+ input: Record<string, unknown>,
2469
+ context: ToolContext,
2470
+ ): Promise<ToolExecutionResult> {
2471
+ const parsedMode = parseBrowserMode(input);
2472
+ if (!parsedMode.ok) {
2473
+ return { content: parsedMode.error, isError: true };
2474
+ }
2475
+
2476
+ if (
2477
+ input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] !== undefined &&
2478
+ typeof input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] !== "boolean"
2479
+ ) {
2480
+ return {
2481
+ content: `Error: ${BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH} must be a boolean when provided.`,
2482
+ isError: true,
2483
+ };
2484
+ }
2485
+
2486
+ const checkLocalLaunch =
2487
+ input[BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH] === true;
2488
+ const requestedMode = parsedMode.mode;
2489
+ const modesToCheck: readonly StatusCheckMode[] =
2490
+ requestedMode === BROWSER_MODE.AUTO
2491
+ ? BROWSER_STATUS_MODES
2492
+ : [requestedMode];
2493
+
2494
+ const autoCandidateKinds = buildCandidateList(context).map((c) => c.kind);
2495
+ const autoCandidateSet = new Set<CdpClientKind>(autoCandidateKinds);
2496
+
2497
+ try {
2498
+ const modeResults: BrowserStatusModeResult[] = [];
2499
+ for (const mode of modesToCheck) {
2500
+ const autoCandidate = autoCandidateSet.has(mode);
2501
+ if (mode === BROWSER_STATUS_MODE.EXTENSION) {
2502
+ modeResults.push(
2503
+ await checkExtensionModeStatus(context, autoCandidate),
2504
+ );
2505
+ } else if (mode === BROWSER_STATUS_MODE.CDP_INSPECT) {
2506
+ modeResults.push(
2507
+ await checkCdpInspectModeStatus(context, autoCandidate),
2508
+ );
2509
+ } else {
2510
+ modeResults.push(
2511
+ await checkLocalModeStatus(context, autoCandidate, checkLocalLaunch),
2512
+ );
2513
+ }
2514
+ }
2515
+
2516
+ const stickyMode = browserManager.getPreferredBackendKind(
2517
+ context.conversationId,
2518
+ );
2519
+ const availableModes = modeResults
2520
+ .filter((r) => r.available)
2521
+ .map((r) => r.mode);
2522
+ const recommendedMode =
2523
+ autoCandidateKinds.find((candidate) =>
2524
+ modeResults.some(
2525
+ (result) => result.mode === candidate && result.available,
2526
+ ),
2527
+ ) ??
2528
+ availableModes[0] ??
2529
+ null;
2530
+
2531
+ return {
2532
+ content: JSON.stringify(
2533
+ {
2534
+ requestedMode,
2535
+ checkedModes: modesToCheck,
2536
+ autoCandidateOrder: autoCandidateKinds,
2537
+ stickyConversationMode: stickyMode,
2538
+ recommendedMode,
2539
+ checkLocalLaunch,
2540
+ modes: modeResults,
2541
+ },
2542
+ null,
2543
+ 2,
2544
+ ),
2545
+ isError: false,
2546
+ };
2547
+ } catch (err) {
2548
+ const msg = err instanceof Error ? err.message : String(err);
2549
+ return {
2550
+ content: `Error: browser_status failed: ${msg}`,
2551
+ isError: true,
2552
+ };
2553
+ }
2554
+ }