@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
@@ -13,24 +13,7 @@ const BUTTON_MASKS = {
13
13
  iconify: [0, 0, 0, 0, 0x3ff, 0x3ff, 0, 0, 0, 0],
14
14
  };
15
15
 
16
- function windowTheme(accentHex: string | null): string {
17
- const tint = (base: string, amount: number): string => {
18
- if (!accentHex || !/^#[\da-f]{6}$/i.test(accentHex)) {
19
- return base;
20
- }
21
- return (
22
- "#" +
23
- [1, 3, 5]
24
- .map((offset) => {
25
- const from = parseInt(base.slice(offset, offset + 2), 16);
26
- const to = parseInt(accentHex.slice(offset, offset + 2), 16);
27
- return Math.round(from + (to - from) * amount)
28
- .toString(16)
29
- .padStart(2, "0");
30
- })
31
- .join("")
32
- );
33
- };
16
+ function windowTheme(): string {
34
17
  return `
35
18
  border.width: 1
36
19
  padding.width: 8
@@ -47,17 +30,15 @@ window.*.grip.bg: Flat Solid
47
30
  window.*.button.*.bg: Parentrelative
48
31
  window.*.button.*.hover.bg: Flat Solid
49
32
  window.*.button.*.pressed.bg: Flat Solid
50
- window.active.title.bg: Flat Gradient Horizontal
51
- window.active.title.bg.color: ${tint("#292930", 0.28)}
52
- window.active.title.bg.colorTo: #292930
33
+ window.active.title.bg.color: #292930
53
34
  window.active.border.color: #4a4a54
54
35
  window.active.title.separator.color: #202026
55
36
  window.active.label.text.color: #f0f0f4
56
37
  window.active.handle.bg.color: #292930
57
38
  window.active.grip.bg.color: #292930
58
39
  window.active.button.*.image.color: #d4d4dc
59
- window.active.button.*.hover.bg.color: ${tint("#45454f", 0.22)}
60
- window.active.button.*.pressed.bg.color: ${tint("#555561", 0.18)}
40
+ window.active.button.*.hover.bg.color: #45454f
41
+ window.active.button.*.pressed.bg.color: #555561
61
42
  window.active.button.*.disabled.image.color: #62626e
62
43
  window.active.button.close.hover.bg.color: #b64150
63
44
  window.active.button.close.pressed.bg.color: #963442
@@ -97,7 +78,6 @@ osd.label.text.color: #f0f0f4
97
78
  export function writeDesktopWindowTheme(
98
79
  configDir: string,
99
80
  home?: string,
100
- accentHex: string | null = null,
101
81
  ): string {
102
82
  let source: string | undefined;
103
83
  let sourcePath = "";
@@ -134,7 +114,7 @@ export function writeDesktopWindowTheme(
134
114
  mkdirSync(join(themeDir, "openbox-3"), { recursive: true });
135
115
  writeFileSync(
136
116
  join(themeDir, "openbox-3", "themerc"),
137
- windowTheme(accentHex).trimStart(),
117
+ windowTheme().trimStart(),
138
118
  );
139
119
  for (const [name, rows] of Object.entries(BUTTON_MASKS)) {
140
120
  const bytes = rows.flatMap((row) => [row & 0xff, row >> 8]);
@@ -1,9 +1,17 @@
1
1
  import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
2
2
  import { getIsContainerized, getIsPlatform } from "../config/env-registry.js";
3
3
  import type { AssistantConfig } from "../config/schema.js";
4
+ import type { ToolContext } from "../tools/types.js";
4
5
 
5
6
  const VIRTUAL_DESKTOP_FLAG = "assistant-desktop" as const;
6
7
 
8
+ export function isVirtualDesktopPlatform(
9
+ containerized: boolean = getIsContainerized(),
10
+ platformHosted: boolean = getIsPlatform(),
11
+ ): boolean {
12
+ return platformHosted && containerized;
13
+ }
14
+
7
15
  /** Gates desktop streaming and control to enabled, platform-hosted containers. */
8
16
  export function isVirtualDesktopEnabled(
9
17
  config?: AssistantConfig,
@@ -11,8 +19,50 @@ export function isVirtualDesktopEnabled(
11
19
  platformHosted: boolean = getIsPlatform(),
12
20
  ): boolean {
13
21
  return (
14
- platformHosted &&
15
- containerized &&
22
+ isVirtualDesktopPlatform(containerized, platformHosted) &&
16
23
  isAssistantFeatureFlagEnabled(VIRTUAL_DESKTOP_FLAG, config)
17
24
  );
18
25
  }
26
+
27
+ export function canUseVirtualDesktop(
28
+ context: Pick<ToolContext, "trustClass" | "sourceActorPrincipalId">,
29
+ enabled: boolean = isVirtualDesktopEnabled(),
30
+ ): boolean {
31
+ return (
32
+ context.trustClass === "guardian" &&
33
+ !!context.sourceActorPrincipalId &&
34
+ enabled
35
+ );
36
+ }
37
+
38
+ /** Web browser turns default to the streamed desktop; native apps keep their host. */
39
+ export function shouldUseVirtualDesktop(
40
+ context: Pick<
41
+ ToolContext,
42
+ "transportInterface" | "clientOs" | "trustClass" | "sourceActorPrincipalId"
43
+ >,
44
+ enabled: boolean = isVirtualDesktopEnabled(),
45
+ ): boolean {
46
+ return (
47
+ context.transportInterface === "web" &&
48
+ context.clientOs !== "macos" &&
49
+ context.clientOs !== "windows" &&
50
+ context.clientOs !== "linux" &&
51
+ canUseVirtualDesktop(context, enabled)
52
+ );
53
+ }
54
+
55
+ export function supportsVirtualDesktopComputerUse(toolName: string): boolean {
56
+ return [
57
+ "computer_use_observe",
58
+ "computer_use_click",
59
+ "computer_use_type_text",
60
+ "computer_use_key",
61
+ "computer_use_scroll",
62
+ "computer_use_drag",
63
+ "computer_use_wait",
64
+ "computer_use_sequence",
65
+ "computer_use_done",
66
+ "computer_use_respond",
67
+ ].includes(toolName);
68
+ }
@@ -3,7 +3,7 @@ import { afterEach, beforeEach, expect, spyOn, test } from "bun:test";
3
3
  import { setOverridesForTesting } from "../__tests__/feature-flag-test-helpers.js";
4
4
  import { shouldUseVirtualDesktopBrowser } from "../browser/virtual-desktop-target.js";
5
5
  import { executeDesktopBrowserOperation } from "./desktop-browser-operations.js";
6
- import { desktopDependencyInstaller } from "./desktop-dependencies.js";
6
+ import { desktopDependencies } from "./desktop-dependencies.js";
7
7
 
8
8
  const originalPlatform = process.env.IS_PLATFORM;
9
9
  const originalContainerized = process.env.IS_CONTAINERIZED;
@@ -15,15 +15,13 @@ const context = {
15
15
  transportInterface: "web" as const,
16
16
  clientOs: "web" as const,
17
17
  };
18
- let status: ReturnType<
19
- typeof spyOn<typeof desktopDependencyInstaller, "getStatus">
20
- >;
18
+ let status: ReturnType<typeof spyOn<typeof desktopDependencies, "getStatus">>;
21
19
 
22
20
  beforeEach(() => {
23
21
  process.env.IS_CONTAINERIZED = "true";
24
22
  process.env.IS_PLATFORM = "false";
25
23
  setOverridesForTesting({ "assistant-desktop": true });
26
- status = spyOn(desktopDependencyInstaller, "getStatus").mockReturnValue({
24
+ status = spyOn(desktopDependencies, "getStatus").mockReturnValue({
27
25
  state: "ready",
28
26
  });
29
27
  });
@@ -51,11 +49,9 @@ test("self-hosted Docker cannot automatically select or explicitly control virtu
51
49
  expect(status).not.toHaveBeenCalled();
52
50
  });
53
51
 
54
- test("platform web conversations select virtual Chrome before and during installation", () => {
52
+ test("platform web conversations select virtual Chrome even when image components are missing", () => {
55
53
  process.env.IS_PLATFORM = "true";
56
54
  expect(shouldUseVirtualDesktopBrowser(undefined, {}, context)).toBe(true);
57
- status.mockReturnValue({ state: "installing" });
58
- expect(shouldUseVirtualDesktopBrowser(undefined, {}, context)).toBe(true);
59
- status.mockReturnValue({ state: "required" });
55
+ status.mockReturnValue({ state: "failed" });
60
56
  expect(shouldUseVirtualDesktopBrowser(undefined, {}, context)).toBe(true);
61
57
  });
@@ -5,6 +5,8 @@
5
5
  * background jobs (e.g. proactive artifact generation) can persist documents
6
6
  * without going through the HTTP layer.
7
7
  */
8
+ import { randomUUID } from "node:crypto";
9
+
8
10
  import { rawAll, rawGet, rawRun } from "../persistence/raw-query.js";
9
11
  import { getLogger } from "../util/logger.js";
10
12
 
@@ -456,6 +458,27 @@ export function saveDocument(params: {
456
458
  }
457
459
  }
458
460
 
461
+ export const DEFAULT_DOCUMENT_TITLE = "Untitled Document";
462
+
463
+ /**
464
+ * Create a new document owned by `conversationId` under a freshly minted
465
+ * surface ID. A missing or empty title falls back to `DEFAULT_DOCUMENT_TITLE`.
466
+ */
467
+ export function createDocument(params: {
468
+ conversationId: string;
469
+ title?: string;
470
+ content?: string;
471
+ }): ReturnType<typeof saveDocument> {
472
+ const content = params.content ?? "";
473
+ return saveDocument({
474
+ surfaceId: `doc-${randomUUID()}`,
475
+ conversationId: params.conversationId,
476
+ title: params.title || DEFAULT_DOCUMENT_TITLE,
477
+ content,
478
+ wordCount: countWords(content),
479
+ });
480
+ }
481
+
459
482
  // ---------------------------------------------------------------------------
460
483
  // Find-and-replace
461
484
  // ---------------------------------------------------------------------------
@@ -179,10 +179,20 @@ mock.module("../../credential-health/credential-health-service.js", () => ({
179
179
  let heartbeatRunIdCounter = 0;
180
180
  let lastRunAtFromHistory: number | null = null;
181
181
  const skipHeartbeatRunCalls: Array<{ runId: string; reason: string }> = [];
182
+ const completeHeartbeatRunCalls: Array<{
183
+ runId: string;
184
+ result: { status: string; error?: string };
185
+ }> = [];
182
186
  mock.module("../heartbeat-run-store.js", () => ({
183
187
  insertPendingHeartbeatRun: () => `heartbeat-run-${++heartbeatRunIdCounter}`,
184
188
  startHeartbeatRun: () => true,
185
- completeHeartbeatRun: () => true,
189
+ completeHeartbeatRun: (
190
+ runId: string,
191
+ result: { status: string; error?: string },
192
+ ) => {
193
+ completeHeartbeatRunCalls.push({ runId, result });
194
+ return true;
195
+ },
186
196
  skipHeartbeatRun: (runId: string, reason: string) => {
187
197
  skipHeartbeatRunCalls.push({ runId, reason });
188
198
  },
@@ -226,6 +236,7 @@ beforeEach(() => {
226
236
  onBroadcast = null;
227
237
  runBackgroundJobCalls.length = 0;
228
238
  skipHeartbeatRunCalls.length = 0;
239
+ completeHeartbeatRunCalls.length = 0;
229
240
  lastRunAtFromHistory = null;
230
241
  preFirstMessageGateOpen = true;
231
242
  seedHeartbeat();
@@ -340,14 +351,9 @@ describe("HeartbeatService", () => {
340
351
  await service.runOnce({ force: true });
341
352
 
342
353
  expect(runnerCompleted).toBe(true);
343
- // No heartbeat_alert broadcast because the runner returned ok=true and
344
- // there was no outer-timeout failure to surface.
345
- expect(
346
- broadcastedMessages.filter((m) => m.type === "heartbeat_alert"),
347
- ).toHaveLength(0);
348
354
  });
349
355
 
350
- test("calls alerter with the failure message when the runner reports ok=false", async () => {
356
+ test("records the failure on the run when the runner reports ok=false", async () => {
351
357
  runBackgroundJobImpl = async () => ({
352
358
  conversationId: STUB_CONVERSATION_ID,
353
359
  ok: false,
@@ -359,25 +365,15 @@ describe("HeartbeatService", () => {
359
365
 
360
366
  await service.runOnce({ force: true });
361
367
 
362
- const alerts = broadcastedMessages.filter(
363
- (m) => m.type === "heartbeat_alert",
364
- );
365
- expect(alerts).toHaveLength(1);
366
- expect(alerts[0]).toMatchObject({
367
- type: "heartbeat_alert",
368
- title: "Heartbeat Failed",
369
- body: "LLM call failed",
370
- });
371
- });
372
-
373
- test("does not call alerter when the runner reports ok=true", async () => {
374
- const service = new HeartbeatService();
375
-
376
- await service.runOnce({ force: true });
377
-
378
- expect(
379
- broadcastedMessages.filter((m) => m.type === "heartbeat_alert"),
380
- ).toHaveLength(0);
368
+ expect(completeHeartbeatRunCalls).toEqual([
369
+ {
370
+ runId: expect.any(String),
371
+ result: expect.objectContaining({
372
+ status: "error",
373
+ error: "LLM call failed",
374
+ }),
375
+ },
376
+ ]);
381
377
  });
382
378
 
383
379
  test("scheduled run skips with reason 'pre_first_user_message' when the user has not yet interacted", async () => {
@@ -3,7 +3,6 @@ import { join } from "node:path";
3
3
 
4
4
  import { channelForBotProvider } from "@vellumai/service-contracts/channels";
5
5
 
6
- import type { HeartbeatAlertEvent } from "../api/events/heartbeat-alert.js";
7
6
  import { getConfig } from "../config/loader.js";
8
7
  import type { HeartbeatConfig } from "../config/schemas/heartbeat.js";
9
8
  import { warmGuardianBindings } from "../contacts/guardian-delivery-reader.js";
@@ -737,17 +736,6 @@ export class HeartbeatService {
737
736
  }
738
737
  } catch (err) {
739
738
  log.error({ err }, "Credential health check failed");
740
- try {
741
- broadcastMessage({
742
- type: "heartbeat_alert",
743
- title: "Credential Health Check Failed",
744
- body:
745
- "Could not verify OAuth credential health. " +
746
- (err instanceof Error ? err.message : String(err)),
747
- } satisfies HeartbeatAlertEvent);
748
- } catch {
749
- // Last resort — alerter itself failed. Already logged above.
750
- }
751
739
  }
752
740
  return [];
753
741
  }
@@ -912,32 +900,16 @@ export class HeartbeatService {
912
900
  "Heartbeat failed",
913
901
  );
914
902
 
915
- // The runner has already emitted `activity.failed` for the failure;
916
- // we still record the run-level error and broadcast the in-app
917
- // heartbeat alert so the existing surfacing keeps working.
918
- // Map the runner's error classification onto the run-store's status
919
- // enum so the run history preserves the timeout / error distinction.
903
+ // The runner has already emitted `activity.failed` for the failure, so
904
+ // this only records the run-level error. Map the runner's error
905
+ // classification onto the run-store's status enum so the run history
906
+ // preserves the timeout / error distinction.
920
907
  const runStatus = result.errorKind === "timeout" ? "timeout" : "error";
921
- const transitioned = completeHeartbeatRun(runId, {
908
+ completeHeartbeatRun(runId, {
922
909
  status: runStatus,
923
910
  conversationId: conversationId ?? result.conversationId,
924
911
  error: result.error?.message ?? "Unknown error",
925
912
  });
926
-
927
- // Only fire the in-app alerter when our completion is the one that
928
- // actually wrote — otherwise a parallel finalizer (e.g. a startup
929
- // recovery sweep) already alerted for this run.
930
- if (transitioned) {
931
- try {
932
- broadcastMessage({
933
- type: "heartbeat_alert",
934
- title: "Heartbeat Failed",
935
- body: result.error?.message ?? "Unknown error",
936
- } satisfies HeartbeatAlertEvent);
937
- } catch (alertErr) {
938
- log.error({ alertErr }, "Failed to broadcast heartbeat alert");
939
- }
940
- }
941
913
  }
942
914
 
943
915
  private readChecklist(): string {
@@ -16,7 +16,11 @@ import {
16
16
  test,
17
17
  } from "bun:test";
18
18
 
19
- import { desktopDependencyInstaller } from "../../desktop/desktop-dependencies.js";
19
+ import { setOverridesForTesting } from "../../__tests__/feature-flag-test-helpers.js";
20
+ import { isSessionGroupsEnabled } from "../../config/session-groups-gate.js";
21
+ import type { BrowserOperationToken } from "../../daemon/browser-mode-session.js";
22
+ import { BrowserModeSessionProducer } from "../../daemon/browser-mode-session.js";
23
+ import { desktopDependencies } from "../../desktop/desktop-dependencies.js";
20
24
  import * as desktopFeature from "../../desktop/virtual-desktop-feature.js";
21
25
  import { browserManager } from "../../tools/browser/browser-manager.js";
22
26
  import type { ToolExecutionResult } from "../../tools/types.js";
@@ -46,11 +50,69 @@ let mockConversation: {
46
50
  clientOs?: string;
47
51
  getTurnActorPrincipalId?: () => string | undefined;
48
52
  abortController?: AbortController;
53
+ currentRequestId?: string;
54
+ browserModeSessions?: {
55
+ beginOperation: (input: {
56
+ turnId: string;
57
+ lifecycle: "action" | "terminal" | "status";
58
+ at: number;
59
+ }) => BrowserOperationToken | undefined;
60
+ finishOperation: (
61
+ token: BrowserOperationToken | undefined,
62
+ outcome: {
63
+ at: number;
64
+ isError: boolean;
65
+ cancelled: boolean;
66
+ terminalReason?: "browser_closed" | "browser_detached";
67
+ },
68
+ ) => boolean;
69
+ };
49
70
  } | null = null;
50
71
 
72
+ const lifecycleBegins: Array<{
73
+ turnId: string;
74
+ lifecycle: "action" | "terminal" | "status";
75
+ at: number;
76
+ }> = [];
77
+ const lifecycleFinishes: Array<{
78
+ token: BrowserOperationToken | undefined;
79
+ outcome: {
80
+ at: number;
81
+ isError: boolean;
82
+ cancelled: boolean;
83
+ terminalReason?: "browser_closed" | "browser_detached";
84
+ };
85
+ }> = [];
86
+
87
+ function browserLifecycle() {
88
+ return {
89
+ beginOperation(input: (typeof lifecycleBegins)[number]) {
90
+ lifecycleBegins.push(input);
91
+ return {
92
+ turnId: input.turnId,
93
+ lifecycle: input.lifecycle === "status" ? "action" : input.lifecycle,
94
+ owner: { id: "browser-session", mode: "browser" as const },
95
+ } as BrowserOperationToken;
96
+ },
97
+ finishOperation(
98
+ token: BrowserOperationToken | undefined,
99
+ outcome: (typeof lifecycleFinishes)[number]["outcome"],
100
+ ) {
101
+ lifecycleFinishes.push({ token, outcome });
102
+ return true;
103
+ },
104
+ };
105
+ }
106
+
51
107
  let mockFindConversationCalls: string[] = [];
52
108
 
53
109
  mock.module("../../browser/operations.js", () => ({
110
+ browserOperationLifecycle: (operation: string) =>
111
+ operation === "status"
112
+ ? "status"
113
+ : operation === "close" || operation === "detach"
114
+ ? "terminal"
115
+ : "action",
54
116
  executeBrowserOperation: async (
55
117
  operation: string,
56
118
  input: Record<string, unknown>,
@@ -78,6 +140,7 @@ mock.module("../../daemon/conversation-registry.js", () => ({
78
140
  mockFindConversationCalls.push(conversationId);
79
141
  return mockConversation ?? undefined;
80
142
  },
143
+ findConversationOrSubagent: () => mockConversation ?? undefined,
81
144
  }));
82
145
 
83
146
  let desktopEnabled = false;
@@ -87,10 +150,9 @@ const enabledSpy = spyOn(
87
150
  desktopFeature,
88
151
  "isVirtualDesktopEnabled",
89
152
  ).mockImplementation(() => desktopEnabled);
90
- const readySpy = spyOn(
91
- desktopDependencyInstaller,
92
- "getStatus",
93
- ).mockImplementation(() => ({ state: desktopReady ? "ready" : "required" }));
153
+ const readySpy = spyOn(desktopDependencies, "getStatus").mockImplementation(
154
+ () => ({ state: desktopReady ? "ready" : "failed" }),
155
+ );
94
156
  afterAll(() => {
95
157
  enabledSpy.mockRestore();
96
158
  readySpy.mockRestore();
@@ -147,6 +209,7 @@ function callHandler(
147
209
  // ---------------------------------------------------------------------------
148
210
 
149
211
  afterEach(() => {
212
+ setOverridesForTesting({});
150
213
  desktopEnabled = false;
151
214
  desktopReady = false;
152
215
  desktopFailure = false;
@@ -156,6 +219,8 @@ afterEach(() => {
156
219
  mockOperationCalls = [];
157
220
  mockConversation = null;
158
221
  mockFindConversationCalls = [];
222
+ lifecycleBegins.length = 0;
223
+ lifecycleFinishes.length = 0;
159
224
  });
160
225
 
161
226
  // ---------------------------------------------------------------------------
@@ -471,6 +536,150 @@ describe("browser_execute route", () => {
471
536
  });
472
537
  });
473
538
 
539
+ test("brackets a typed live-turn action with browser lifecycle", async () => {
540
+ mockConversation = {
541
+ currentRequestId: "turn-123",
542
+ browserModeSessions: browserLifecycle(),
543
+ getTurnActorPrincipalId: () => undefined,
544
+ };
545
+
546
+ await callHandler({
547
+ operation: "navigate",
548
+ input: { url: "https://example.com" },
549
+ conversationId: "conv-live",
550
+ });
551
+
552
+ expect(lifecycleBegins).toEqual([
553
+ expect.objectContaining({ turnId: "turn-123", lifecycle: "action" }),
554
+ ]);
555
+ expect(lifecycleFinishes).toEqual([
556
+ expect.objectContaining({
557
+ token: expect.objectContaining({ turnId: "turn-123" }),
558
+ outcome: expect.objectContaining({
559
+ isError: false,
560
+ cancelled: false,
561
+ }),
562
+ }),
563
+ ]);
564
+ });
565
+
566
+ test("executes a live-turn action without tracking while session groups are disabled", async () => {
567
+ setOverridesForTesting({ "session-groups": false });
568
+ const activateSource = mock(() => {
569
+ throw new Error("disabled tracking must not activate a source");
570
+ });
571
+ mockConversation = {
572
+ currentRequestId: "turn-123",
573
+ browserModeSessions: new BrowserModeSessionProducer(
574
+ {
575
+ activateSource,
576
+ claimTurn: mock(() => undefined),
577
+ getTurnOwner: mock(() => undefined),
578
+ recordActivity: mock(() => false),
579
+ retireSource: mock(() => false),
580
+ },
581
+ 1,
582
+ isSessionGroupsEnabled,
583
+ ),
584
+ getTurnActorPrincipalId: () => undefined,
585
+ };
586
+
587
+ const result = await callHandler({
588
+ operation: "navigate",
589
+ input: { url: "https://example.com" },
590
+ conversationId: "conv-live",
591
+ });
592
+
593
+ expect(result).toEqual({ content: "ok", isError: false });
594
+ expect(mockOperationCalls).toHaveLength(1);
595
+ expect(activateSource).not.toHaveBeenCalled();
596
+ });
597
+
598
+ test.each(["navigate", "close"] as const)(
599
+ "executes %s once after tracking admission throws",
600
+ async (operation) => {
601
+ mockConversation = {
602
+ currentRequestId: "turn-123",
603
+ browserModeSessions: {
604
+ ...browserLifecycle(),
605
+ beginOperation() {
606
+ throw new Error("tracking unavailable");
607
+ },
608
+ },
609
+ getTurnActorPrincipalId: () => undefined,
610
+ };
611
+ const result = await callHandler({
612
+ operation,
613
+ conversationId: "conv-live",
614
+ });
615
+ expect(result).toEqual({ content: "ok", isError: false });
616
+ expect(mockOperationCalls).toHaveLength(1);
617
+ },
618
+ );
619
+
620
+ test("passes typed terminal success and failure outcomes", async () => {
621
+ mockConversation = {
622
+ currentRequestId: "turn-123",
623
+ browserModeSessions: browserLifecycle(),
624
+ getTurnActorPrincipalId: () => undefined,
625
+ };
626
+
627
+ await callHandler({ operation: "detach", conversationId: "conv-live" });
628
+ expect(lifecycleBegins[0]).toMatchObject({ lifecycle: "terminal" });
629
+ expect(lifecycleFinishes[0]?.outcome).toMatchObject({
630
+ isError: false,
631
+ terminalReason: "browser_detached",
632
+ });
633
+
634
+ mockOperationResult = { content: "close failed", isError: true };
635
+ await callHandler({ operation: "close", conversationId: "conv-live" });
636
+ expect(lifecycleFinishes[1]?.outcome).toMatchObject({
637
+ isError: true,
638
+ terminalReason: "browser_closed",
639
+ });
640
+ });
641
+
642
+ test.each(["navigate", "close"] as const)(
643
+ "preserves a successful %s result when session tracking fails",
644
+ async (operation) => {
645
+ mockOperationResult = {
646
+ content: `${operation} completed`,
647
+ isError: false,
648
+ };
649
+ const lifecycle = browserLifecycle();
650
+ mockConversation = {
651
+ currentRequestId: "turn-123",
652
+ browserModeSessions: {
653
+ ...lifecycle,
654
+ finishOperation(token, outcome) {
655
+ lifecycle.finishOperation(token, outcome);
656
+ throw new Error("session tracking unavailable");
657
+ },
658
+ },
659
+ getTurnActorPrincipalId: () => undefined,
660
+ };
661
+
662
+ const result = await callHandler({
663
+ operation,
664
+ input: operation === "navigate" ? { url: "https://example.com" } : {},
665
+ conversationId: "conv-live",
666
+ });
667
+
668
+ expect(result).toEqual({
669
+ content: `${operation} completed`,
670
+ isError: false,
671
+ });
672
+ expect(mockOperationCalls).toHaveLength(1);
673
+ expect(lifecycleFinishes).toHaveLength(1);
674
+ },
675
+ );
676
+
677
+ test("does not observe lifecycle for a standalone CLI operation", async () => {
678
+ await callHandler({ operation: "navigate", sessionId: "standalone" });
679
+ expect(lifecycleBegins).toEqual([]);
680
+ expect(lifecycleFinishes).toEqual([]);
681
+ });
682
+
474
683
  // ── Input defaults ─────────────────────────────────────────────────
475
684
 
476
685
  test("defaults input to empty object when omitted", async () => {
@@ -639,7 +848,7 @@ test("a streamed browser failure does not switch to Playwright or personal Chrom
639
848
  expect(mockOperationCalls).toHaveLength(0);
640
849
  });
641
850
 
642
- test("first web browser use routes to virtual desktop setup before installation", async () => {
851
+ test("missing image components do not redirect web browser use to a personal browser", async () => {
643
852
  webConversation();
644
853
  desktopReady = false;
645
854
  await callHandler({
@@ -59,6 +59,7 @@ import { RouteResponse } from "../runtime/routes/types.js";
59
59
  import { getLogger } from "../util/logger.js";
60
60
  import { mapGatewayIpcConnectError } from "./gateway-ipc-errors.js";
61
61
  import { ACTIVATION_SYNC_IPC_METHODS } from "./routes/activation-sync-ipc-routes.js";
62
+ import { CHANNEL_REPLY_IPC_METHODS } from "./routes/channel-reply-ipc-routes.js";
62
63
  import { CONTACTS_INFO_IPC_METHODS } from "./routes/contacts-info-ipc-routes.js";
63
64
  import { CONTACTS_MIRROR_IPC_METHODS } from "./routes/contacts-mirror-ipc-routes.js";
64
65
  import { CONVERSATION_SYNC_IPC_METHODS } from "./routes/conversation-sync-ipc-routes.js";
@@ -216,6 +217,7 @@ export class AssistantIpcServer {
216
217
  // never in ROUTES.
217
218
  for (const methodMap of [
218
219
  INVITE_IPC_METHODS,
220
+ CHANNEL_REPLY_IPC_METHODS,
219
221
  CONTACTS_INFO_IPC_METHODS,
220
222
  CONTACTS_MIRROR_IPC_METHODS,
221
223
  GUARDIAN_LABEL_IPC_METHODS,