@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
@@ -16,7 +16,7 @@
16
16
  */
17
17
 
18
18
  import { repairHistory } from "../agent/history-repair/history-repair.js";
19
- import type { AgentLoopConfig } from "../agent/loop.js";
19
+ import type { AgentLoopConfig, PreparedModelCall } from "../agent/loop.js";
20
20
  import { AgentLoop } from "../agent/loop.js";
21
21
  import type { AssistantActivityStateEvent } from "../api/events/assistant-activity-state.js";
22
22
  import type { ConfirmationStateChangedEvent } from "../api/events/confirmation-state-changed.js";
@@ -39,6 +39,7 @@ import { resolveCallSiteConfig } from "../config/llm-resolver.js";
39
39
  import { getConfig } from "../config/loader.js";
40
40
  import type { LLMCallSite, Speed } from "../config/schemas/llm.js";
41
41
  import { resolveSendUserMessageActive } from "../config/send-user-message-gate.js";
42
+ import { isSessionGroupsEnabled } from "../config/session-groups-gate.js";
42
43
  import {
43
44
  derefToolResultReReads,
44
45
  postTurnTruncateToolResults,
@@ -118,6 +119,8 @@ import { withSqliteRetry } from "../util/sqlite-retry.js";
118
119
  import type { WorkspaceGitService } from "../workspace/git-service.js";
119
120
  import type { commitTurnChanges } from "../workspace/turn-commit.js";
120
121
  import type { AssistantAttachmentDraft } from "./assistant-attachments.js";
122
+ import { BrowserModeSessionProducer } from "./browser-mode-session.js";
123
+ import { ComputerUseModeSessionProducer } from "./computer-use-mode-session.js";
121
124
  import type { AssistantSurface } from "./conversation-agent-loop.js";
122
125
  import {
123
126
  applyCompactionResult,
@@ -141,6 +144,7 @@ import {
141
144
  persistUserMessage as persistUserMessageImpl,
142
145
  redirectToSecurePrompt as redirectToSecurePromptImpl,
143
146
  } from "./conversation-messaging.js";
147
+ import { ConversationModeSessionCoordinator } from "./conversation-mode-session.js";
144
148
  // Extracted modules
145
149
  import { registerConversationNotifiers } from "./conversation-notifiers.js";
146
150
  import type { ProcessMessageOptions } from "./conversation-process.js";
@@ -193,6 +197,7 @@ import { filterMessagesForUntrustedActor } from "./message-provenance.js";
193
197
  import type { ConversationTransportMetadata } from "./message-types/conversations.js";
194
198
  import { isHostProxyTransport } from "./message-types/conversations.js";
195
199
  import { conversationMetadataSyncTag } from "./message-types/sync.js";
200
+ import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
196
201
  import { renderReactionHistoryText } from "./reaction-history-render.js";
197
202
  import type { QueuedReactionRecord } from "./reaction-record.js";
198
203
  import {
@@ -201,6 +206,7 @@ import {
201
206
  } from "./summarize-boundary.js";
202
207
 
203
208
  const log = getLogger("conversation");
209
+ const PROMPT_CACHE_WARM_MAX_TOKENS = 16;
204
210
 
205
211
  /**
206
212
  * First text block of a persisted message row's content, mirroring
@@ -525,6 +531,13 @@ export class Conversation {
525
531
  */
526
532
  enabledPlugins: string[] | null = null;
527
533
  /** @internal */ currentRequestId?: string;
534
+ /** Canonical recorded-session ownership for this conversation. */
535
+ readonly modeSessions: ConversationModeSessionCoordinator;
536
+ /** Computer-use producer mapped onto the canonical session coordinator. */
537
+ readonly computerUseModeSessions: ComputerUseModeSessionProducer;
538
+ /** Browser producer mapped onto the canonical session coordinator. */
539
+ readonly browserModeSessions: BrowserModeSessionProducer;
540
+ private liveVoiceResidencyLeases = 0;
528
541
  /**
529
542
  * The `clientMessageId` the running turn was started by, recorded in the same
530
543
  * synchronous step that takes the processing lock.
@@ -783,17 +796,22 @@ export class Conversation {
783
796
  */
784
797
  pendingInterruptActivityBridge = false;
785
798
  /**
786
- * Set by `interruptRunningTurn` when the abort it ran landed with no tool
787
- * call in flight, and consumed exactly once by the persist of the
788
- * interrupting user message, which appends
799
+ * Set by `interruptRunningTurn` on every handover it completes, and consumed
800
+ * exactly once by the first user message persisted after it, which appends
789
801
  * {@link INTERRUPTED_TURN_NOTE_TEXT} to that message's LLM-facing content
790
802
  * and stamps `interruptedPriorTurn` on the row.
791
803
  *
792
- * An interrupt caught mid-tool needs nothing here: the synthetic
793
- * `tool_result` the loop or the repair writes already tells the model a
794
- * message preempted it. Caught mid-provider-call there is no `tool_use` to
795
- * answer, so the note is the only signal, and it rides on the message that
796
- * did the interrupting.
804
+ * The note is the only place the behavior after an interrupt is spelled out.
805
+ * A synthetic `tool_result` states what happened to the one call it answers
806
+ * and nothing more, so an interrupt caught mid-tool arms this too.
807
+ *
808
+ * It belongs to the history position, not to the send that armed it. That
809
+ * first row is the one sitting directly under the work the handover stopped,
810
+ * so it is the row whose note the model reads in the right place. Normally it
811
+ * is the interrupting message itself. When that send loses the lock race and
812
+ * queues, the message that persists first was also sent while the assistant
813
+ * was working, and the note is true of it; the queued one drains after a
814
+ * completed turn, where the same note would be stale.
797
815
  * @internal
798
816
  */
799
817
  pendingInterruptNote = false;
@@ -975,6 +993,16 @@ export class Conversation {
975
993
  const { maxTokens, speedOverride, cacheTtl, modelOverride } = options ?? {};
976
994
  const enableNativeWebSearch = options?.enableNativeWebSearch ?? false;
977
995
  this.conversationId = conversationId;
996
+ this.modeSessions = new ConversationModeSessionCoordinator(conversationId);
997
+ this.computerUseModeSessions = new ComputerUseModeSessionProducer(
998
+ this.modeSessions,
999
+ isSessionGroupsEnabled,
1000
+ );
1001
+ this.browserModeSessions = new BrowserModeSessionProducer(
1002
+ this.modeSessions,
1003
+ 1,
1004
+ isSessionGroupsEnabled,
1005
+ );
978
1006
  this.parentConversationId = options?.parentConversationId;
979
1007
  this.systemPrompt = systemPrompt;
980
1008
  this.provider = provider;
@@ -1237,48 +1265,91 @@ export class Conversation {
1237
1265
  // ── Prompt Cache Warming ─────────────────────────────────────────
1238
1266
 
1239
1267
  /**
1240
- * Fire-and-forget LLM call with max_tokens=1 to populate the provider's
1241
- * prompt cache (system prompt + tools). Called after the canned first
1242
- * greeting so the user's next real message gets a cache hit.
1268
+ * Non-rejecting LLM call with a minimal output budget to populate the selected
1269
+ * provider's prompt cache (system prompt + tools).
1243
1270
  */
1244
- warmPromptCache(): void {
1271
+ async warmPromptCache(options?: {
1272
+ callSite?: LLMCallSite;
1273
+ overrideProfile?: string;
1274
+ forceOverrideProfile?: boolean;
1275
+ signal?: AbortSignal;
1276
+ systemPrompt?: string | null;
1277
+ tools?: ToolDefinition[];
1278
+ }): Promise<void> {
1245
1279
  this.cacheWarmAbort?.abort();
1246
1280
  const abort = new AbortController();
1247
1281
  this.cacheWarmAbort = abort;
1248
1282
 
1249
- const systemPrompt = this.buildCurrentSystemPrompt();
1250
- const tools = getAllToolDefinitions();
1251
- const provider = this.provider;
1283
+ const externalSignal = options?.signal;
1284
+ const relayAbort = (): void => abort.abort(externalSignal?.reason);
1285
+ if (externalSignal?.aborted) {
1286
+ relayAbort();
1287
+ } else {
1288
+ externalSignal?.addEventListener("abort", relayAbort, { once: true });
1289
+ }
1252
1290
 
1253
- const warmMessage: Message = {
1254
- role: "user",
1255
- content: [{ type: "text", text: "hi" }],
1256
- };
1291
+ const callSite = options?.callSite ?? "mainAgent";
1257
1292
 
1258
- provider
1259
- .sendMessage([warmMessage], {
1260
- tools,
1261
- systemPrompt,
1293
+ try {
1294
+ const hasSystemPrompt =
1295
+ options !== undefined &&
1296
+ Object.prototype.hasOwnProperty.call(options, "systemPrompt");
1297
+ const systemPrompt = hasSystemPrompt
1298
+ ? (options.systemPrompt ?? undefined)
1299
+ : this.buildCurrentSystemPrompt();
1300
+ const tools =
1301
+ options?.tools ?? this.agentLoop.getResolvedTools(this.messages);
1302
+ const providerConfig = {
1303
+ ...(options?.overrideProfile !== undefined
1304
+ ? { overrideProfile: options.overrideProfile }
1305
+ : {}),
1306
+ ...(options?.forceOverrideProfile !== undefined
1307
+ ? { forceOverrideProfile: options.forceOverrideProfile }
1308
+ : {}),
1309
+ selectionSeed: this.conversationId,
1310
+ };
1311
+ const warmMessage: Message = {
1312
+ role: "user",
1313
+ content: [{ type: "text", text: "hi" }],
1314
+ };
1315
+
1316
+ await this.provider.sendMessage([warmMessage], {
1317
+ tools: tools.length > 0 ? tools : undefined,
1318
+ ...(systemPrompt !== undefined ? { systemPrompt } : {}),
1262
1319
  config: {
1263
- max_tokens: 1,
1264
- callSite: "mainAgent",
1265
- usageTracking: "manual",
1320
+ max_tokens: PROMPT_CACHE_WARM_MAX_TOKENS,
1321
+ callSite,
1322
+ ...providerConfig,
1323
+ conversationId: this.conversationId,
1266
1324
  },
1267
1325
  signal: abort.signal,
1268
- })
1269
- .then(() => {
1270
- log.info("Prompt cache warmed successfully");
1271
- })
1272
- .catch((err) => {
1273
- if (!abort.signal.aborted) {
1274
- log.warn({ err }, "Prompt cache warming failed (non-fatal)");
1275
- }
1276
- })
1277
- .finally(() => {
1278
- if (this.cacheWarmAbort === abort) {
1279
- this.cacheWarmAbort = undefined;
1280
- }
1281
1326
  });
1327
+ if (!abort.signal.aborted) {
1328
+ log.info(
1329
+ {
1330
+ callSite,
1331
+ profile: options?.overrideProfile ?? null,
1332
+ },
1333
+ "Prompt cache warmed successfully",
1334
+ );
1335
+ }
1336
+ } catch (err) {
1337
+ if (!abort.signal.aborted) {
1338
+ log.warn(
1339
+ {
1340
+ err,
1341
+ callSite,
1342
+ profile: options?.overrideProfile ?? null,
1343
+ },
1344
+ "Prompt cache warming failed (non-fatal)",
1345
+ );
1346
+ }
1347
+ } finally {
1348
+ externalSignal?.removeEventListener("abort", relayAbort);
1349
+ if (this.cacheWarmAbort === abort) {
1350
+ this.cacheWarmAbort = undefined;
1351
+ }
1352
+ }
1282
1353
  }
1283
1354
 
1284
1355
  // ── Lifecycle ────────────────────────────────────────────────────
@@ -2509,14 +2580,31 @@ export class Conversation {
2509
2580
  return !this.queue.isEmpty;
2510
2581
  }
2511
2582
 
2583
+ acquireLiveVoiceResidency(): () => void {
2584
+ this.liveVoiceResidencyLeases += 1;
2585
+ let released = false;
2586
+ return () => {
2587
+ if (released) {
2588
+ return;
2589
+ }
2590
+ released = true;
2591
+ this.liveVoiceResidencyLeases = Math.max(
2592
+ 0,
2593
+ this.liveVoiceResidencyLeases - 1,
2594
+ );
2595
+ };
2596
+ }
2597
+
2512
2598
  /**
2513
2599
  * True when dropping this instance would lose work that is still in flight:
2514
- * a live turn, a queued successor, or a child subagent.
2600
+ * a live turn, queued successor, child subagent, or mode-session lifecycle.
2515
2601
  */
2516
2602
  hasInFlightWork(): boolean {
2517
2603
  return (
2518
2604
  this.isProcessing() ||
2519
2605
  this.hasQueuedMessages() ||
2606
+ this.liveVoiceResidencyLeases > 0 ||
2607
+ this.modeSessions.hasResidentWork() ||
2520
2608
  getSubagentManager().hasActiveChildren(this.conversationId)
2521
2609
  );
2522
2610
  }
@@ -2643,6 +2731,16 @@ export class Conversation {
2643
2731
 
2644
2732
  setHostCuProxy(proxy: HostCuProxy | undefined): void {
2645
2733
  if (this.hostCuProxy && this.hostCuProxy !== proxy) {
2734
+ const previousProxy = this.hostCuProxy;
2735
+ bestEffortModeSessionTracking("computer use proxy replacement", () => {
2736
+ this.computerUseModeSessions.endTask({
2737
+ turnId: this.currentRequestId,
2738
+ source: {
2739
+ sourceId: previousProxy.sourceId,
2740
+ generation: previousProxy.resetGeneration,
2741
+ },
2742
+ });
2743
+ });
2646
2744
  this.hostCuProxy.dispose();
2647
2745
  }
2648
2746
  this.hostCuProxy = proxy;
@@ -3417,6 +3515,8 @@ export class Conversation {
3417
3515
  */
3418
3516
  replyDeliveredInAppOnly?: boolean;
3419
3517
  callSite?: LLMCallSite;
3518
+ /** Provider configuration source when distinct from turn semantics. */
3519
+ inferenceCallSite?: LLMCallSite;
3420
3520
  /**
3421
3521
  * Optional ad-hoc inference-profile override applied to every LLM call
3422
3522
  * the loop issues for this turn. Forwarded into
@@ -3428,6 +3528,8 @@ export class Conversation {
3428
3528
  overrideProfile?: string;
3429
3529
  /** Float `overrideProfile` above call-site layers for this run. */
3430
3530
  forceOverrideProfile?: boolean;
3531
+ /** Observe the first finalized model request without delaying it. */
3532
+ onFirstModelCallPrepared?: (prepared: PreparedModelCall) => void;
3431
3533
  /**
3432
3534
  * Firing's `cron_runs.id` stamped onto this turn's usage rows. Per-turn:
3433
3535
  * forwarded into {@link runAgentLoopImpl} and threaded to `recordUsage`.
@@ -98,14 +98,16 @@ export function setImageGenModel(modelId: string, ctx: ModelSetContext): void {
98
98
  // Keep the derived provider in sync with the selected model so downstream
99
99
  // routing never sends a Gemini request to an OpenAI model (or vice versa).
100
100
  // The prefix logic is shared with workspace migration 006-services-config
101
- // via providerForImageModelPrefix(). Provider "vellum" is exempt: it is
102
- // managed for every model (the backend derives from the model at request
103
- // time), and rewriting it here would silently move a managed user onto a
104
- // BYOK provider they may hold no key for.
101
+ // via providerForImageModelPrefix(). Providers "vellum" and "openrouter"
102
+ // are exempt: vellum is managed for every model, and openrouter owns every
103
+ // selected slug (including google/ and openai/ prefixes). Rewriting either
104
+ // would silently move the user onto a BYOK provider they may hold no key
105
+ // for.
105
106
  const services = raw.services as
106
107
  | Record<string, { provider?: string }>
107
108
  | undefined;
108
- if (services?.["image-generation"]?.provider !== "vellum") {
109
+ const currentProvider = services?.["image-generation"]?.provider;
110
+ if (currentProvider !== "vellum" && currentProvider !== "openrouter") {
109
111
  setServiceField(
110
112
  raw,
111
113
  "image-generation",
@@ -3,8 +3,13 @@ import {
3
3
  SENTINEL_REDACTION_VERSION,
4
4
  } from "@vellumai/service-contracts/redacted-credential";
5
5
  import { v4 as uuid } from "uuid";
6
+ import { z } from "zod";
6
7
 
7
8
  import { AnsweredQuestionSchema } from "../../api/events/question-answered.js";
9
+ import {
10
+ type ToolActivityMetadata,
11
+ ToolActivityMetadataSchema,
12
+ } from "../../api/events/tool-result.js";
8
13
  import type {
9
14
  ConversationContentBlock,
10
15
  ConversationMessageAttachment,
@@ -246,6 +251,33 @@ export interface ConversationCreateOptions {
246
251
  conversationType?: ConversationCreateType;
247
252
  }
248
253
 
254
+ /** One schema per activity entry, so each entry validates on its own. */
255
+ const ACTIVITY_ENTRY_SCHEMAS = Object.entries(
256
+ ToolActivityMetadataSchema.shape,
257
+ ).map(([key, schema]) => z.object({ [key]: schema }));
258
+
259
+ /**
260
+ * Validates a persisted `_activityMetadata` rider one tool's entry at a time.
261
+ * Cards render these fields directly, so an entry that no longer matches its
262
+ * schema is dropped (its card degrades to the result text) without taking a
263
+ * valid sibling entry with it.
264
+ */
265
+ function readPersistedActivityMetadata(
266
+ value: unknown,
267
+ ): ToolActivityMetadata | undefined {
268
+ if (!isRecord(value)) {
269
+ return undefined;
270
+ }
271
+ const activity: ToolActivityMetadata = {};
272
+ for (const schema of ACTIVITY_ENTRY_SCHEMAS) {
273
+ const parsed = schema.safeParse(value);
274
+ if (parsed.success) {
275
+ Object.assign(activity, parsed.data);
276
+ }
277
+ }
278
+ return Object.keys(activity).length > 0 ? activity : undefined;
279
+ }
280
+
249
281
  function isRecord(value: unknown): value is Record<string, unknown> {
250
282
  return typeof value === "object" && value != null;
251
283
  }
@@ -661,12 +693,12 @@ export function renderHistoryContent(
661
693
  entry.riskDirectoryScopeOptions =
662
694
  block._riskDirectoryScopeOptions as HistoryToolCall["riskDirectoryScopeOptions"];
663
695
  }
664
- // Read back tool activity (web_search / web_fetch) persisted by
665
- // `annotatePersistedAssistantMessage` so the activity card survives a
666
- // history reopen instead of degrading to the plain result text.
667
- if (isRecord(block._activityMetadata)) {
668
- entry.activityMetadata =
669
- block._activityMetadata as HistoryToolCall["activityMetadata"];
696
+ // Read back tool activity persisted by `annotatePersistedAssistantMessage`
697
+ // so the activity card survives a history reopen instead of degrading to
698
+ // the plain result text.
699
+ const activity = readPersistedActivityMetadata(block._activityMetadata);
700
+ if (activity) {
701
+ entry.activityMetadata = activity;
670
702
  }
671
703
  // Read back the answered `ask_question` record so the answered card
672
704
  // rehydrates from history. Validated (rather than trusted like the
@@ -708,9 +740,9 @@ export function renderHistoryContent(
708
740
  }
709
741
  // Native server tools (Anthropic web_search) persist their activity on
710
742
  // the server_tool_use block, so read it back here too.
711
- if (isRecord(block._activityMetadata)) {
712
- entry.activityMetadata =
713
- block._activityMetadata as HistoryToolCall["activityMetadata"];
743
+ const activity = readPersistedActivityMetadata(block._activityMetadata);
744
+ if (activity) {
745
+ entry.activityMetadata = activity;
714
746
  }
715
747
  toolCalls.push(entry);
716
748
  if (id) {
@@ -936,17 +968,24 @@ async function mintCollectionLinkFallback(
936
968
  *
937
969
  * Lifecycle state (resolver, timer) is registered in pendingInteractions — the
938
970
  * same tracker the in-conversation SecretPrompter uses — so `POST /v1/secret`
939
- * resolves the prompt generically. When a `conversationId` is supplied (the CLI
940
- * `credentials prompt` command forwards `__CONVERSATION_ID`), the broadcast is
941
- * scoped to that conversation so clients deliver it; otherwise it is
942
- * conversation-less. When that conversation's channel cannot render dynamic UI
943
- * (e.g. slack, telegram), resolves immediately with `unsupported_channel` —
944
- * carrying a one-time collection link when the gateway can mint one — instead
945
- * of broadcasting a request that can only time out.
971
+ * resolves the prompt generically. The broadcast is scoped to the supplied
972
+ * `conversationId` (the CLI `credentials prompt` command forwards
973
+ * `__CONVERSATION_ID`) so clients deliver it. Two cases have no surface that
974
+ * can render the card and resolve immediately with `unsupported_channel`,
975
+ * carrying a one-time collection link when the gateway can mint one, instead
976
+ * of broadcasting a request that can only time out:
977
+ * - the conversation's channel cannot render dynamic UI (slack, telegram);
978
+ * - there is no conversation at all (a headless exec such as Doctor's
979
+ * `run_assistant_cli`). Clients drop conversation-scoped events without a
980
+ * conversationId and `/v1/pending-interactions` is keyed by conversation,
981
+ * so a conversation-less `secret_request` is invisible everywhere.
946
982
  */
947
983
  export function requestSecretStandalone(
948
984
  params: StandaloneSecretParams,
949
985
  ): Promise<SecretPromptResult> {
986
+ if (!params.conversationId) {
987
+ return mintCollectionLinkFallback(params);
988
+ }
950
989
  const conversation = findConversation(params.conversationId);
951
990
  if (conversation && !conversationSupportsDynamicUi(conversation)) {
952
991
  return mintCollectionLinkFallback(params);
@@ -13,7 +13,6 @@
13
13
  * RPC lifecycle (resolve/reject/timer/detachAbort) is stored in
14
14
  * pendingInteractions alongside routing metadata.
15
15
  */
16
-
17
16
  import { v4 as uuid } from "uuid";
18
17
 
19
18
  import { loadConfig } from "../config/loader.js";
@@ -30,6 +29,7 @@ import type { ToolExecutionResult } from "../tools/types.js";
30
29
  import { AssistantError, ErrorCode } from "../util/errors.js";
31
30
  import { getLogger } from "../util/logger.js";
32
31
  import { resolveHostCuTarget } from "./host-cu-target.js";
32
+ import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
33
33
 
34
34
  const log = getLogger("host-cu-proxy");
35
35
 
@@ -161,27 +161,14 @@ function isNoDiffKeyAction(action: ActionIdentity | undefined): boolean {
161
161
  );
162
162
  }
163
163
 
164
- /**
165
- * Canonical signature for loop detection. Key presses collapse equivalent
166
- * spellings (`cmd+a`, `command+a`, `cmd + a`) of the same combo so a stuck
167
- * session retrying it with alias/whitespace variants is still caught —
168
- * important now that exempt keys no longer emit no-effect warnings. Only the
169
- * `key` value is normalized; all other input fields (e.g. the routing
170
- * `target_client_id`) are preserved, so the same combo sent to different
171
- * desktop clients is not mistaken for a repeat.
172
- */
164
+ /** Normalize key aliases and ignore observation tokens when detecting repeats. */
173
165
  function actionSignature(record: ActionRecord): string {
174
- if (
175
- record.toolName === "computer_use_key" &&
176
- typeof record.input.key === "string"
177
- ) {
178
- const normalizedInput = {
179
- ...record.input,
180
- key: canonicalizeKeyCombo(record.input.key),
181
- };
182
- return `computer_use_key:${JSON.stringify(normalizedInput)}`;
166
+ const input = { ...record.input };
167
+ delete input.observation_id;
168
+ if (record.toolName === "computer_use_key" && typeof input.key === "string") {
169
+ input.key = canonicalizeKeyCombo(input.key);
183
170
  }
184
- return `${record.toolName}:${JSON.stringify(record.input)}`;
171
+ return `${record.toolName}:${JSON.stringify(input)}`;
185
172
  }
186
173
 
187
174
  /**
@@ -223,6 +210,8 @@ function clientAdvertises(
223
210
  // ---------------------------------------------------------------------------
224
211
 
225
212
  export class HostCuProxy {
213
+ /** Stable identity for this conversation-owned proxy instance. */
214
+ private readonly _sourceId = uuid();
226
215
  // CU state tracking (per-conversation)
227
216
  private _stepCount = 0;
228
217
  private _maxSteps: number;
@@ -294,6 +283,14 @@ export class HostCuProxy {
294
283
  return this._actionHistory;
295
284
  }
296
285
 
286
+ get sourceId(): string {
287
+ return this._sourceId;
288
+ }
289
+
290
+ get resetGeneration(): number {
291
+ return this._resetGeneration;
292
+ }
293
+
297
294
  // ---------------------------------------------------------------------------
298
295
  // Availability
299
296
  // ---------------------------------------------------------------------------
@@ -328,6 +325,7 @@ export class HostCuProxy {
328
325
  signal?: AbortSignal,
329
326
  targetClientId?: string,
330
327
  sourceActorPrincipalId?: string,
328
+ onValidatedDispatch?: () => void,
331
329
  ): Promise<ToolExecutionResult> {
332
330
  if (signal?.aborted) {
333
331
  return Promise.resolve({
@@ -399,6 +397,9 @@ export class HostCuProxy {
399
397
  isError: true,
400
398
  });
401
399
  }
400
+ bestEffortModeSessionTracking("computer admission", () =>
401
+ onValidatedDispatch?.(),
402
+ );
402
403
  const scopedObservation = hasCaptureTarget(input);
403
404
  if (scopedObservation) {
404
405
  this._previousAXTree = undefined;
@@ -597,6 +598,30 @@ export class HostCuProxy {
597
598
  return result;
598
599
  }
599
600
 
601
+ /** Run a screenshot-only backend through the shared CU budget and formatter. */
602
+ async executeLocal(
603
+ toolName: string,
604
+ input: Record<string, unknown>,
605
+ execute: () => Promise<CuObservationResult>,
606
+ ): Promise<ToolExecutionResult> {
607
+ if (this._stepCount >= this._maxSteps) {
608
+ return {
609
+ content: `Step limit (${this._maxSteps}) exceeded. Call computer_use_done to finish.`,
610
+ isError: true,
611
+ };
612
+ }
613
+ const reasoning =
614
+ typeof input.reasoning === "string" ? input.reasoning : undefined;
615
+ this.recordAction(toolName, input, reasoning);
616
+ this._previousAXTree = undefined;
617
+ this._consecutiveUnchangedSteps = 0;
618
+ const observation = await execute();
619
+ return this.formatObservation(observation, undefined, false, {
620
+ toolName,
621
+ input,
622
+ });
623
+ }
624
+
600
625
  // ---------------------------------------------------------------------------
601
626
  // CU state management
602
627
  // ---------------------------------------------------------------------------
@@ -629,7 +654,8 @@ export class HostCuProxy {
629
654
  * stopping and puts the pointer back where the user left it, instead of
630
655
  * waiting out its idle fallback.
631
656
  */
632
- endTask(conversationId: string): void {
657
+ endTask(conversationId: string): number {
658
+ const endedGeneration = this._resetGeneration;
633
659
  for (const targetClientId of this._dispatchedTargets.values()) {
634
660
  try {
635
661
  broadcastMessage(
@@ -647,10 +673,12 @@ export class HostCuProxy {
647
673
  }
648
674
  }
649
675
  this.reset();
676
+ return endedGeneration;
650
677
  }
651
678
 
652
679
  /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
653
- reset(): void {
680
+ reset(): number {
681
+ const endedGeneration = this._resetGeneration;
654
682
  this._dispatchedTargets.clear();
655
683
  this._stepCount = 0;
656
684
  this._previousAXTree = undefined;
@@ -658,6 +686,7 @@ export class HostCuProxy {
658
686
  this._actionHistory = [];
659
687
  this._observedTargets.clear();
660
688
  this._resetGeneration++;
689
+ return endedGeneration;
661
690
  }
662
691
 
663
692
  /**
@@ -77,7 +77,6 @@ import { writePid } from "./daemon-control.js";
77
77
  import {
78
78
  setDbMigrating,
79
79
  setDbMigrationFailed,
80
- setDbReady,
81
80
  setStartupComplete,
82
81
  } from "./daemon-readiness.js";
83
82
  import { startDiskPressureGuardForLifecycle } from "./disk-pressure-guard-lifecycle.js";
@@ -91,6 +90,7 @@ import {
91
90
  reconcileInterruptedConversations,
92
91
  resumeInterruptedConversations,
93
92
  } from "./interrupted-turn-reconciler.js";
93
+ import { recoverModeSessionsBeforeDbReady } from "./mode-session-startup-recovery.js";
94
94
  import { startOrphanReaper } from "./orphan-reaper.js";
95
95
  import { runProfilerSweep } from "./profiler-run-store.js";
96
96
  import {
@@ -290,22 +290,39 @@ export async function runDaemon(): Promise<void> {
290
290
  "stream seq floor from persisted anchors failed — continuing startup",
291
291
  );
292
292
  }
293
+ const migrationFailureDetails = {
294
+ failedMigrations: initResult.failedMigrations,
295
+ deferredMigrations: initResult.deferredMigrations,
296
+ validationError: initResult.validationError,
297
+ };
298
+ const modeSessionRecovery = migrationsOk
299
+ ? recoverModeSessionsBeforeDbReady()
300
+ : null;
301
+ if (modeSessionRecovery?.ok && modeSessionRecovery.interruptedCount > 0) {
302
+ log.info(
303
+ { interruptedModeSessions: modeSessionRecovery.interruptedCount },
304
+ "Recovered active mode sessions as interrupted",
305
+ );
306
+ }
307
+ if (modeSessionRecovery && !modeSessionRecovery.ok) {
308
+ log.error(
309
+ { err: modeSessionRecovery.error },
310
+ "Mode session recovery failed; tracking is unavailable for this boot",
311
+ );
312
+ }
293
313
  if (migrationsOk) {
294
- setDbReady(true);
295
314
  log.info("Daemon startup: DB initialized");
296
315
  } else {
297
- setDbMigrationFailed(undefined, {
298
- failedMigrations: initResult.failedMigrations,
299
- deferredMigrations: initResult.deferredMigrations,
300
- validationError: initResult.validationError,
301
- });
316
+ setDbMigrationFailed(undefined, migrationFailureDetails);
317
+ }
318
+ if (!migrationsOk) {
302
319
  log.error(
303
320
  {
304
321
  failedMigrations: initResult.failedMigrations,
305
322
  deferredMigrations: initResult.deferredMigrations,
306
323
  validationError: initResult.validationError,
307
324
  },
308
- "Daemon startup: DB opened but one or more migrations failed or were deferred — /readyz will remain unready",
325
+ "Daemon startup: DB migrations failed; /readyz will remain unready",
309
326
  );
310
327
  }
311
328
  // Migrations have settled (successfully or in the failed degraded mode),
@@ -673,11 +690,11 @@ export async function runDaemon(): Promise<void> {
673
690
  // blocked.
674
691
  startConsentRefresh();
675
692
 
676
- // Bring up the assistant's CES connection (process + handshake + reconnect
677
- // wiring). Blocks up to a 20s timeout so credential reads route through CES
678
- // before provider init; non-fatal, falls back to the direct credential store
679
- // on failure. CES serves a multi-connection bootstrap socket, so this
680
- // happens at the process level and child processes can connect independently.
693
+ // Open the assistant's CES RPC client (handshake + reconnect wiring).
694
+ // Blocks up to a 20s timeout so credential reads route through CES before
695
+ // provider init; non-fatal, falls back to the direct credential store on
696
+ // failure. CES serves a multi-connection bootstrap socket, so child
697
+ // processes can open the same `openCesRpcSession` path independently.
681
698
  await startCes(config);
682
699
 
683
700
  // Bring up the plugin layer: install the runtime bridge, register the
@@ -18,7 +18,6 @@ export * from "./message-types/shared.js";
18
18
  export * from "./message-types/skills.js";
19
19
  export * from "./message-types/surfaces.js";
20
20
  export * from "./message-types/sync.js";
21
- export * from "./message-types/web-activity.js";
22
21
 
23
22
  // Client-message domain aliases for the ClientMessage union.
24
23
  import type { _ComputerUseClientMessages } from "./message-types/computer-use.js";