@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
@@ -29,6 +29,7 @@ import {
29
29
  resolveTurnCommitWaitMs,
30
30
  } from "../daemon/abort-watchdog.js";
31
31
  import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
32
+ import type { ModeSessionSourceHandle } from "../daemon/conversation-mode-session.js";
32
33
  import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
33
34
  import { getOrCreateConversation } from "../daemon/conversation-store.js";
34
35
  import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
@@ -48,15 +49,16 @@ import {
48
49
  updateMessageContent,
49
50
  } from "../persistence/conversation-crud.js";
50
51
  import { VOICE_ESCALATION_CONTINUATION_MESSAGE_KIND } from "../plugin-api/constants.js";
51
- import { doesSupportVision } from "../plugin-api/vision-support.js";
52
+ import { dispatchProviderResolvable } from "../providers/connection-resolution.js";
52
53
  import { pinnedListeningLanguage } from "../providers/speech-to-text/provider-catalog.js";
53
- import type { ContentBlock, Message } from "../providers/types.js";
54
+ import type { ContentBlock } from "../providers/types.js";
54
55
  import { broadcastMessage } from "../runtime/assistant-event-hub.js";
55
56
  import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
56
57
  import { getCurrentSeq } from "../runtime/assistant-stream-state.js";
57
58
  import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
58
59
  import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
59
60
  import { sttCatalogKeyForRole } from "../stt/roles.js";
61
+ import type { SubagentParentNotification } from "../subagent/parent-notification.js";
60
62
  import { getAllTools } from "../tools/registry.js";
61
63
  import { sensitiveToolReach } from "../tools/tool-approval-handler.js";
62
64
  import { createAbortReason } from "../util/abort-reasons.js";
@@ -74,6 +76,8 @@ import {
74
76
  stripInternalSpeechMarkers,
75
77
  terminalControlMarkerLength,
76
78
  } from "./voice-control-protocol.js";
79
+ import { judgeEscalation } from "./voice-escalation-judge.js";
80
+ import type { VoiceEscalationTarget } from "./voice-escalation-target.js";
77
81
  import {
78
82
  createFrontDoorStreamGate,
79
83
  escalatedContinuationRule,
@@ -87,81 +91,35 @@ import {
87
91
  const log = getLogger("voice-session-bridge");
88
92
 
89
93
  /**
90
- * Profile an image-bearing voice leg is pinned to.
94
+ * The conversation's effective `mainAgent` profile, or null when nothing
95
+ * selected one and the ordinary call-agent resolution applies. Images in the
96
+ * history need no special profile: a text-only model gets them captioned by
97
+ * the `image-fallback` plugin, exactly as a typed turn does.
91
98
  *
92
- * The latency-class profile is the one voice already leans on (it fronts every
93
- * turn through `voiceFrontDoor`); `callAgent`'s `balanced` profile carries no
94
- * guarantee that its model takes images, and a model that rejects an image
95
- * fails the whole leg rather than degrading it.
96
- *
97
- * Whether THIS profile takes images is an install-level question, not a
98
- * constant: a BYO provider resolves the key through its own column of the
99
- * intent matrix, and on Fireworks that lands on a text-only model while its
100
- * `balanced` column is vision-capable. Pinning there would break the exact
101
- * turns this pin exists to save, hence the capability check at the call site.
102
- */
103
- const VOICE_IMAGE_PROFILE = "latency-optimized";
104
-
105
- /**
106
- * The profile the conversation's own text turns run on, resolved the way a
107
- * `mainAgent` turn resolves it: the conversation's pinned profile when it
108
- * carries one, else the workspace chat-model selection (`llm.activeProfile`),
109
- * else the main agent's call-site pin.
110
- *
111
- * The escalated voice leg runs through `callAgent`, whose chain never
112
- * consults `llm.activeProfile`, so without this the hand-off lands on that
113
- * site's shipped `balanced` default while the same conversation's typed
114
- * turns run on whatever the user picked. Pinning the text-turn winner keeps
115
- * the stronger model the front door escalates to the one the conversation is
116
- * already using.
117
- *
118
- * `profile` is the name to pin (a mix's own name, so dispatch re-expands it
119
- * to the same arm from the conversation seed); `modelProfile` is the concrete
120
- * profile whose model actually runs (the chosen arm of a mix), which is what
121
- * capability checks must judge: a mix reads as vision-capable when any arm
122
- * is, but only one arm serves this conversation.
123
- *
124
- * Null when nothing above named a profile (the winner is the code-owned
125
- * anchor): the leg then keeps its ordinary call-site resolution, which lands
126
- * on the same anchor intent and still honors a `callAgent` site pin.
99
+ * A mix resolves to the arm this conversation's seed selects, not the mix's
100
+ * own name. Dispatch lands on that arm either way, but the image-fallback
101
+ * check judges the profile it is handed, and a mix reads as vision-capable
102
+ * when any arm is: a text-only chosen arm would get raw images.
127
103
  */
128
104
  function conversationProfileForEscalation(
129
105
  conversation: OverrideProfileFields & { conversationId: string },
130
- ): { profile: string; modelProfile: string } | null {
106
+ ): string | null {
131
107
  const overrideProfile = resolveOverrideProfile(conversation);
132
- let chosenArm: string | undefined;
108
+ const mix: { arm?: { mixProfile: string; chosenProfile: string } } = {};
133
109
  const selection = selectWinningProfile("mainAgent", getConfig().llm, {
134
110
  ...(overrideProfile != null ? { overrideProfile } : {}),
135
111
  selectionSeed: conversation.conversationId,
136
- onMixSelected: ({ chosenProfile }) => {
137
- chosenArm = chosenProfile;
112
+ isResolvableProvider: dispatchProviderResolvable,
113
+ onMixSelected: (selected) => {
114
+ mix.arm = selected;
138
115
  },
139
116
  });
140
117
  if (selection.source === "default" || selection.profileName == null) {
141
118
  return null;
142
119
  }
143
- return {
144
- profile: selection.profileName,
145
- modelProfile: chosenArm ?? selection.profileName,
146
- };
147
- }
148
-
149
- /**
150
- * Does this conversation's history carry an image?
151
- *
152
- * Images persist inline and are re-sent on every later turn, so one photo
153
- * taken mid-call makes every remaining turn of that call an image turn -- the
154
- * check is over the whole history, not just this turn's own content.
155
- */
156
- function conversationCarriesImage(messages: readonly Message[]): boolean {
157
- return messages.some((message) =>
158
- message.content.some(
159
- (block) =>
160
- block.type === "image" ||
161
- (block.type === "tool_result" &&
162
- block.contentBlocks?.some((nested) => nested.type === "image")),
163
- ),
164
- );
120
+ return mix.arm?.mixProfile === selection.profileName
121
+ ? mix.arm.chosenProfile
122
+ : selection.profileName;
165
123
  }
166
124
 
167
125
  /**
@@ -388,9 +346,20 @@ export interface VoiceTurnCallbacks {
388
346
  tool_result?: (event: VoiceToolResultEvent) => void;
389
347
  }
390
348
 
349
+ export type { VoiceEscalationTarget } from "./voice-escalation-target.js";
350
+
391
351
  export interface VoiceTurnOptions {
352
+ /** Internal task update delivered through the call, with its original attribution. */
353
+ subagentNotification?: SubagentParentNotification;
392
354
  /** The conversation ID for this voice call's session. */
393
355
  conversationId: string;
356
+ /** Camera source captured when this voice turn was accepted. */
357
+ modeSessionSource?: ModeSessionSourceHandle;
358
+ /** Camera ownership already claimed by the accepting live session. */
359
+ preacceptedModeSession?: {
360
+ requestId: string;
361
+ source: ModeSessionSourceHandle;
362
+ };
394
363
  /** Voice session ID for scoped grant matching. Defaults to callSessionId. */
395
364
  voiceSessionId?: string;
396
365
  /** The call session ID for scoped grant matching. */
@@ -497,6 +466,8 @@ export interface VoiceTurnOptions {
497
466
  onError?: (message: string) => void;
498
467
  /** Event-name callbacks: tool activity, persisted row ids, raw stream. */
499
468
  callbacks?: VoiceTurnCallbacks;
469
+ /** Called once the escalated leg's actual target profile is resolved. */
470
+ onEscalationTargetResolved?: (target: VoiceEscalationTarget) => void;
500
471
  /**
501
472
  * Called when this turn leaves a confirmation for the user to answer instead
502
473
  * of deciding it, so the client can put the prompt where they can see it.
@@ -553,6 +524,8 @@ export interface VoiceTurnOptions {
553
524
  * fixed sentinel.
554
525
  */
555
526
  hiddenSyntheticPrompt?: boolean;
527
+ /** Original caller request resumed by a synthetic turn, used for routing. */
528
+ routingUtterance?: string;
556
529
  /**
557
530
  * Unified front-door: this leg was dispatched speculatively at a silence
558
531
  * boundary, so its decision rule includes the hold branch (leading token
@@ -586,6 +559,19 @@ export interface VoiceTurnHandle {
586
559
  * rollback — a missing discard degrades to abort-without-rollback.
587
560
  */
588
561
  discard?: () => Promise<void>;
562
+ /**
563
+ * Front-door legs only: the escalation judge's verdict on this turn,
564
+ * resolving true when the turn needs the escalated leg. Never rejects.
565
+ * Absent when the judge does not apply (other legs, synthetic prompts).
566
+ */
567
+ escalationJudgement?: Promise<boolean>;
568
+ /**
569
+ * Abort a front-door leg whose answer the escalation judge overruled. The
570
+ * caller never heard that answer, so the teardown transcript-hygiene pass
571
+ * deletes its row instead of leaving it in the history the escalated leg
572
+ * reads. The user row stays: the turn continues on the escalated leg.
573
+ */
574
+ overrule?: () => void;
589
575
  }
590
576
 
591
577
  // ---------------------------------------------------------------------------
@@ -1004,10 +990,8 @@ export async function startVoiceTurn(
1004
990
  // control markers (ASK_GUARDIAN, END_CALL, etc.) and recognize opener turns.
1005
991
  const isCallerGuardian = opts.trustContext?.trustClass === "guardian";
1006
992
 
1007
- // The front-door rule anchors on the words the model sees as the user
1008
- // turn, which for a phone sentinel is its neutral persisted form rather
1009
- // than the marker itself.
1010
- const routingLegRule = routingLegRuleFor(opts, persistedContent);
993
+ const routingUtterance = opts.routingUtterance?.trim() || persistedContent;
994
+ const routingLegRule = routingLegRuleFor(opts, routingUtterance);
1011
995
  let voiceCallControlPrompt: string | null;
1012
996
  if (opts.voiceControlPrompt === undefined) {
1013
997
  voiceCallControlPrompt = buildVoiceCallControlPrompt({
@@ -1201,7 +1185,7 @@ export async function startVoiceTurn(
1201
1185
  detachApprovalObserver = undefined;
1202
1186
  };
1203
1187
 
1204
- const requestId = uuidv7();
1188
+ const requestId = opts.preacceptedModeSession?.requestId ?? uuidv7();
1205
1189
  const turnId = crypto.randomUUID();
1206
1190
  // Ids this turn's row actually links, read back by `discardFn` so a rollback
1207
1191
  // does not take the attachments down with the row.
@@ -1219,6 +1203,7 @@ export async function startVoiceTurn(
1219
1203
  ...(turnAttachments.length > 0 ? { attachments: turnAttachments } : {}),
1220
1204
  requestId,
1221
1205
  metadata: {
1206
+ ...opts.subagentNotification?.metadata,
1222
1207
  // Durable "this turn came from an open voice session" marker; see
1223
1208
  // `isVoiceSessionUserMessage` for why the channel fields cannot carry it.
1224
1209
  voiceSessionTurn: true,
@@ -1484,6 +1469,13 @@ export async function startVoiceTurn(
1484
1469
  restoreTurnState(preInstallState);
1485
1470
  throw err;
1486
1471
  }
1472
+ if (opts.modeSessionSource && !opts.preacceptedModeSession) {
1473
+ conversation.modeSessions.claimTurn(
1474
+ requestId,
1475
+ opts.modeSessionSource,
1476
+ Date.now(),
1477
+ );
1478
+ }
1487
1479
  try {
1488
1480
  messageId = await persistTurnUserMessage();
1489
1481
  } catch (err) {
@@ -1497,6 +1489,7 @@ export async function startVoiceTurn(
1497
1489
  // this turn still owns the state it installed. Release it to
1498
1490
  // defaults, matching the agent-loop finally of a turn that ran.
1499
1491
  cleanup();
1492
+ conversation.modeSessions?.releaseTurn?.(requestId);
1500
1493
  throw err;
1501
1494
  }
1502
1495
  // A busy failure ALWAYS means a live winner holds the lock — even with
@@ -1522,6 +1515,7 @@ export async function startVoiceTurn(
1522
1515
  // The retry lost again (or failed outright) without this turn ever
1523
1516
  // running — leave the conversation exactly as the winner left it.
1524
1517
  restoreTurnState(preRetryState);
1518
+ conversation.modeSessions?.releaseTurn?.(requestId);
1525
1519
  throw retryErr;
1526
1520
  }
1527
1521
  }
@@ -1563,12 +1557,14 @@ export async function startVoiceTurn(
1563
1557
  // (JARVIS-1258). Synthetic opener/verification prompts persist a row but are
1564
1558
  // not user speech, so their echo is suppressed.
1565
1559
  if (!isSyntheticVoicePrompt) {
1560
+ const modeSession = conversation.modeSessions?.getTurnOwner?.(requestId);
1566
1561
  broadcastMessage({
1567
1562
  type: "user_message_echo",
1568
1563
  text: persistedContent,
1569
1564
  conversationId: opts.conversationId,
1570
1565
  messageId,
1571
1566
  requestId,
1567
+ ...(modeSession ? { modeSession } : {}),
1572
1568
  });
1573
1569
  // The echoed row is already durably persisted and the agent loop hasn't
1574
1570
  // started, so advance the snapshot↔stream anchor to the echo's seq — else
@@ -1797,6 +1793,8 @@ export async function startVoiceTurn(
1797
1793
  let reservedAssistantRowId: string | null = null;
1798
1794
  // Set by the handle's discard(): the whole leg must leave no trace.
1799
1795
  let discarded = false;
1796
+ // Set by the handle's overrule(): the leg's answer was never spoken.
1797
+ let overruled = false;
1800
1798
 
1801
1799
  // Verdict-first gate on the hub broadcast. A front-door leg's raw stream
1802
1800
  // carries its routing verdict, so hub subscribers (web, passive devices)
@@ -1817,12 +1815,46 @@ export async function startVoiceTurn(
1817
1815
  */
1818
1816
  const broadcastLegEvent = (msg: AssistantEvent): void => {
1819
1817
  if (frontDoorStreamGate === null || msg.type !== "assistant_text_delta") {
1820
- broadcastMessage(msg);
1818
+ emitHubEvent(msg);
1821
1819
  return;
1822
1820
  }
1823
1821
  const released = frontDoorStreamGate.push(msg.text);
1824
1822
  if (released.length > 0) {
1825
- broadcastMessage({ ...msg, text: released });
1823
+ // Answer text while the escalation judge is out: hold it, and every
1824
+ // leg event after it, until the verdict says the caller hears it.
1825
+ if (
1826
+ frontDoorStreamGate.answering &&
1827
+ !escalationJudgeSettled &&
1828
+ hubHold === null
1829
+ ) {
1830
+ hubHold = [];
1831
+ }
1832
+ emitHubEvent({ ...msg, text: released });
1833
+ }
1834
+ };
1835
+
1836
+ // Hub events held while the escalation judge decides whether the
1837
+ // front-door answer is spoken. Null when nothing is held.
1838
+ let hubHold: AssistantEvent[] | null = null;
1839
+ let escalationJudgeSettled = true;
1840
+ const emitHubEvent = (msg: AssistantEvent): void => {
1841
+ if (hubHold !== null) {
1842
+ hubHold.push(msg);
1843
+ return;
1844
+ }
1845
+ broadcastMessage(msg);
1846
+ };
1847
+ // Release held hub events once the judge settles. An overruled answer's
1848
+ // text is dropped: the caller never heard it, and its row is deleted.
1849
+ const releaseHubHold = (): void => {
1850
+ escalationJudgeSettled = true;
1851
+ const held = hubHold ?? [];
1852
+ hubHold = null;
1853
+ for (const msg of held) {
1854
+ if (overruled && msg.type === "assistant_text_delta") {
1855
+ continue;
1856
+ }
1857
+ broadcastMessage(msg);
1826
1858
  }
1827
1859
  };
1828
1860
 
@@ -1879,6 +1911,9 @@ export async function startVoiceTurn(
1879
1911
  if (discarded) {
1880
1912
  deleteMessageById(reservedAssistantRowId);
1881
1913
  action = "delete_discarded";
1914
+ } else if (overruled) {
1915
+ deleteMessageById(reservedAssistantRowId);
1916
+ action = "delete_overruled";
1882
1917
  } else {
1883
1918
  const row = getMessageById(reservedAssistantRowId, opts.conversationId);
1884
1919
  const terminalMarkerLength = row
@@ -1959,6 +1994,46 @@ export async function startVoiceTurn(
1959
1994
  }
1960
1995
  };
1961
1996
 
1997
+ // The escalation judge runs beside the front-door leg's model call, so its
1998
+ // verdict is usually in before the leg's first answer word. Snapshot the
1999
+ // history now: the leg's own reply must not be part of what is judged.
2000
+ // Hidden fresh-view follow-ups route with their image; this text-only judge
2001
+ // cannot assess that context.
2002
+ const escalationJudgement =
2003
+ opts.routingLeg === "front-door" && !isHiddenSyntheticPrompt
2004
+ ? judgeEscalation({
2005
+ conversationId: opts.conversationId,
2006
+ history: conversation.getMessages(),
2007
+ utterance: opts.content,
2008
+ ...(opts.signal ? { signal: opts.signal } : {}),
2009
+ }).then((judgement) => {
2010
+ if (judgement.outcome !== "unavailable") {
2011
+ log.info(
2012
+ {
2013
+ turnId,
2014
+ outcome: judgement.outcome,
2015
+ noul: judgement.noul,
2016
+ latencyMs: judgement.latencyMs,
2017
+ },
2018
+ "Voice escalation judge verdict",
2019
+ );
2020
+ }
2021
+ return judgement.escalate;
2022
+ })
2023
+ : undefined;
2024
+ if (escalationJudgement !== undefined) {
2025
+ escalationJudgeSettled = false;
2026
+ // A clear verdict releases at once; an escalate verdict waits a macrotask
2027
+ // so the driver's overrule, reacting to the same promise, lands first.
2028
+ void escalationJudgement.then((escalate) => {
2029
+ if (escalate) {
2030
+ setTimeout(releaseHubHold, 0);
2031
+ } else {
2032
+ releaseHubHold();
2033
+ }
2034
+ });
2035
+ }
2036
+
1962
2037
  // Fire-and-forget the agent loop
1963
2038
  void (async () => {
1964
2039
  const loopEnterAt = Date.now();
@@ -2000,47 +2075,26 @@ export async function startVoiceTurn(
2000
2075
  conversation.toolsDisabledDepth++;
2001
2076
  frontDoorToolsSuppressed = true;
2002
2077
  }
2003
- // An escalated leg follows the conversation's own model: the front
2004
- // door hands off to the profile the caller's typed turns already run
2005
- // on, not to `callAgent`'s shipped default. Null keeps the ordinary
2006
- // call-site resolution.
2078
+ // An escalated leg follows the conversation's effective main-agent
2079
+ // target, including direct call-site tuning and the concrete arm of a
2080
+ // mix. The semantic call site remains `callAgent` below so voice tool
2081
+ // and delivery behavior do not change.
2007
2082
  const conversationProfile =
2008
2083
  opts.routingLeg === "escalated"
2009
2084
  ? conversationProfileForEscalation(conversation)
2010
2085
  : null;
2011
- // Resolved once here rather than inside the options literal below, so
2012
- // the history scan happens once per leg. A front-door leg is skipped:
2013
- // its own call site already resolves to the same profile. A
2014
- // conversation profile whose model takes images needs no image pin
2015
- // either; one that does not yields to the image pin, since a model
2016
- // that rejects an image fails the whole leg. The judged profile is the
2017
- // concrete arm that serves this conversation, not a mix's name. The
2018
- // capability checks come before the scan because they are the cheaper
2019
- // of the two and they decide whether the pin is worth anything at all.
2020
- const needsImagePin =
2021
- opts.routingLeg !== "front-door" &&
2022
- !(
2023
- conversationProfile != null &&
2024
- doesSupportVision(conversationProfile.modelProfile)
2025
- ) &&
2026
- doesSupportVision(VOICE_IMAGE_PROFILE) &&
2027
- conversationCarriesImage(conversation.getMessages());
2028
- if (needsImagePin) {
2029
- log.info(
2030
- { turnId, routingLeg: opts.routingLeg ?? null },
2031
- "Voice leg carries an image; pinning the image-capable profile",
2032
- );
2033
- } else if (conversationProfile != null) {
2086
+ if (conversationProfile != null) {
2034
2087
  log.info(
2035
- { turnId, profile: conversationProfile.profile },
2088
+ { turnId, profile: conversationProfile },
2036
2089
  "Escalated voice leg pinned to the conversation's own profile",
2037
2090
  );
2038
2091
  }
2039
- const profilePin =
2040
- opts.overrideProfile ??
2041
- (needsImagePin
2042
- ? VOICE_IMAGE_PROFILE
2043
- : (conversationProfile?.profile ?? null));
2092
+ const profilePin = opts.overrideProfile ?? conversationProfile;
2093
+ // Optional cache traffic must not consume the last admitted request.
2094
+ // A configured cap reserves its whole budget for user-visible calls.
2095
+ const shouldWarmEscalation =
2096
+ opts.routingLeg === "escalated" &&
2097
+ config.rateLimit.maxRequestsPerMinute === 0;
2044
2098
  if (opts.macosDesktopSession === true && !frontDoorToolsSuppressed) {
2045
2099
  const sourceInterface = turnInterfaceContext.userMessageInterface;
2046
2100
  const sourceActorPrincipalId =
@@ -2056,6 +2110,9 @@ export async function startVoiceTurn(
2056
2110
  );
2057
2111
  }
2058
2112
  await conversation.runAgentLoop(persistedContent, messageId, {
2113
+ ...(opts.subagentNotification?.cronRunId
2114
+ ? { cronRunId: opts.subagentNotification.cronRunId }
2115
+ : {}),
2059
2116
  onEvent: (msg: AssistantEvent) => {
2060
2117
  if (msg.type === "assistant_turn_start") {
2061
2118
  reservedAssistantRowId = msg.messageId;
@@ -2123,6 +2180,9 @@ export async function startVoiceTurn(
2123
2180
  // ordinary call-agent resolution.
2124
2181
  callSite:
2125
2182
  opts.routingLeg === "front-door" ? "voiceFrontDoor" : "callAgent",
2183
+ ...(opts.routingLeg === "escalated"
2184
+ ? { inferenceCallSite: "mainAgent" as const }
2185
+ : {}),
2126
2186
  // A caller is on the line, so the turn is interactive: approval prompts
2127
2187
  // must be raised rather than pre-denied, because the approval observer
2128
2188
  // above is what decides them (auto-resolve for a non-guardian caller,
@@ -2138,18 +2198,83 @@ export async function startVoiceTurn(
2138
2198
  : {}),
2139
2199
  // Triage-and-escalate routing pins this turn to the fast front-door
2140
2200
  // profile or to the conversation's own profile for the escalated
2141
- // leg. `forceOverrideProfile` floats it above the callAgent call-site
2142
- // layers (callAgent is not `mainAgent`, so the override would
2143
- // otherwise sit below the call-site profile).
2201
+ // leg. `forceOverrideProfile` preserves an explicit routing pin.
2144
2202
  //
2145
- // An explicit routing pin wins; failing that, a leg whose history
2146
- // carries an image is pinned to a profile whose model takes one;
2147
- // failing that, an escalated leg is pinned to the conversation's
2148
- // profile. A front-door leg needs none of these: its own call site
2149
- // already resolves there.
2203
+ // An explicit routing pin wins; failing that, an escalated leg is
2204
+ // pinned to the conversation's profile. A front-door leg needs
2205
+ // neither: its own call site already resolves there.
2150
2206
  ...(profilePin != null
2151
2207
  ? { overrideProfile: profilePin, forceOverrideProfile: true }
2152
2208
  : {}),
2209
+ ...(opts.routingLeg === "escalated"
2210
+ ? {
2211
+ // Warming and diagnostics use the final post-hook request. The
2212
+ // observer stays synchronous so neither delays provider dispatch.
2213
+ onFirstModelCallPrepared: (prepared) => {
2214
+ if (shouldWarmEscalation && !prepared.disableCache) {
2215
+ void conversation.warmPromptCache({
2216
+ callSite: prepared.callSite ?? "mainAgent",
2217
+ ...(prepared.overrideProfile !== undefined
2218
+ ? { overrideProfile: prepared.overrideProfile }
2219
+ : {}),
2220
+ forceOverrideProfile: prepared.forceOverrideProfile,
2221
+ signal: prepared.signal ?? opts.signal,
2222
+ systemPrompt: prepared.systemPrompt,
2223
+ tools: prepared.tools,
2224
+ });
2225
+ }
2226
+ let selectedMixArm:
2227
+ | { mixProfile: string; chosenProfile: string }
2228
+ | undefined;
2229
+ const escalationSelection = selectWinningProfile(
2230
+ prepared.callSite ?? "mainAgent",
2231
+ config.llm,
2232
+ {
2233
+ ...(prepared.overrideProfile !== undefined
2234
+ ? { overrideProfile: prepared.overrideProfile }
2235
+ : {}),
2236
+ forceOverrideProfile: prepared.forceOverrideProfile,
2237
+ selectionSeed: conversation.conversationId,
2238
+ isResolvableProvider: dispatchProviderResolvable,
2239
+ onMixSelected: (selection) => {
2240
+ selectedMixArm = selection;
2241
+ },
2242
+ },
2243
+ );
2244
+ let source: VoiceEscalationTarget["source"] = "call_site";
2245
+ if (escalationSelection.source === "override") {
2246
+ if (prepared.overrideProfile !== profilePin) {
2247
+ source = "pre_model_hook";
2248
+ } else if (opts.overrideProfile != null) {
2249
+ source = "turn_override";
2250
+ } else {
2251
+ source = "conversation";
2252
+ }
2253
+ }
2254
+ const target: VoiceEscalationTarget = {
2255
+ profile:
2256
+ (selectedMixArm?.mixProfile ===
2257
+ escalationSelection.profileName
2258
+ ? selectedMixArm.chosenProfile
2259
+ : escalationSelection.profileName) ?? "balanced",
2260
+ source,
2261
+ };
2262
+ try {
2263
+ opts.onEscalationTargetResolved?.(target);
2264
+ } catch (err) {
2265
+ log.warn(
2266
+ {
2267
+ err,
2268
+ turnId,
2269
+ profile: target.profile,
2270
+ source: target.source,
2271
+ },
2272
+ "Voice escalation target callback failed",
2273
+ );
2274
+ }
2275
+ },
2276
+ }
2277
+ : {}),
2153
2278
  });
2154
2279
  if (lastError) {
2155
2280
  log.error(
@@ -2166,6 +2291,17 @@ export async function startVoiceTurn(
2166
2291
  conversation.toolsDisabledDepth--;
2167
2292
  }
2168
2293
  cleanup();
2294
+ // A judge verdict that lands after the model finished can still
2295
+ // overrule the answer. Wait for it (bounded by the judge's own budget)
2296
+ // so the overrule's row cleanup runs here, before the escalated leg,
2297
+ // blocked on this teardown, reads history.
2298
+ if (
2299
+ escalationJudgement !== undefined &&
2300
+ !discarded &&
2301
+ (await escalationJudgement)
2302
+ ) {
2303
+ await new Promise<void>((resolve) => setTimeout(resolve, 0));
2304
+ }
2169
2305
  await finalizeVoiceLegTranscript();
2170
2306
  settleTurnTeardown();
2171
2307
  }
@@ -2225,9 +2361,17 @@ export async function startVoiceTurn(
2225
2361
  }
2226
2362
  };
2227
2363
 
2364
+ const overruleFn = () => {
2365
+ overruled = true;
2366
+ abortFn();
2367
+ };
2368
+
2228
2369
  return {
2229
2370
  turnId,
2230
2371
  abort: abortFn,
2231
2372
  discard: discardFn,
2373
+ ...(escalationJudgement !== undefined
2374
+ ? { escalationJudgement, overrule: overruleFn }
2375
+ : {}),
2232
2376
  };
2233
2377
  }
@@ -401,6 +401,8 @@ export function createFrontDoorVerdictMachine(
401
401
  export interface FrontDoorStreamGate {
402
402
  push(deltaText: string): string;
403
403
  finish(): string;
404
+ /** Whether the leg's verdict classified its output as the answer. */
405
+ readonly answering: boolean;
404
406
  }
405
407
 
406
408
  /**
@@ -424,11 +426,13 @@ export function createFrontDoorStreamGate(
424
426
  holdEnabled: boolean,
425
427
  ): FrontDoorStreamGate {
426
428
  const machine = createFrontDoorVerdictMachine(holdEnabled);
429
+ let answering = false;
427
430
  const releasable = (bridge: string): string =>
428
431
  bridge.length < MIN_SPOKEN_BRIDGE_CHARS ? "" : bridge;
429
432
  const release = (step: FrontDoorStep): string => {
430
433
  switch (step.kind) {
431
434
  case "answer":
435
+ answering = true;
432
436
  return step.text;
433
437
  case "escalate":
434
438
  return step.bridge === null ? "" : releasable(step.bridge);
@@ -441,6 +445,9 @@ export function createFrontDoorStreamGate(
441
445
  return {
442
446
  push: (deltaText) => release(machine.push(deltaText)),
443
447
  finish: () => release(machine.finish()),
448
+ get answering() {
449
+ return answering;
450
+ },
444
451
  };
445
452
  }
446
453