@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
@@ -3,6 +3,8 @@ import { join } from "node:path";
3
3
  import { afterAll, beforeAll, describe, expect, mock, test } from "bun:test";
4
4
 
5
5
  import type { TurnDetectorConfig } from "../../calls/media-turn-detector.js";
6
+ import type { VoiceProgressNarrator } from "../../calls/progress-narration.js";
7
+ import { TASK_UPDATE_SILENT_MARKER } from "../../calls/voice-control-protocol.js";
6
8
  import type {
7
9
  VoiceTurnCallbacks,
8
10
  VoiceTurnOptions,
@@ -13,14 +15,15 @@ import {
13
15
  saveRawConfig,
14
16
  } from "../../config/loader.js";
15
17
  import type { VoiceFrontModelConfig } from "../../config/schemas/voice.js";
18
+ import type {
19
+ ConversationModeSessionCoordinator,
20
+ ModeSessionSourceHandle,
21
+ } from "../../daemon/conversation-mode-session.js";
22
+ import { RiskLevel } from "../../permissions/types.js";
16
23
  import type {
17
24
  StreamingTranscriber,
18
25
  SttStreamServerEvent,
19
26
  } from "../../stt/types.js";
20
- import {
21
- computerUseKeyTool,
22
- computerUseObserveTool,
23
- } from "../../tools/computer-use/definitions.js";
24
27
  import {
25
28
  __resetRegistryForTesting,
26
29
  registerSkillTools,
@@ -28,6 +31,7 @@ import {
28
31
  } from "../../tools/registry.js";
29
32
  import { finalizeTool } from "../../tools/tool-defaults.js";
30
33
  import { getWorkspaceSkillsDir } from "../../util/platform.js";
34
+ import type { LiveVoiceContinuationJudge } from "../continuation-judge.js";
31
35
  import type { LiveVoiceContinuationLabeler } from "../continuation-label.js";
32
36
  import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
33
37
  import {
@@ -36,6 +40,7 @@ import {
36
40
  type LiveVoiceBackgroundContinuationSpawner,
37
41
  LiveVoiceSession,
38
42
  type LiveVoiceSessionAudioArchiver,
43
+ type LiveVoiceSessionOptions,
39
44
  type LiveVoiceTtsStreamer,
40
45
  type LiveVoiceTurnStarter,
41
46
  } from "../live-voice-session.js";
@@ -58,7 +63,12 @@ import {
58
63
  // `mock.module` is process-global in Bun and leaks into sibling files that run
59
64
  // in the same process — run this file on its own (`bun test <path>`).
60
65
  const injectMessageIntoParentMock = mock(
61
- (_parentConversationId: string, _message: string) => {},
66
+ (
67
+ _parentConversationId: string,
68
+ _message: string,
69
+ _metadata?: Record<string, unknown>,
70
+ _opts?: { cronRunId?: string | null; bypassLiveVoice?: boolean },
71
+ ) => {},
62
72
  );
63
73
  mock.module("../../subagent/notify.js", () => ({
64
74
  injectMessageIntoParent: injectMessageIntoParentMock,
@@ -208,12 +218,17 @@ function createHarness(options: {
208
218
  viaFactory?: boolean;
209
219
  spawnBackgroundContinuation?: LiveVoiceBackgroundContinuationSpawner;
210
220
  labelBackgroundContinuation?: LiveVoiceContinuationLabeler;
221
+ judgeBackgroundContinuation?: LiveVoiceContinuationJudge;
211
222
  getTurnTeardown?: (conversationId: string) => Promise<void> | undefined;
212
223
  detachTeardownSettleTimeoutMs?: number;
213
224
  continuationAnnounceSilenceMs?: number;
225
+ progressNarrator?: VoiceProgressNarrator;
214
226
  foregroundTaskResumeSilenceMs?: number;
215
227
  foregroundTaskMaxSuspendedMs?: number;
216
228
  foregroundTaskMaxInterveningTurns?: number;
229
+ acquireModeSessionResidency?: NonNullable<
230
+ LiveVoiceSessionOptions["acquireModeSessionResidency"]
231
+ >;
217
232
  }) {
218
233
  const sequencer = createLiveVoiceServerFrameSequencer();
219
234
  const frames: LiveVoiceServerFrame[] = [];
@@ -259,6 +274,7 @@ function createHarness(options: {
259
274
  options.startVoiceTurn ??
260
275
  mock(async () => ({ turnId: "bridge-turn", abort: mock() })),
261
276
  streamTtsAudio: options.streamTtsAudio ?? null,
277
+ progressNarrator: options.progressNarrator,
262
278
  archiveAudio: options.archiveAudio ?? null,
263
279
  emitMetrics: options.emitMetrics ?? false,
264
280
  ...(options.metricsClock ? { metricsClock: options.metricsClock } : {}),
@@ -288,6 +304,9 @@ function createHarness(options: {
288
304
  ...(options.spawnBackgroundContinuation
289
305
  ? { spawnBackgroundContinuation: options.spawnBackgroundContinuation }
290
306
  : {}),
307
+ ...(options.judgeBackgroundContinuation
308
+ ? { judgeBackgroundContinuation: options.judgeBackgroundContinuation }
309
+ : {}),
291
310
  ...(options.labelBackgroundContinuation
292
311
  ? { labelBackgroundContinuation: options.labelBackgroundContinuation }
293
312
  : {}),
@@ -314,6 +333,9 @@ function createHarness(options: {
314
333
  options.foregroundTaskMaxInterveningTurns,
315
334
  }
316
335
  : {}),
336
+ ...(options.acquireModeSessionResidency
337
+ ? { acquireModeSessionResidency: options.acquireModeSessionResidency }
338
+ : {}),
317
339
  };
318
340
  const session = options.viaFactory
319
341
  ? createLiveVoiceSession(context, {
@@ -514,6 +536,7 @@ async function startForegroundTaskBargeInScenario(options?: {
514
536
  skillExecuteInput?: Record<string, unknown>;
515
537
  skillExecuteAllowedToolNames?: ReadonlySet<string>;
516
538
  startHostTool?: boolean;
539
+ judgeBackgroundContinuation?: LiveVoiceContinuationJudge;
517
540
  }): Promise<{
518
541
  calls: VoiceTurnOptions[];
519
542
  frames: LiveVoiceServerFrame[];
@@ -538,6 +561,9 @@ async function startForegroundTaskBargeInScenario(options?: {
538
561
  return makeTtsResult("assistant audio");
539
562
  }),
540
563
  spawnBackgroundContinuation,
564
+ ...(options?.judgeBackgroundContinuation
565
+ ? { judgeBackgroundContinuation: options.judgeBackgroundContinuation }
566
+ : {}),
541
567
  foregroundTaskResumeSilenceMs: options?.foregroundTaskResumeSilenceMs ?? 20,
542
568
  ...(options?.foregroundTaskMaxInterveningTurns !== undefined
543
569
  ? {
@@ -576,17 +602,550 @@ async function startForegroundTaskBargeInScenario(options?: {
576
602
  return { calls, frames, session, spawnBackgroundContinuation };
577
603
  }
578
604
 
605
+ describe("LiveVoiceSession subagent outcomes", () => {
606
+ function outcome(subagentId: string, status = "completed") {
607
+ return {
608
+ taskId: subagentId,
609
+ message: `Task ${subagentId} ${status}`,
610
+ metadata: {
611
+ subagentNotification: { subagentId, label: subagentId, status },
612
+ },
613
+ cronRunId: "run-123",
614
+ };
615
+ }
616
+
617
+ function immediateTts(): LiveVoiceTtsStreamer {
618
+ return async (options) => {
619
+ options.onAudioChunk(makeTtsChunk("assistant audio"));
620
+ return makeTtsResult("assistant audio");
621
+ };
622
+ }
623
+
624
+ test("speaks independent outcomes through hidden tool-capable turns", async () => {
625
+ const { calls, startVoiceTurn } = makeAutoCompletingTurnStarter([
626
+ "The comparison is ready.",
627
+ "The export failed. It needs a connection.",
628
+ ]);
629
+ const { session, frames } = createHarness({
630
+ startVoiceTurn,
631
+ streamTtsAudio: immediateTts(),
632
+ continuationAnnounceSilenceMs: 20,
633
+ });
634
+ const before = injectMessageIntoParentMock.mock.calls.length;
635
+ await session.start();
636
+ try {
637
+ session.receiveSubagentNotification(outcome("task-1"));
638
+ session.receiveSubagentNotification(outcome("task-2", "failed"));
639
+ await waitFor(() => countType(frames, "tts_done") === 2);
640
+ await new Promise((resolve) => setTimeout(resolve, 50));
641
+ expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
642
+ "task-1",
643
+ "task-2",
644
+ ]);
645
+ for (const call of calls) {
646
+ expect(call.hiddenSyntheticPrompt).toBe(true);
647
+ expect(call.directEscalated).toBe(true);
648
+ expect(call.subagentNotification?.cronRunId).toBe("run-123");
649
+ expect(call.voiceControlPrompt).toContain(
650
+ "internal background task update",
651
+ );
652
+ }
653
+ expect(countType(frames, "tts_audio")).toBeGreaterThanOrEqual(2);
654
+ } finally {
655
+ await session.close("client_end");
656
+ }
657
+ expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
658
+ });
659
+
660
+ test("silently consumes a redundant update and still speaks the next useful result", async () => {
661
+ const { calls, startVoiceTurn } = makeAutoCompletingTurnStarter([
662
+ TASK_UPDATE_SILENT_MARKER,
663
+ "The export is ready.",
664
+ ]);
665
+ const { session, frames } = createHarness({
666
+ startVoiceTurn,
667
+ streamTtsAudio: immediateTts(),
668
+ continuationAnnounceSilenceMs: 10,
669
+ });
670
+ const before = injectMessageIntoParentMock.mock.calls.length;
671
+ await session.start();
672
+ try {
673
+ session.receiveSubagentNotification(outcome("task-1", "running"));
674
+ await waitFor(() => countType(frames, "tts_done") === 1);
675
+ expect(countType(frames, "tts_audio")).toBe(0);
676
+ expect(countType(frames, "assistant_text_delta")).toBe(0);
677
+ session.receiveSubagentNotification(outcome("task-2"));
678
+ await waitFor(() => countType(frames, "tts_done") === 2);
679
+ await new Promise((resolve) => setTimeout(resolve, 30));
680
+ expect(calls).toHaveLength(2);
681
+ expect(countType(frames, "tts_audio")).toBe(1);
682
+ } finally {
683
+ await session.close("client_end");
684
+ }
685
+ expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
686
+ });
687
+
688
+ test("an interrupted silent decision leaves the update pending", async () => {
689
+ let callbacks: VoiceTurnCallbacks | undefined;
690
+ const { session } = createHarness({
691
+ startVoiceTurn: async (turn) => {
692
+ callbacks = turn.callbacks;
693
+ return { turnId: "turn-1", abort: mock() };
694
+ },
695
+ streamTtsAudio: immediateTts(),
696
+ continuationAnnounceSilenceMs: 10,
697
+ });
698
+ const first = outcome("task-1");
699
+ const before = injectMessageIntoParentMock.mock.calls.length;
700
+ await session.start();
701
+ session.receiveSubagentNotification(first);
702
+ await waitFor(() => callbacks !== undefined);
703
+ callbacks?.assistant_text_delta?.(makeTextDelta(TASK_UPDATE_SILENT_MARKER));
704
+ await session.handleClientFrame({ type: "interrupt" });
705
+ callbacks?.message_complete?.(makeMessageComplete());
706
+ await session.close("client_end");
707
+ expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
708
+ [
709
+ "conversation-123",
710
+ first.message,
711
+ first.metadata,
712
+ { cronRunId: "run-123", bypassLiveVoice: true },
713
+ ],
714
+ ]);
715
+ });
716
+
717
+ test("waits through user speech and the current reply before announcing", async () => {
718
+ const calls: VoiceTurnOptions[] = [];
719
+ const { session } = createHarness({
720
+ finals: ["explain this setting"],
721
+ continuationAnnounceSilenceMs: 10,
722
+ streamTtsAudio: immediateTts(),
723
+ startVoiceTurn: async (turn) => {
724
+ calls.push(turn);
725
+ return { turnId: `turn-${calls.length}`, abort: mock() };
726
+ },
727
+ });
728
+ await session.start();
729
+ try {
730
+ await session.handleBinaryAudio(LOUD_CHUNK);
731
+ session.receiveSubagentNotification(outcome("task-1"));
732
+ await waitFor(() => calls.length === 1);
733
+ expect(calls[0]?.content).toBe("explain this setting");
734
+ await new Promise((resolve) => setTimeout(resolve, 40));
735
+ expect(calls).toHaveLength(1);
736
+ calls[0]?.callbacks?.assistant_text_delta?.(
737
+ makeTextDelta("This is the setting."),
738
+ );
739
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
740
+ await waitFor(() => calls.length === 2);
741
+ expect(calls[1]?.subagentNotification?.taskId).toBe("task-1");
742
+ } finally {
743
+ await session.close("client_end");
744
+ }
745
+ });
746
+
747
+ test.each(["what does this setting mean", ""])(
748
+ "a barge-in (%j) retains both outcomes without spawning another worker",
749
+ async (utterance) => {
750
+ const calls: VoiceTurnOptions[] = [];
751
+ const spawnBackgroundContinuation = mock(
752
+ async () => "unexpected continuation",
753
+ );
754
+ const { session, frames } = createHarness({
755
+ finals: [utterance],
756
+ continuationAnnounceSilenceMs: 10,
757
+ streamTtsAudio: immediateTts(),
758
+ spawnBackgroundContinuation,
759
+ startVoiceTurn: async (turn) => {
760
+ calls.push(turn);
761
+ if (calls.length > 1) {
762
+ turn.callbacks?.assistant_text_delta?.(
763
+ makeTextDelta("Here is the answer."),
764
+ );
765
+ turn.callbacks?.message_complete?.(makeMessageComplete());
766
+ }
767
+ return { turnId: `turn-${calls.length}`, abort: mock() };
768
+ },
769
+ });
770
+ await session.start();
771
+ try {
772
+ session.receiveSubagentNotification(outcome("task-1"));
773
+ session.receiveSubagentNotification(outcome("task-2"));
774
+ await waitFor(() => calls.length === 1);
775
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
776
+ await waitFor(
777
+ () => countType(frames, "tts_done") === (utterance ? 3 : 2),
778
+ );
779
+ expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
780
+ "task-1",
781
+ ...(utterance ? [undefined] : []),
782
+ "task-1",
783
+ "task-2",
784
+ ]);
785
+ expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
786
+ } finally {
787
+ await session.close("client_end");
788
+ }
789
+ },
790
+ );
791
+
792
+ test("a completion replacing interrupted findings carries the delivery receipt into its retry", async () => {
793
+ const calls: VoiceTurnOptions[] = [];
794
+ const { session, frames } = createHarness({
795
+ finals: ["tell me about birds"],
796
+ continuationAnnounceSilenceMs: 10,
797
+ streamTtsAudio: immediateTts(),
798
+ startVoiceTurn: async (turn) => {
799
+ calls.push(turn);
800
+ return { turnId: `turn-${calls.length}`, abort: mock() };
801
+ },
802
+ });
803
+ await session.start();
804
+ try {
805
+ session.receiveSubagentNotification(outcome("task-1", "running"));
806
+ await waitFor(() => calls.length === 1);
807
+ calls[0]?.callbacks?.assistant_text_delta?.(
808
+ makeTextDelta("Here are all the generated findings."),
809
+ );
810
+ await waitFor(() => countType(frames, "tts_audio") > 0);
811
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
812
+ await waitFor(() => calls.length === 2);
813
+ session.receiveSubagentNotification(outcome("task-1", "completed"));
814
+ calls[1]?.callbacks?.assistant_text_delta?.(
815
+ makeTextDelta("Birds have feathers."),
816
+ );
817
+ calls[1]?.callbacks?.message_complete?.(makeMessageComplete());
818
+ await waitFor(() => calls.length === 3);
819
+ expect(calls[2]?.subagentNotification?.message).toBe(
820
+ "Task task-1 completed",
821
+ );
822
+ expect(calls[2]?.voiceControlPrompt).toContain(
823
+ "interrupted before playback completed",
824
+ );
825
+ expect(calls[2]?.voiceControlPrompt).toContain("Birds have feathers.");
826
+ expect(calls[2]?.voiceControlPrompt).not.toContain(
827
+ "Here are all the generated findings.",
828
+ );
829
+ } finally {
830
+ await session.close("client_end");
831
+ }
832
+ });
833
+
834
+ test("hang-up returns every undelivered outcome with its metadata and attribution", async () => {
835
+ const { session } = createHarness({ continuationAnnounceSilenceMs: 1_000 });
836
+ await session.start();
837
+ const first = outcome("task-1");
838
+ const second = outcome("task-2", "running");
839
+ const before = injectMessageIntoParentMock.mock.calls.length;
840
+ session.receiveSubagentNotification(first);
841
+ session.receiveSubagentNotification(second);
842
+ await session.close("client_end");
843
+ await session.close("client_end");
844
+ expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
845
+ [
846
+ "conversation-123",
847
+ first.message,
848
+ first.metadata,
849
+ { cronRunId: "run-123", bypassLiveVoice: true },
850
+ ],
851
+ [
852
+ "conversation-123",
853
+ second.message,
854
+ second.metadata,
855
+ { cronRunId: "run-123", bypassLiveVoice: true },
856
+ ],
857
+ ]);
858
+ expect(session.receiveSubagentNotification(outcome("task-3"))).toBe(false);
859
+ });
860
+
861
+ test("hang-up hands back results only after the voice turn's queue-clearing abort settles", async () => {
862
+ let finishTeardown!: () => void;
863
+ const teardown = new Promise<void>((resolve) => {
864
+ finishTeardown = resolve;
865
+ });
866
+ const abort = mock(() => {});
867
+ const startVoiceTurn = mock(async () => ({ turnId: "turn-1", abort }));
868
+ const { session } = createHarness({
869
+ startVoiceTurn,
870
+ getTurnTeardown: () => teardown,
871
+ continuationAnnounceSilenceMs: 10,
872
+ });
873
+ await session.start();
874
+ const first = outcome("task-1");
875
+ session.receiveSubagentNotification(first);
876
+ await waitFor(() => startVoiceTurn.mock.calls.length === 1);
877
+ const before = injectMessageIntoParentMock.mock.calls.length;
878
+ const closing = session.close("client_end");
879
+ const duringClose = outcome("task-2");
880
+ expect(session.receiveSubagentNotification(duringClose)).toBe(true);
881
+ await waitFor(() => abort.mock.calls.length > 0);
882
+ const duringTeardown = outcome("task-3");
883
+ expect(session.receiveSubagentNotification(duringTeardown)).toBe(true);
884
+ expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
885
+ finishTeardown();
886
+ await closing;
887
+ expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual(
888
+ [first, duringClose, duringTeardown].map((notification) => [
889
+ "conversation-123",
890
+ notification.message,
891
+ notification.metadata,
892
+ { cronRunId: "run-123", bypassLiveVoice: true },
893
+ ]),
894
+ );
895
+ });
896
+
897
+ test.each(["resolved", "rejected"])(
898
+ "hang-up waits for pending startup (%s) before handing back results",
899
+ async (mode) => {
900
+ let finishStartup!: () => void;
901
+ const startup = new Promise<void>((resolve) => {
902
+ finishStartup = resolve;
903
+ });
904
+ let finishTeardown!: () => void;
905
+ const teardown = new Promise<void>((resolve) => {
906
+ finishTeardown = resolve;
907
+ });
908
+ let registeredTeardown: Promise<void> | undefined;
909
+ let turn: VoiceTurnOptions | undefined;
910
+ const abort = mock(() => {});
911
+ const { session } = createHarness({
912
+ startVoiceTurn: async (options) => {
913
+ turn = options;
914
+ await startup;
915
+ if (mode === "rejected") {
916
+ throw new Error("startup unavailable");
917
+ }
918
+ registeredTeardown = teardown;
919
+ return { turnId: "turn-1", abort };
920
+ },
921
+ getTurnTeardown: () => registeredTeardown,
922
+ continuationAnnounceSilenceMs: 10,
923
+ });
924
+ await session.start();
925
+ const first = outcome("task-1");
926
+ const before = injectMessageIntoParentMock.mock.calls.length;
927
+ session.receiveSubagentNotification(first);
928
+ await waitFor(() => turn !== undefined);
929
+ const closing = session.close("client_end");
930
+ await waitFor(() => turn?.signal?.aborted === true);
931
+ expect(registeredTeardown).toBeUndefined();
932
+ expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
933
+ finishStartup();
934
+ if (mode === "resolved") {
935
+ await waitFor(() => abort.mock.calls.length > 0);
936
+ expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
937
+ }
938
+ finishTeardown();
939
+ await closing;
940
+ expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
941
+ [
942
+ "conversation-123",
943
+ first.message,
944
+ first.metadata,
945
+ { cronRunId: "run-123", bypassLiveVoice: true },
946
+ ],
947
+ ]);
948
+ },
949
+ );
950
+
951
+ test("a failed announcement waits for a user turn before retrying", async () => {
952
+ const calls: VoiceTurnOptions[] = [];
953
+ const { session, frames } = createHarness({
954
+ finals: ["are you still there"],
955
+ continuationAnnounceSilenceMs: 10,
956
+ streamTtsAudio: immediateTts(),
957
+ startVoiceTurn: async (turn) => {
958
+ calls.push(turn);
959
+ if (calls.length === 1) {
960
+ throw new Error("conversation busy");
961
+ }
962
+ turn.callbacks?.assistant_text_delta?.(
963
+ makeTextDelta("Here is the answer."),
964
+ );
965
+ turn.callbacks?.message_complete?.(makeMessageComplete());
966
+ return { turnId: `turn-${calls.length}`, abort: mock() };
967
+ },
968
+ });
969
+ await session.start();
970
+ try {
971
+ session.receiveSubagentNotification(outcome("task-1"));
972
+ await waitFor(() => calls.length === 1);
973
+ await new Promise((resolve) => setTimeout(resolve, 60));
974
+ expect(calls).toHaveLength(1);
975
+ await session.handleBinaryAudio(LOUD_CHUNK);
976
+ await waitFor(() => countType(frames, "tts_done") === 2);
977
+ expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
978
+ "task-1",
979
+ undefined,
980
+ "task-1",
981
+ ]);
982
+ } finally {
983
+ await session.close("client_end");
984
+ }
985
+ });
986
+
987
+ test.each(["throws", "empty", "unspeakable"])(
988
+ "speech without delivered audio (%s) keeps the result for hang-up delivery",
989
+ async (mode) => {
990
+ const { startVoiceTurn, calls } = makeAutoCompletingTurnStarter([
991
+ mode === "unspeakable" ? "" : "The result is ready.",
992
+ ]);
993
+ const { session, frames } = createHarness({
994
+ startVoiceTurn,
995
+ continuationAnnounceSilenceMs: 10,
996
+ streamTtsAudio: async () => {
997
+ if (mode === "throws") {
998
+ throw new Error("speech unavailable");
999
+ }
1000
+ return { ...makeTtsResult(""), chunks: 0 };
1001
+ },
1002
+ });
1003
+ await session.start();
1004
+ const first = outcome("task-1");
1005
+ const before = injectMessageIntoParentMock.mock.calls.length;
1006
+ session.receiveSubagentNotification(first);
1007
+ await waitFor(() => countType(frames, "tts_done") === 1);
1008
+ await new Promise((resolve) => setTimeout(resolve, 50));
1009
+ expect(calls).toHaveLength(1);
1010
+ await session.close("client_end");
1011
+ expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
1012
+ [
1013
+ "conversation-123",
1014
+ first.message,
1015
+ first.metadata,
1016
+ { cronRunId: "run-123", bypassLiveVoice: true },
1017
+ ],
1018
+ ]);
1019
+ },
1020
+ );
1021
+
1022
+ test("progress audio alone does not acknowledge the outcome", async () => {
1023
+ const calls: VoiceTurnOptions[] = [];
1024
+ const { session, frames } = createHarness({
1025
+ startVoiceTurn: async (turn) => {
1026
+ calls.push(turn);
1027
+ return { turnId: "turn-1", abort: mock() };
1028
+ },
1029
+ streamTtsAudio: immediateTts(),
1030
+ continuationAnnounceSilenceMs: 10,
1031
+ frontModelConfig: {
1032
+ progress: {
1033
+ enabled: true,
1034
+ opsThreshold: 3,
1035
+ idleIntervalMs: 40,
1036
+ maxSilenceMs: 40,
1037
+ longOpMs: 15_000,
1038
+ minGapMs: 60_000,
1039
+ generationTimeoutMs: 1_500,
1040
+ },
1041
+ },
1042
+ progressNarrator: { generateProgressText: async () => "Still working." },
1043
+ });
1044
+ const first = outcome("task-1");
1045
+ const before = injectMessageIntoParentMock.mock.calls.length;
1046
+ await session.start();
1047
+ try {
1048
+ session.receiveSubagentNotification(first);
1049
+ await waitFor(() => countType(frames, "tts_audio") === 1);
1050
+ calls[0]?.callbacks?.message_complete?.(makeMessageComplete());
1051
+ await waitFor(() => countType(frames, "tts_done") === 1);
1052
+ await new Promise((resolve) => setTimeout(resolve, 50));
1053
+ expect(calls).toHaveLength(1);
1054
+ } finally {
1055
+ await session.close("client_end");
1056
+ }
1057
+ expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
1058
+ [
1059
+ "conversation-123",
1060
+ first.message,
1061
+ first.metadata,
1062
+ { cronRunId: "run-123", bypassLiveVoice: true },
1063
+ ],
1064
+ ]);
1065
+ });
1066
+
1067
+ test("speech during the playback tail retries the outcome after answering the user", async () => {
1068
+ const { startVoiceTurn, calls } = makeAutoCompletingTurnStarter([
1069
+ "The result is ready.",
1070
+ "Here is the setting.",
1071
+ "The result is ready.",
1072
+ ]);
1073
+ const audio = "x".repeat(28_800);
1074
+ const { session, frames } = createHarness({
1075
+ finals: ["explain this setting"],
1076
+ startVoiceTurn,
1077
+ continuationAnnounceSilenceMs: 10,
1078
+ streamTtsAudio: async (options) => {
1079
+ options.onAudioChunk(makeTtsChunk(audio));
1080
+ return makeTtsResult(audio);
1081
+ },
1082
+ });
1083
+ await session.start();
1084
+ try {
1085
+ session.receiveSubagentNotification(outcome("task-1"));
1086
+ await waitFor(() => countType(frames, "tts_done") === 1);
1087
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
1088
+ await waitFor(() => countType(frames, "tts_done") === 3);
1089
+ expect(calls.map((call) => call.subagentNotification?.taskId)).toEqual([
1090
+ "task-1",
1091
+ undefined,
1092
+ "task-1",
1093
+ ]);
1094
+ } finally {
1095
+ await session.close("client_end");
1096
+ }
1097
+ });
1098
+
1099
+ test("hang-up during playback preserves a generated but unheard result", async () => {
1100
+ const { startVoiceTurn } = makeAutoCompletingTurnStarter([
1101
+ "The comparison is ready.",
1102
+ ]);
1103
+ const audio = "x".repeat(28_800);
1104
+ const { session, frames } = createHarness({
1105
+ startVoiceTurn,
1106
+ continuationAnnounceSilenceMs: 10,
1107
+ streamTtsAudio: async (options) => {
1108
+ options.onAudioChunk(makeTtsChunk(audio));
1109
+ return makeTtsResult(audio);
1110
+ },
1111
+ });
1112
+ await session.start();
1113
+ const first = outcome("task-1");
1114
+ const before = injectMessageIntoParentMock.mock.calls.length;
1115
+ session.receiveSubagentNotification(first);
1116
+ await waitFor(() => countType(frames, "tts_done") === 1);
1117
+ await session.close("client_end");
1118
+ expect(injectMessageIntoParentMock.mock.calls.slice(before)).toEqual([
1119
+ [
1120
+ "conversation-123",
1121
+ first.message,
1122
+ first.metadata,
1123
+ { cronRunId: "run-123", bypassLiveVoice: true },
1124
+ ],
1125
+ ]);
1126
+ });
1127
+ });
1128
+
579
1129
  describe("LiveVoiceSession server VAD", () => {
580
1130
  // The foreground-wins classification consults the tool registry's owner map
581
1131
  // (a tool is only provably read-only when it is the trusted built-in);
582
1132
  // register the core baseline so built-in names resolve like in the daemon.
583
1133
  beforeAll(() => {
584
1134
  __resetRegistryForTesting();
585
- registerSkillTools("computer-use-test", [
586
- finalizeTool(computerUseKeyTool),
587
- finalizeTool(computerUseObserveTool),
588
- finalizeTool({ ...computerUseObserveTool, name: "read_file" }),
589
- ]);
1135
+ registerSkillTools(
1136
+ "computer-use-test",
1137
+ ["computer_use_key", "computer_use_observe", "read_file"].map((name) =>
1138
+ finalizeTool({
1139
+ name,
1140
+ description: name,
1141
+ category: "computer-use",
1142
+ defaultRiskLevel: RiskLevel.Low,
1143
+ executionTarget: "host",
1144
+ input_schema: { type: "object", properties: {} },
1145
+ execute: async () => ({ content: "ok", isError: false }),
1146
+ }),
1147
+ ),
1148
+ );
590
1149
  });
591
1150
  afterAll(() => unregisterSkillTools("computer-use-test"));
592
1151
 
@@ -739,6 +1298,85 @@ describe("LiveVoiceSession server VAD", () => {
739
1298
  await waitFor(() => foregroundTaskStateOf(session) === null);
740
1299
  });
741
1300
 
1301
+ function scriptedJudge(verdict: Promise<boolean> | boolean) {
1302
+ const judged: Array<{
1303
+ interruptedRequest: string;
1304
+ interruption: string | null;
1305
+ }> = [];
1306
+ const judge: LiveVoiceContinuationJudge = async (args) => {
1307
+ const interruption = await args.interruption;
1308
+ judged.push({
1309
+ interruptedRequest: args.interruptedRequest,
1310
+ interruption,
1311
+ });
1312
+ const keep = await verdict;
1313
+ return keep
1314
+ ? { keep: true, outcome: "keep", noul: 0.9, latencyMs: 1 }
1315
+ : { keep: false, outcome: "drop", noul: 0.1, latencyMs: 1 };
1316
+ };
1317
+ return { judge, judged };
1318
+ }
1319
+
1320
+ async function answerInterruption(calls: VoiceTurnOptions[]) {
1321
+ const answer = calls[2];
1322
+ answer?.callbacks?.assistant_text_delta?.(makeTextDelta("Okay."));
1323
+ answer?.callbacks?.message_complete?.(makeMessageComplete());
1324
+ }
1325
+
1326
+ test("a suspended task the caller called off is cleared, not resumed", async () => {
1327
+ const { judge, judged } = scriptedJudge(false);
1328
+ const { calls, session } = await startForegroundTaskBargeInScenario({
1329
+ startHostTool: false,
1330
+ judgeBackgroundContinuation: judge,
1331
+ });
1332
+
1333
+ await answerInterruption(calls);
1334
+ await waitFor(() => judged.length === 1);
1335
+ expect(judged[0]).toEqual({
1336
+ interruptedRequest: "change the title",
1337
+ interruption: "what title is there now",
1338
+ });
1339
+ await waitFor(() => foregroundTaskStateOf(session) === null);
1340
+ // Well past the resume silence: no hidden resume turn was launched.
1341
+ await new Promise((resolve) => setTimeout(resolve, 80));
1342
+ expect(calls).toHaveLength(3);
1343
+ });
1344
+
1345
+ test("a suspended task the caller still wants resumes as before", async () => {
1346
+ const { judge } = scriptedJudge(true);
1347
+ const { calls } = await startForegroundTaskBargeInScenario({
1348
+ startHostTool: false,
1349
+ judgeBackgroundContinuation: judge,
1350
+ });
1351
+
1352
+ await answerInterruption(calls);
1353
+ await waitFor(() => calls.length === 4);
1354
+ expect(calls[3]).toMatchObject({ hiddenSyntheticPrompt: true });
1355
+ expect(calls[3]?.content).toContain("change the title");
1356
+ });
1357
+
1358
+ test("the resume waits for a verdict still in flight", async () => {
1359
+ let decide!: (keep: boolean) => void;
1360
+ const { judge } = scriptedJudge(
1361
+ new Promise<boolean>((resolve) => {
1362
+ decide = resolve;
1363
+ }),
1364
+ );
1365
+ const { calls, session } = await startForegroundTaskBargeInScenario({
1366
+ startHostTool: false,
1367
+ judgeBackgroundContinuation: judge,
1368
+ });
1369
+
1370
+ await answerInterruption(calls);
1371
+ await new Promise((resolve) => setTimeout(resolve, 80));
1372
+ expect(calls).toHaveLength(3);
1373
+
1374
+ decide(false);
1375
+ await waitFor(() => foregroundTaskStateOf(session) === null);
1376
+ await new Promise((resolve) => setTimeout(resolve, 40));
1377
+ expect(calls).toHaveLength(3);
1378
+ });
1379
+
742
1380
  test("a new barged-in task becomes the resume anchor", async () => {
743
1381
  const calls: VoiceTurnOptions[] = [];
744
1382
  const startVoiceTurn = mock(async (turn: VoiceTurnOptions) => {
@@ -1489,6 +2127,133 @@ describe("LiveVoiceSession server VAD", () => {
1489
2127
  expect(spawnArgs?.objective).toContain("first question");
1490
2128
  });
1491
2129
 
2130
+ async function bargeInWithJudge(keep: boolean) {
2131
+ const judged: Array<{
2132
+ interruptedRequest: string;
2133
+ interruption: string | null;
2134
+ }> = [];
2135
+ const judgeBackgroundContinuation: LiveVoiceContinuationJudge = async (
2136
+ args,
2137
+ ) => {
2138
+ const interruption = await args.interruption;
2139
+ judged.push({
2140
+ interruptedRequest: args.interruptedRequest,
2141
+ interruption,
2142
+ });
2143
+ return keep
2144
+ ? { keep: true, outcome: "keep", noul: 0.9, latencyMs: 1 }
2145
+ : { keep: false, outcome: "drop", noul: 0.1, latencyMs: 1 };
2146
+ };
2147
+ const spawnBackgroundContinuation = mock(
2148
+ async (_args: {
2149
+ parentConversationId: string;
2150
+ objective: string;
2151
+ label: string;
2152
+ signal: AbortSignal;
2153
+ }): Promise<string> => "",
2154
+ );
2155
+ const calls: VoiceTurnOptions[] = [];
2156
+ const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
2157
+ calls.push(options);
2158
+ return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
2159
+ });
2160
+ const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
2161
+ options.onAudioChunk(makeTtsChunk("assistant audio"));
2162
+ return makeTtsResult("assistant audio");
2163
+ });
2164
+ const { frames, session } = createHarness({
2165
+ finals: ["first question", "never mind that"],
2166
+ startVoiceTurn,
2167
+ streamTtsAudio,
2168
+ spawnBackgroundContinuation,
2169
+ judgeBackgroundContinuation,
2170
+ });
2171
+
2172
+ await session.start();
2173
+ await session.handleBinaryAudio(LOUD_CHUNK);
2174
+ await waitFor(() => frames.some((frame) => frame.type === "thinking"));
2175
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
2176
+ await waitFor(() => judged.length === 1);
2177
+ await flushAsyncCallbacks();
2178
+ return { judged, spawnBackgroundContinuation, calls };
2179
+ }
2180
+
2181
+ test("the continuation judge sees the interrupting words and can drop the continuation", async () => {
2182
+ const { judged, spawnBackgroundContinuation, calls } =
2183
+ await bargeInWithJudge(false);
2184
+
2185
+ expect(judged).toEqual([
2186
+ { interruptedRequest: "first question", interruption: "never mind that" },
2187
+ ]);
2188
+ expect(spawnBackgroundContinuation).not.toHaveBeenCalled();
2189
+ // The foreground turn still carries the interrupted request.
2190
+ const followUp = calls.find((c) => c.content === "never mind that");
2191
+ expect(followUp?.voiceControlPrompt).toContain("first question");
2192
+ });
2193
+
2194
+ test("a speculative interruption reaches the judge only once its turn commits", async () => {
2195
+ const judged: Array<string | null> = [];
2196
+ const judgeBackgroundContinuation: LiveVoiceContinuationJudge = async (
2197
+ args,
2198
+ ) => {
2199
+ judged.push(await args.interruption);
2200
+ return { keep: false, outcome: "drop", noul: 0.1, latencyMs: 1 };
2201
+ };
2202
+ let openGate!: () => void;
2203
+ const gate = new Promise<void>((resolve) => {
2204
+ openGate = resolve;
2205
+ });
2206
+ const calls: VoiceTurnOptions[] = [];
2207
+ const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
2208
+ calls.push(options);
2209
+ if (options.content !== "first question") {
2210
+ // The leg's first word is its verdict: nothing commits until then.
2211
+ void gate.then(() => {
2212
+ options.callbacks?.assistant_text_delta?.(
2213
+ makeTextDelta("Sure thing."),
2214
+ );
2215
+ options.callbacks?.message_complete?.(makeMessageComplete());
2216
+ });
2217
+ }
2218
+ return { turnId: `bridge-turn-${calls.length}`, abort: mock() };
2219
+ });
2220
+ const { frames, session, transcribers } = createHarness({
2221
+ finals: ["first question", "never mind that"],
2222
+ startVoiceTurn,
2223
+ streamTtsAudio: makeImmediateTts(),
2224
+ spawnBackgroundContinuation: mock(async () => ""),
2225
+ judgeBackgroundContinuation,
2226
+ });
2227
+
2228
+ await session.start();
2229
+ await session.handleBinaryAudio(LOUD_CHUNK);
2230
+ await waitFor(() => frames.some((frame) => frame.type === "thinking"));
2231
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
2232
+ await waitFor(() => transcribers.length === 2);
2233
+ transcribers[1]?.emit({ type: "partial", text: "never mind that" });
2234
+ await session.handleBinaryAudio(LOUD_CHUNK);
2235
+ await waitFor(() => calls.some((c) => c.content === "never mind that"));
2236
+ expect(
2237
+ calls.find((c) => c.content === "never mind that")?.unifiedVerdict,
2238
+ ).toBe(true);
2239
+
2240
+ await flushAsyncCallbacks();
2241
+ expect(judged).toEqual([]);
2242
+
2243
+ openGate();
2244
+ await waitFor(() => judged.length === 1);
2245
+ expect(judged).toEqual(["never mind that"]);
2246
+ });
2247
+
2248
+ test("a keep verdict spawns the continuation as before", async () => {
2249
+ const { spawnBackgroundContinuation } = await bargeInWithJudge(true);
2250
+
2251
+ await waitFor(() => spawnBackgroundContinuation.mock.calls.length === 1);
2252
+ expect(spawnBackgroundContinuation.mock.calls[0]?.[0]?.objective).toContain(
2253
+ "first question",
2254
+ );
2255
+ });
2256
+
1492
2257
  test("the continuation carries the label the model phrased", async () => {
1493
2258
  const labelCalls: Array<{
1494
2259
  parentConversationId: string;
@@ -1884,7 +2649,7 @@ describe("LiveVoiceSession server VAD", () => {
1884
2649
  return { startVoiceTurn, calls };
1885
2650
  }
1886
2651
 
1887
- test("a completed continuation's result folds into the next turn's control prompt", async () => {
2652
+ test("a completed continuation remains pending across unrelated user turns", async () => {
1888
2653
  // Control the continuation's resolution so we know exactly when its result
1889
2654
  // is stashed relative to the turns we inspect.
1890
2655
  let resolveContinuation: ((result: string) => void) | undefined;
@@ -1942,14 +2707,14 @@ describe("LiveVoiceSession server VAD", () => {
1942
2707
  expect(resurfaced?.voiceControlPrompt).toContain("THE_RESULT");
1943
2708
  expect(resurfaced?.voiceControlPrompt).toContain("background");
1944
2709
 
1945
- // Consume-once: a later turn does not repeat it.
2710
+ // Context exposure is not delivery, so a later turn still has the result.
1946
2711
  await session.handleBinaryAudio(LOUD_CHUNK);
1947
2712
  await waitFor(() => calls.some((c) => c.content === "fourth question"));
1948
2713
  const later = calls.find((c) => c.content === "fourth question");
1949
- expect(later?.voiceControlPrompt).not.toContain("THE_RESULT");
2714
+ expect(later?.voiceControlPrompt).toContain("THE_RESULT");
1950
2715
  });
1951
2716
 
1952
- test("a client interrupt drops a stashed continuation result", async () => {
2717
+ test("a client interrupt defers delivery without dropping a finished result", async () => {
1953
2718
  let resolveContinuation: ((result: string) => void) | undefined;
1954
2719
  const spawnBackgroundContinuation = mock(
1955
2720
  (_args: {
@@ -1994,15 +2759,15 @@ describe("LiveVoiceSession server VAD", () => {
1994
2759
  resolveContinuation?.("THE_RESULT");
1995
2760
  await flushAsyncCallbacks();
1996
2761
 
1997
- // ...but a client interrupt is a hard stop that drops it.
2762
+ // Stopping speech defers delivery until the next user turn.
1998
2763
  await session.handleClientFrame({ type: "interrupt" });
1999
2764
  await flushAsyncCallbacks();
2000
2765
 
2001
- // The next turn carries no resurfaced result.
2766
+ // Finished work remains available to the next turn.
2002
2767
  await session.handleBinaryAudio(LOUD_CHUNK);
2003
2768
  await waitFor(() => calls.some((c) => c.content === "third question"));
2004
2769
  const later = calls.find((c) => c.content === "third question");
2005
- expect(later?.voiceControlPrompt).not.toContain("THE_RESULT");
2770
+ expect(later?.voiceControlPrompt).toContain("THE_RESULT");
2006
2771
  });
2007
2772
 
2008
2773
  test("an empty continuation result adds no context to the next turn", async () => {
@@ -2195,7 +2960,7 @@ describe("LiveVoiceSession server VAD", () => {
2195
2960
  resolvers[1]?.("");
2196
2961
  });
2197
2962
 
2198
- test("a new barge-in drops an already-stashed continuation result", async () => {
2963
+ test("a new barge-in preserves an already-finished continuation", async () => {
2199
2964
  const resolvers: Array<(result: string) => void> = [];
2200
2965
  const spawnBackgroundContinuation = mock(
2201
2966
  (_args: {
@@ -2248,15 +3013,14 @@ describe("LiveVoiceSession server VAD", () => {
2248
3013
  resolvers[0]?.("A_RESULT");
2249
3014
  await flushAsyncCallbacks();
2250
3015
 
2251
- // Barge #2 is a fresh interruption: it must drop A's already-stashed result
2252
- // (and detach the newer continuation).
3016
+ // Barge #2 starts a new continuation while the finished result stays pending.
2253
3017
  await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
2254
3018
  await waitFor(() => calls.some((c) => c.content === "third question"));
2255
3019
  await waitFor(() => resolvers.length === 2);
2256
3020
 
2257
- // The barge #2 follow-up turn carries no stale A result.
3021
+ // The follow-up still has the completed result as context.
2258
3022
  const followUp = calls.find((c) => c.content === "third question");
2259
- expect(followUp?.voiceControlPrompt).not.toContain("A_RESULT");
3023
+ expect(followUp?.voiceControlPrompt).toContain("A_RESULT");
2260
3024
 
2261
3025
  // Cleanup the still-pending newer continuation.
2262
3026
  resolvers[1]?.("");
@@ -2494,6 +3258,45 @@ describe("LiveVoiceSession server VAD", () => {
2494
3258
  expect(announcement?.voiceControlPrompt).toContain("first question");
2495
3259
  });
2496
3260
 
3261
+ test("continuations and explicit workers share delivery and finish independently", async () => {
3262
+ const continuation = makeControlledContinuation();
3263
+ const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
3264
+ const { frames, session } = createHarness({
3265
+ finals: ["first question", ""],
3266
+ startVoiceTurn,
3267
+ streamTtsAudio: makeImmediateTts(),
3268
+ spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
3269
+ continuationAnnounceSilenceMs: 20,
3270
+ });
3271
+ await session.start();
3272
+ await session.handleBinaryAudio(LOUD_CHUNK);
3273
+ await waitFor(() => calls.length === 1);
3274
+ await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
3275
+ await waitFor(
3276
+ () => continuation.spawnBackgroundContinuation.mock.calls.length === 1,
3277
+ );
3278
+ await waitFor(() => countType(frames, "utterance_discarded") === 1);
3279
+ continuation.finish("CONTINUATION_RESULT");
3280
+ session.receiveSubagentNotification({
3281
+ taskId: "task-1",
3282
+ message: "WORKER_RESULT",
3283
+ metadata: {},
3284
+ });
3285
+ await waitFor(() => countType(frames, "tts_done") === 2);
3286
+ expect(
3287
+ calls.filter((call) => call.content === "WORKER_RESULT"),
3288
+ ).toHaveLength(1);
3289
+ expect(announcementCount(calls)).toBe(1);
3290
+ expect(announcementOf(calls)?.voiceControlPrompt).toContain(
3291
+ "CONTINUATION_RESULT",
3292
+ );
3293
+ expect(announcementOf(calls)?.directEscalated).toBe(true);
3294
+ await new Promise((resolve) => setTimeout(resolve, 30));
3295
+ const before = injectMessageIntoParentMock.mock.calls.length;
3296
+ await session.close("client_end");
3297
+ expect(injectMessageIntoParentMock.mock.calls.length).toBe(before);
3298
+ });
3299
+
2497
3300
  // Deepgram Flux sends interim updates through silence, each an empty
2498
3301
  // partial. A call nobody is talking on is still idle.
2499
3302
  test("an empty partial on an idle call does not hold the announcement back", async () => {
@@ -2565,7 +3368,7 @@ describe("LiveVoiceSession server VAD", () => {
2565
3368
  expect(later?.voiceControlPrompt).not.toContain("THE_RESULT");
2566
3369
  });
2567
3370
 
2568
- test("a user utterance before the silence elapses cancels the announcement", async () => {
3371
+ test("an unrelated user reply does not consume the queued continuation", async () => {
2569
3372
  const continuation = makeControlledContinuation();
2570
3373
  const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
2571
3374
  const { frames, session } = createHarness({
@@ -2574,7 +3377,7 @@ describe("LiveVoiceSession server VAD", () => {
2574
3377
  streamTtsAudio: makeImmediateTts(),
2575
3378
  spawnBackgroundContinuation: continuation.spawnBackgroundContinuation,
2576
3379
  // Long enough that the user always speaks first.
2577
- continuationAnnounceSilenceMs: 5_000,
3380
+ continuationAnnounceSilenceMs: 250,
2578
3381
  });
2579
3382
 
2580
3383
  await session.start();
@@ -2591,14 +3394,16 @@ describe("LiveVoiceSession server VAD", () => {
2591
3394
  continuation.finish("THE_RESULT");
2592
3395
  await flushAsyncCallbacks();
2593
3396
 
2594
- // The user speaks first: their turn delivers the answer through the stash,
2595
- // and the announcement is dropped rather than repeating it after.
3397
+ // The unrelated reply can see the result, but only says "ok".
2596
3398
  await session.handleBinaryAudio(LOUD_CHUNK);
2597
3399
  await waitFor(() => calls.some((c) => c.content === "third question"));
2598
3400
  const spoken = calls.find((c) => c.content === "third question");
2599
3401
  expect(spoken?.voiceControlPrompt).toContain("THE_RESULT");
2600
- await flushAsyncCallbacks();
2601
- expect(announcementOf(calls)).toBeUndefined();
3402
+ await waitFor(() => announcementCount(calls) === 1);
3403
+ expect(announcementOf(calls)?.voiceControlPrompt).toContain("THE_RESULT");
3404
+ expect(announcementOf(calls)?.voiceControlPrompt).toContain(
3405
+ 'completed audio playback for this task was: "ok"',
3406
+ );
2602
3407
  });
2603
3408
 
2604
3409
  test("an active turn at fire time announces after the turn becomes idle", async () => {
@@ -2821,52 +3626,42 @@ describe("LiveVoiceSession server VAD", () => {
2821
3626
  expect(later?.voiceControlPrompt).toContain("THE_RESULT");
2822
3627
  });
2823
3628
 
2824
- test("an announcement whose launch throws hands the answer back to the stash", async () => {
3629
+ test("an announcement whose launch throws retains its result", async () => {
2825
3630
  const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
2826
3631
  const { session } = createHarness({
2827
3632
  finals: ["stashed question"],
2828
3633
  startVoiceTurn,
2829
3634
  continuationAnnounceSilenceMs: 20,
2830
3635
  });
2831
-
2832
3636
  await session.start();
2833
- await flushAsyncCallbacks();
2834
-
2835
3637
  const internals = session as unknown as {
2836
- pendingContinuationResult: string | null;
2837
- pendingAnnouncement: { request: string; answer: string } | null;
2838
- scheduleContinuationAnnouncement: () => void;
3638
+ receiveTaskOutcome: (
3639
+ outcome: import("../task-outcomes.js").VoiceTaskOutcome,
3640
+ ) => boolean;
2839
3641
  launchAssistantTurn: (
2840
3642
  utterance: unknown,
2841
3643
  content: string,
2842
3644
  opts?: unknown,
2843
3645
  ) => Promise<boolean>;
2844
3646
  };
2845
- // The announcement's launch REJECTS rather than returning false — a frame
2846
- // write failing ahead of the leg's own error handling. The `started ===
2847
- // false` re-stash never runs on that path, so only the catch can save the
2848
- // answer.
2849
3647
  const launch = internals.launchAssistantTurn.bind(session);
3648
+ const failed = mock(() => {
3649
+ throw new Error("thinking frame write failed");
3650
+ });
2850
3651
  internals.launchAssistantTurn = async (utterance, content, opts) => {
2851
3652
  if (content === CONTINUATION_DELIVERY_CONTENT) {
2852
- throw new Error("thinking frame write failed");
3653
+ return failed();
2853
3654
  }
2854
3655
  return launch(utterance, content, opts);
2855
3656
  };
2856
- internals.pendingContinuationResult = "THE_RESULT";
2857
- internals.pendingAnnouncement = {
2858
- request: "the first question",
2859
- answer: "THE_RESULT",
2860
- };
2861
- internals.scheduleContinuationAnnouncement();
2862
-
2863
- // The attempt runs (the queue empties) and throws.
2864
- await waitFor(() => internals.pendingAnnouncement === null);
2865
- await flushAsyncCallbacks();
3657
+ internals.receiveTaskOutcome({
3658
+ taskId: "continuation-1",
3659
+ message: "THE_RESULT",
3660
+ metadata: {},
3661
+ source: "continuation",
3662
+ });
3663
+ await waitFor(() => failed.mock.calls.length === 1);
2866
3664
  expect(announcementOf(calls)).toBeUndefined();
2867
- expect(internals.pendingContinuationResult).toBe("THE_RESULT");
2868
-
2869
- // ...so the user's next turn still carries it.
2870
3665
  await session.handleBinaryAudio(LOUD_CHUNK);
2871
3666
  await waitFor(() => calls.some((c) => c.content === "stashed question"));
2872
3667
  expect(
@@ -2952,7 +3747,7 @@ describe("LiveVoiceSession server VAD", () => {
2952
3747
  expect(message).toContain("first question");
2953
3748
  });
2954
3749
 
2955
- test("a speculatively dispatched turn carries the stashed answer and cancels the announcement", async () => {
3750
+ test("a speculative user reply preserves the pending announcement", async () => {
2956
3751
  const continuation = makeControlledContinuation();
2957
3752
  const { startVoiceTurn, calls } = makeResurfaceTurnStarter();
2958
3753
  const { frames, session, transcribers } = createHarness({
@@ -2995,22 +3790,66 @@ describe("LiveVoiceSession server VAD", () => {
2995
3790
  expect(speculative?.unifiedVerdict).toBe(true);
2996
3791
  expect(speculative?.voiceControlPrompt).toContain("THE_RESULT");
2997
3792
 
2998
- // The user's own turn is the delivery, so the queued announcement is
2999
- // cancelled rather than speaking the same answer again afterwards.
3000
- await new Promise((resolve) => setTimeout(resolve, 600));
3001
- expect(announcementCount(calls)).toBe(0);
3793
+ // Completing an unrelated speculative reply does not acknowledge delivery.
3794
+ await waitFor(() => announcementCount(calls) === 1);
3002
3795
  });
3003
3796
 
3004
3797
  test("a held speculative turn hands the stashed answer back for the replay", async () => {
3005
3798
  const continuation = makeControlledContinuation();
3006
3799
  const calls: VoiceTurnOptions[] = [];
3007
- const discard = mock(async () => {});
3800
+ const claimedTurnIds: string[] = [];
3801
+ const releasedTurnIds = new Set<string>();
3802
+ let activation = 0;
3803
+ let activeSource: ModeSessionSourceHandle | undefined;
3804
+ const modeSessions = {
3805
+ activateSource: (input: {
3806
+ sourceId: string;
3807
+ generation: number;
3808
+ mode: "ambient" | "live_vision";
3809
+ }) => {
3810
+ activation += 1;
3811
+ activeSource = {
3812
+ sourceId: input.sourceId,
3813
+ generation: input.generation,
3814
+ activation,
3815
+ id: `camera-session-${activation}`,
3816
+ mode: input.mode,
3817
+ };
3818
+ return activeSource;
3819
+ },
3820
+ claimTurn: (turnId: string, source: ModeSessionSourceHandle) => {
3821
+ if (source !== activeSource) {
3822
+ return undefined;
3823
+ }
3824
+ claimedTurnIds.push(turnId);
3825
+ return { id: source.id, mode: source.mode };
3826
+ },
3827
+ recordActivity: () => true,
3828
+ releaseTurn: (turnId: string) => {
3829
+ releasedTurnIds.add(turnId);
3830
+ return true;
3831
+ },
3832
+ retireSource: (source: ModeSessionSourceHandle) => {
3833
+ if (source !== activeSource) {
3834
+ return false;
3835
+ }
3836
+ activeSource = undefined;
3837
+ return true;
3838
+ },
3839
+ trackPersistedRow: () => {},
3840
+ } as unknown as ConversationModeSessionCoordinator;
3008
3841
  // The first turn stays thinking so the barge-in lands on it. Every
3009
3842
  // speculative leg afterwards answers with the hold token while it is
3010
3843
  // offered one (`unifiedVerdict`), and answers for real on the replay.
3011
3844
  const startVoiceTurn: LiveVoiceTurnStarter = async (options) => {
3012
3845
  calls.push(options);
3013
3846
  const index = calls.length;
3847
+ const discard = mock(async () => {
3848
+ const requestId = options.preacceptedModeSession?.requestId;
3849
+ if (requestId) {
3850
+ modeSessions.releaseTurn(requestId);
3851
+ }
3852
+ });
3014
3853
  if (options.content !== "first question") {
3015
3854
  const reply = options.unifiedVerdict === true ? "[0]" : "Sure thing.";
3016
3855
  setTimeout(() => {
@@ -3031,9 +3870,19 @@ describe("LiveVoiceSession server VAD", () => {
3031
3870
  // The announcement must not fire during the hold window — the replay
3032
3871
  // turn is the delivery this test is about.
3033
3872
  continuationAnnounceSilenceMs: 5_000,
3873
+ acquireModeSessionResidency: async () => ({
3874
+ coordinator: modeSessions,
3875
+ release: () => {},
3876
+ }),
3034
3877
  });
3035
3878
 
3036
3879
  await session.start();
3880
+ await session.handleClientFrame({
3881
+ type: "sight_start",
3882
+ cameraEpoch: 1,
3883
+ source: "live",
3884
+ });
3885
+ expect(claimedTurnIds).toHaveLength(1);
3037
3886
  await session.handleBinaryAudio(LOUD_CHUNK);
3038
3887
  await waitFor(() => frames.some((frame) => frame.type === "thinking"));
3039
3888
  await session.handleBinaryAudio(SUSTAINED_LOUD_CHUNK);
@@ -3052,7 +3901,26 @@ describe("LiveVoiceSession server VAD", () => {
3052
3901
  await session.handleBinaryAudio(LOUD_CHUNK);
3053
3902
 
3054
3903
  // The speculative leg holds: it is rolled back, taking nothing with it.
3055
- await waitFor(() => discard.mock.calls.length === 1);
3904
+ await waitFor(() =>
3905
+ calls.some(
3906
+ (call) =>
3907
+ call.content === "third question" && call.unifiedVerdict === true,
3908
+ ),
3909
+ );
3910
+ const speculative = calls.find(
3911
+ (call) =>
3912
+ call.content === "third question" && call.unifiedVerdict === true,
3913
+ );
3914
+ expect(speculative?.preacceptedModeSession).toBeDefined();
3915
+ await waitFor(() =>
3916
+ releasedTurnIds.has(speculative?.preacceptedModeSession?.requestId ?? ""),
3917
+ );
3918
+ const primaryIndex = claimedTurnIds.indexOf(
3919
+ speculative!.preacceptedModeSession!.requestId,
3920
+ );
3921
+ expect(primaryIndex).toBeGreaterThanOrEqual(0);
3922
+ expect(releasedTurnIds.has(claimedTurnIds[primaryIndex + 1]!)).toBe(true);
3923
+ expect(releasedTurnIds.has(claimedTurnIds[primaryIndex + 2]!)).toBe(true);
3056
3924
 
3057
3925
  // The extension replays the boundary and the replay leg answers. It is a
3058
3926
  // different dispatch, so it only carries the answer if the rollback gave
@@ -3063,6 +3931,7 @@ describe("LiveVoiceSession server VAD", () => {
3063
3931
  const replay = calls.filter((c) => c.content === "third question")[1];
3064
3932
  expect(replay?.unifiedVerdict).toBeUndefined();
3065
3933
  expect(replay?.voiceControlPrompt).toContain("THE_RESULT");
3934
+ await session.close("client_end");
3066
3935
  });
3067
3936
 
3068
3937
  test("an announcement waits out the client's queued playback", async () => {
@@ -3126,24 +3995,19 @@ describe("LiveVoiceSession server VAD", () => {
3126
3995
  // every transcript-derived idle signal still reads "nobody is speaking".
3127
3996
  await session.handleBinaryAudio(LOUD_CHUNK);
3128
3997
 
3129
- // A manual session reaches the announcement path only through machinery a
3130
- // server_vad barge-in owns, so the state a finished continuation leaves
3131
- // behind is staged directly: the stashed answer plus its queued
3132
- // announcement.
3133
3998
  const internals = session as unknown as {
3134
- pendingContinuationResult: string | null;
3135
- pendingAnnouncement: { request: string; answer: string } | null;
3136
- scheduleContinuationAnnouncement: () => void;
3999
+ receiveTaskOutcome: (
4000
+ outcome: import("../task-outcomes.js").VoiceTaskOutcome,
4001
+ ) => boolean;
3137
4002
  };
3138
- internals.pendingContinuationResult = "THE_RESULT";
3139
- internals.pendingAnnouncement = {
3140
- request: "the first question",
3141
- answer: "THE_RESULT",
3142
- };
3143
- internals.scheduleContinuationAnnouncement();
4003
+ internals.receiveTaskOutcome({
4004
+ taskId: "continuation-1",
4005
+ message: "THE_RESULT",
4006
+ metadata: {},
4007
+ source: "continuation",
4008
+ });
3144
4009
 
3145
- // The silence timer and the single retry both find the utterance in
3146
- // flight, so the session never speaks over the in-progress utterance.
4010
+ // The floor stays blocked while captured speech is awaiting release.
3147
4011
  await new Promise((resolve) => setTimeout(resolve, 200));
3148
4012
  expect(announcementOf(calls)).toBeUndefined();
3149
4013