@vellumai/assistant 0.12.2 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (772) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +17 -2
  7. package/docs/architecture/security.md +10 -5
  8. package/docs/architecture/turn-actor.md +2 -0
  9. package/docs/credential-execution-service.md +1 -1
  10. package/docs/desktop-browser-cli.md +50 -5
  11. package/docs/live-voice-task-outcomes.md +100 -0
  12. package/docs/mode-sessions.md +183 -0
  13. package/docs/voice-input-diagnostics.md +58 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  19. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  20. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  26. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  28. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  29. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  30. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  31. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  32. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  33. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  35. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  36. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  37. package/openapi.yaml +716 -16
  38. package/package.json +1 -1
  39. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  40. package/scripts/smoke-desktop-window-theme.ts +2 -3
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/scripts/voice-ttft-spike.ts +2 -2
  43. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  44. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  45. package/src/__tests__/agent-loop.test.ts +259 -0
  46. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  47. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  48. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  49. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  50. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  51. package/src/__tests__/anthropic-provider.test.ts +55 -0
  52. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  53. package/src/__tests__/approval-message-composer.test.ts +1 -118
  54. package/src/__tests__/approval-routes-http.test.ts +3 -0
  55. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  56. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  57. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  58. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  59. package/src/__tests__/attachments-store.test.ts +22 -3
  60. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  61. package/src/__tests__/btw-routes.test.ts +4 -1
  62. package/src/__tests__/build-persisted-content.test.ts +1 -1
  63. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  64. package/src/__tests__/call-recovery.test.ts +7 -2
  65. package/src/__tests__/call-routes-http.test.ts +12 -0
  66. package/src/__tests__/call-store.test.ts +7 -2
  67. package/src/__tests__/channel-approval.test.ts +9 -14
  68. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  69. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  70. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  71. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  72. package/src/__tests__/checker.test.ts +11 -3
  73. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  74. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  75. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  76. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  77. package/src/__tests__/computer-use-tools.test.ts +1 -233
  78. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  79. package/src/__tests__/config-schema.test.ts +16 -0
  80. package/src/__tests__/config-watcher.test.ts +24 -0
  81. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  82. package/src/__tests__/context-search-fanout.test.ts +1 -1
  83. package/src/__tests__/context-search-types.test.ts +7 -7
  84. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  85. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  86. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  89. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  90. package/src/__tests__/conversation-attachments.test.ts +106 -0
  91. package/src/__tests__/conversation-evictor.test.ts +62 -37
  92. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  93. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  94. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  95. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  96. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  97. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  98. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  99. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  101. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  102. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  103. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  104. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  105. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  106. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  107. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  108. package/src/__tests__/conversation-undo.test.ts +61 -0
  109. package/src/__tests__/credential-execution-client.test.ts +72 -55
  110. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  111. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  112. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  113. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  114. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  115. package/src/__tests__/dm-backfill.test.ts +91 -1
  116. package/src/__tests__/document-create-route.test.ts +88 -0
  117. package/src/__tests__/document-sync-tags.test.ts +27 -0
  118. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  119. package/src/__tests__/events-tail-route.test.ts +58 -0
  120. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  121. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  122. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  123. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  124. package/src/__tests__/heartbeat-service.test.ts +2 -59
  125. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  126. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  127. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  128. package/src/__tests__/history-repair.test.ts +53 -0
  129. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  130. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  131. package/src/__tests__/image-credentials.test.ts +49 -0
  132. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  133. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  134. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  135. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  136. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  137. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  138. package/src/__tests__/media-generate-image.test.ts +49 -1
  139. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  140. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  141. package/src/__tests__/mode-session-history.test.ts +245 -0
  142. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  143. package/src/__tests__/mode-session-wire.test.ts +141 -0
  144. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  145. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  146. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  147. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  148. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  149. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  150. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  151. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  152. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  153. package/src/__tests__/persist-media-references.test.ts +50 -0
  154. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  155. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  156. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  157. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  158. package/src/__tests__/process-message-display-content.test.ts +56 -0
  159. package/src/__tests__/reaction-persistence.test.ts +5 -0
  160. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  161. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  162. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  163. package/src/__tests__/schedule-routes.test.ts +20 -0
  164. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  165. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  166. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  167. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  168. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  169. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  170. package/src/__tests__/secret-scanner.test.ts +20 -0
  171. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  172. package/src/__tests__/secure-keys.test.ts +50 -4
  173. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  174. package/src/__tests__/server-history-render.test.ts +91 -1
  175. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  176. package/src/__tests__/settings-routes.test.ts +50 -1
  177. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  178. package/src/__tests__/skills.test.ts +5 -4
  179. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  180. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  181. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  182. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  183. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  184. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  185. package/src/__tests__/terminal-tools.test.ts +8 -0
  186. package/src/__tests__/thread-backfill.test.ts +53 -0
  187. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  188. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  189. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  190. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  191. package/src/__tests__/twilio-routes.test.ts +1 -0
  192. package/src/__tests__/unicode.test.ts +36 -0
  193. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  194. package/src/activation/progress-store.test.ts +12 -7
  195. package/src/agent/loop.ts +136 -53
  196. package/src/agent/tool-result-follow-up.ts +70 -0
  197. package/src/api/attachment-provenance.test.ts +68 -0
  198. package/src/api/computer-use-tool.test.ts +55 -0
  199. package/src/api/computer-use-tool.ts +27 -0
  200. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  201. package/src/api/events/assistant-turn-start.ts +3 -0
  202. package/src/api/events/desktop-activity-changed.ts +10 -0
  203. package/src/api/events/generation-handoff.ts +2 -0
  204. package/src/api/events/message-complete.ts +2 -0
  205. package/src/api/events/notification-conversation-created.ts +14 -16
  206. package/src/api/events/notification-intent.ts +4 -2
  207. package/src/api/events/question-request.ts +1 -0
  208. package/src/api/events/tool-result.ts +85 -0
  209. package/src/api/events/tool-use-start.ts +3 -0
  210. package/src/api/events/user-message-echo.ts +3 -0
  211. package/src/api/index.ts +42 -6
  212. package/src/api/mode-session.test.ts +131 -0
  213. package/src/api/mode-session.ts +148 -0
  214. package/src/api/remember-tool.ts +15 -0
  215. package/src/api/responses/conversation-message.ts +9 -0
  216. package/src/approvals/AGENTS.md +4 -2
  217. package/src/approvals/approval-primitive.ts +5 -2
  218. package/src/approvals/scoped-approval-grants.ts +6 -2
  219. package/src/browser/__tests__/operations.test.ts +29 -0
  220. package/src/browser/operation-meta.ts +2 -1
  221. package/src/browser/operations.ts +29 -0
  222. package/src/browser/virtual-desktop-target.ts +2 -9
  223. package/src/bundler/bundle-signer.ts +1 -3
  224. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  225. package/src/calls/__tests__/call-controller.test.ts +94 -0
  226. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  227. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  228. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  229. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  230. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  231. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  232. package/src/calls/barge-in-guard.ts +5 -0
  233. package/src/calls/call-controller.ts +196 -1
  234. package/src/calls/call-domain.ts +4 -0
  235. package/src/calls/call-store.ts +87 -2
  236. package/src/calls/guardian-action-sweep.ts +22 -8
  237. package/src/calls/media-stream-audio-transcode.ts +1 -3
  238. package/src/calls/typesafe-noul.ts +156 -0
  239. package/src/calls/voice-control-protocol.ts +6 -0
  240. package/src/calls/voice-escalation-judge.ts +145 -0
  241. package/src/calls/voice-escalation-target.ts +10 -0
  242. package/src/calls/voice-leg-coordinator.ts +121 -3
  243. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  244. package/src/calls/voice-session-bridge.ts +256 -112
  245. package/src/calls/voice-triage-escalate.ts +7 -0
  246. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  247. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  248. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  249. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  250. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  251. package/src/cli/commands/bash.help.ts +4 -3
  252. package/src/cli/commands/browser.help.ts +9 -6
  253. package/src/cli/commands/browser.ts +0 -1
  254. package/src/cli/commands/clients.help.ts +46 -0
  255. package/src/cli/commands/clients.ts +116 -0
  256. package/src/cli/commands/credentials.help.ts +9 -4
  257. package/src/cli/commands/notifications.help.ts +16 -2
  258. package/src/cli/commands/notifications.ts +30 -8
  259. package/src/cli/commands/oauth/index.help.ts +10 -0
  260. package/src/cli/commands/oauth/providers.ts +12 -0
  261. package/src/cli/commands/oauth/request.test.ts +179 -2
  262. package/src/cli/commands/oauth/request.ts +41 -10
  263. package/src/cli/commands/plugins.ts +12 -5
  264. package/src/cli/commands/roadmap.help.ts +18 -21
  265. package/src/cli/commands/roadmap.ts +4 -21
  266. package/src/cli/commands/schedules.ts +2 -0
  267. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  268. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  269. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  270. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  271. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  272. package/src/cli/lib/bundled-marketplace.json +906 -1
  273. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  274. package/src/cli/lib/inspect-plugin.ts +11 -4
  275. package/src/cli/lib/plugin-marketplace.ts +8 -0
  276. package/src/cli/lib/publish-plugin.ts +1 -2
  277. package/src/cli/lib/upgrade-plugin.ts +19 -11
  278. package/src/cli/output.ts +18 -4
  279. package/src/config/__tests__/default-provider.test.ts +6 -0
  280. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  281. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  282. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  283. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  284. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  285. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  286. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  287. package/src/config/call-site-defaults.ts +9 -0
  288. package/src/config/feature-flag-registry.json +16 -0
  289. package/src/config/loader.ts +9 -17
  290. package/src/config/platform-identity.ts +8 -8
  291. package/src/config/profile-text-generation.test.ts +51 -0
  292. package/src/config/profile-text-generation.ts +51 -0
  293. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  294. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  295. package/src/config/schemas/call-site-catalog.ts +14 -0
  296. package/src/config/schemas/channels.ts +0 -18
  297. package/src/config/schemas/llm.ts +9 -2
  298. package/src/config/schemas/mcp.ts +6 -1
  299. package/src/config/schemas/memory-retrospective.ts +11 -2
  300. package/src/config/schemas/memory-v3.ts +12 -0
  301. package/src/config/schemas/services.ts +15 -6
  302. package/src/config/schemas/tools.ts +10 -0
  303. package/src/config/session-groups-gate.ts +21 -0
  304. package/src/config/skills.ts +2 -6
  305. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  306. package/src/conversations/message-consolidation.ts +102 -5
  307. package/src/credential-execution/ces-connect.test.ts +69 -0
  308. package/src/credential-execution/ces-connect.ts +115 -0
  309. package/src/credential-execution/ces-runtime.ts +55 -93
  310. package/src/credential-execution/executable-discovery.ts +34 -142
  311. package/src/credential-execution/process-manager.test.ts +20 -24
  312. package/src/credential-execution/process-manager.ts +24 -19
  313. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  314. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  315. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  316. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  317. package/src/daemon/assistant-attachments.ts +20 -12
  318. package/src/daemon/browser-mode-session.test.ts +402 -0
  319. package/src/daemon/browser-mode-session.ts +186 -0
  320. package/src/daemon/chat-credential-redaction.ts +16 -1
  321. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  322. package/src/daemon/computer-use-mode-session.ts +105 -0
  323. package/src/daemon/config-watcher.ts +10 -0
  324. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  325. package/src/daemon/conversation-agent-loop.ts +231 -25
  326. package/src/daemon/conversation-attachments.ts +93 -12
  327. package/src/daemon/conversation-evictor.ts +5 -24
  328. package/src/daemon/conversation-history.ts +10 -0
  329. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  330. package/src/daemon/conversation-interrupt.ts +7 -11
  331. package/src/daemon/conversation-messaging.ts +58 -5
  332. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  333. package/src/daemon/conversation-mode-session.ts +1008 -0
  334. package/src/daemon/conversation-process.ts +5 -0
  335. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  336. package/src/daemon/conversation-skill-tools.ts +1 -0
  337. package/src/daemon/conversation-store.ts +4 -4
  338. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  339. package/src/daemon/conversation-surfaces.ts +113 -5
  340. package/src/daemon/conversation-tool-setup.ts +79 -6
  341. package/src/daemon/conversation-turn-finalize.ts +26 -13
  342. package/src/daemon/conversation.ts +215 -58
  343. package/src/daemon/daemon-control.ts +3 -7
  344. package/src/daemon/handlers/config-model.ts +7 -5
  345. package/src/daemon/handlers/shared.ts +55 -16
  346. package/src/daemon/host-cu-proxy.ts +51 -22
  347. package/src/daemon/lifecycle.ts +30 -13
  348. package/src/daemon/mcp-reload-service.ts +1 -1
  349. package/src/daemon/message-protocol.ts +0 -1
  350. package/src/daemon/message-provenance.ts +28 -4
  351. package/src/daemon/message-types/shared.ts +1 -0
  352. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  353. package/src/daemon/mode-session-tracking.ts +54 -0
  354. package/src/daemon/orphan-reaper.ts +4 -3
  355. package/src/daemon/persist-media-references.ts +20 -3
  356. package/src/daemon/process-message.ts +13 -4
  357. package/src/daemon/tool-setup-types.ts +6 -0
  358. package/src/daemon/virtual-desktop-context.ts +21 -0
  359. package/src/daemon/wake-conversation-ops.ts +50 -15
  360. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  361. package/src/desktop/desktop-accessibility-script.ts +88 -0
  362. package/src/desktop/desktop-accessibility.test.ts +160 -0
  363. package/src/desktop/desktop-accessibility.ts +169 -0
  364. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  365. package/src/desktop/desktop-automation-lease.ts +129 -54
  366. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  367. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  368. package/src/desktop/desktop-computer-use.test.ts +449 -0
  369. package/src/desktop/desktop-computer-use.ts +502 -0
  370. package/src/desktop/desktop-dependencies.ts +47 -293
  371. package/src/desktop/desktop-display.ts +2 -2
  372. package/src/desktop/desktop-help.ts +15 -0
  373. package/src/desktop/desktop-panel-config.test.ts +13 -3
  374. package/src/desktop/desktop-panel-config.ts +109 -15
  375. package/src/desktop/desktop-session-manager.test.ts +69 -21
  376. package/src/desktop/desktop-session-manager.ts +76 -22
  377. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  378. package/src/desktop/desktop-stream-bridge.ts +9 -11
  379. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  380. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  381. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  382. package/src/desktop/desktop-wallpaper.ts +34 -102
  383. package/src/desktop/desktop-window-theme.ts +5 -25
  384. package/src/desktop/virtual-desktop-feature.ts +53 -3
  385. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  386. package/src/documents/document-store.ts +23 -0
  387. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  388. package/src/heartbeat/heartbeat-service.ts +5 -33
  389. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  390. package/src/ipc/assistant-server.ts +2 -0
  391. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  392. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  393. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  394. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  395. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  396. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  397. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  398. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  399. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  400. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  401. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  402. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  403. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  404. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  405. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  406. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  407. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  408. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  409. package/src/live-voice/camera-mode-session.test.ts +265 -0
  410. package/src/live-voice/camera-mode-session.ts +277 -0
  411. package/src/live-voice/continuation-judge.ts +123 -0
  412. package/src/live-voice/live-activity-reporter.ts +115 -6
  413. package/src/live-voice/live-voice-manager.ts +36 -1
  414. package/src/live-voice/live-voice-photo.ts +88 -7
  415. package/src/live-voice/live-voice-session-manager.ts +15 -0
  416. package/src/live-voice/live-voice-session.ts +1443 -747
  417. package/src/live-voice/live-voice-tts.ts +22 -4
  418. package/src/live-voice/microphone-idle-gate.ts +56 -0
  419. package/src/live-voice/protocol.ts +171 -4
  420. package/src/live-voice/session-controls.ts +11 -3
  421. package/src/live-voice/task-outcomes.ts +131 -0
  422. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  423. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  424. package/src/mcp/__tests__/startup.test.ts +30 -7
  425. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  426. package/src/mcp/manager.ts +171 -101
  427. package/src/mcp/startup.ts +33 -13
  428. package/src/mcp/tool-caps.ts +184 -0
  429. package/src/media/__tests__/image-models.test.ts +58 -0
  430. package/src/media/image-credentials.ts +5 -0
  431. package/src/media/image-models.ts +52 -0
  432. package/src/media/image-service.ts +8 -0
  433. package/src/media/openrouter-image-service.test.ts +133 -0
  434. package/src/media/openrouter-image-service.ts +153 -0
  435. package/src/media/types.ts +1 -1
  436. package/src/messaging/provider-message-metadata.ts +3 -3
  437. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  438. package/src/messaging/providers/discord/send.ts +4 -17
  439. package/src/messaging/providers/discord/transport.ts +2 -2
  440. package/src/messaging/providers/outlook/client.ts +16 -22
  441. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  442. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  443. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  444. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  445. package/src/monitoring/__tests__/control.test.ts +17 -5
  446. package/src/monitoring/control.ts +15 -5
  447. package/src/notifications/AGENTS.md +1 -1
  448. package/src/notifications/README.md +83 -99
  449. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  450. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  451. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  452. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  453. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  454. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  455. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  456. package/src/notifications/adapters/macos.ts +37 -46
  457. package/src/notifications/broadcaster.ts +5 -6
  458. package/src/notifications/channel-allowlist.ts +44 -0
  459. package/src/notifications/copy-composer.ts +11 -26
  460. package/src/notifications/decision-engine.ts +24 -14
  461. package/src/notifications/deterministic-checks.ts +1 -1
  462. package/src/notifications/emit-signal.ts +123 -37
  463. package/src/notifications/home-feed-side-effect.ts +9 -7
  464. package/src/notifications/notification-utils.ts +19 -0
  465. package/src/notifications/schedule-result-producer.ts +4 -6
  466. package/src/notifications/signal.ts +4 -19
  467. package/src/notifications/types.ts +15 -0
  468. package/src/oauth/AGENTS.md +2 -0
  469. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  470. package/src/oauth/connection-resolver.ts +1 -2
  471. package/src/oauth/identity-verifier.ts +24 -0
  472. package/src/oauth/oauth-store.ts +12 -4
  473. package/src/oauth/provider-serializer.ts +8 -0
  474. package/src/oauth/seed-providers.ts +42 -0
  475. package/src/onboarding/onboarding-events-store.ts +99 -12
  476. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  477. package/src/permissions/confirmation-guardian-request.ts +2 -3
  478. package/src/permissions/question-prompter.test.ts +31 -0
  479. package/src/permissions/question-prompter.ts +2 -0
  480. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  481. package/src/persistence/attachments-store.ts +84 -44
  482. package/src/persistence/client-connection-events-store.ts +377 -0
  483. package/src/persistence/conversation-crud.ts +166 -26
  484. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  485. package/src/persistence/conversation-mode-sessions.ts +460 -0
  486. package/src/persistence/conversation-plugin-facade.ts +30 -0
  487. package/src/persistence/conversation-queries.ts +6 -19
  488. package/src/persistence/conversation-tool-surface.ts +135 -0
  489. package/src/persistence/conversation-types.test.ts +32 -0
  490. package/src/persistence/conversation-types.ts +26 -4
  491. package/src/persistence/llm-usage-store.ts +3 -9
  492. package/src/persistence/message-metadata.ts +25 -0
  493. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  494. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  495. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  496. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  497. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  498. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  499. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  500. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  501. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  502. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  503. package/src/persistence/schema/conversations.ts +67 -0
  504. package/src/persistence/schema/index.ts +1 -0
  505. package/src/persistence/schema/infrastructure.ts +29 -0
  506. package/src/persistence/schema/oauth.ts +1 -0
  507. package/src/persistence/steps.ts +41 -3
  508. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  509. package/src/plugin-api/conversation-turn.ts +31 -7
  510. package/src/plugin-api/effective-context-window.ts +33 -0
  511. package/src/plugin-api/index.ts +35 -6
  512. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  513. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  514. package/src/plugin-api/plugin-name-env.ts +125 -0
  515. package/src/plugin-api/resolve-credential.ts +79 -20
  516. package/src/plugin-api/vision-support.test.ts +22 -0
  517. package/src/plugin-api/vision-support.ts +17 -5
  518. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  519. package/src/plugins/active-plugin-env.ts +51 -0
  520. package/src/plugins/defaults/index.ts +24 -0
  521. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  522. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  523. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  524. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  525. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  526. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  527. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  528. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  529. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  530. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  531. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  532. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  533. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  534. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  535. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  536. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  537. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  538. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  539. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  540. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  541. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  542. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  543. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  544. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  545. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  546. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  547. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  548. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  549. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  550. package/src/plugins/defaults/memory/injectors.ts +4 -3
  551. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  552. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  553. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  554. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  555. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  556. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  557. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  558. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  559. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  560. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  561. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  562. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  563. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  564. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  565. package/src/plugins/defaults/memory/tools.ts +38 -11
  566. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  567. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  568. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  569. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  570. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  571. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  572. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  574. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  575. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  576. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  578. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  579. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  580. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  581. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  582. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  583. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  584. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  585. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  586. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  587. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  588. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  589. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  590. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  591. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  592. package/src/plugins/defaults/task-progress/package.json +14 -0
  593. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  594. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  595. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  596. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  597. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  598. package/src/prompts/delegation-gate.ts +57 -0
  599. package/src/prompts/system-prompt.ts +14 -32
  600. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  601. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  602. package/src/providers/anthropic/client.ts +10 -1
  603. package/src/providers/anthropic-gateway-shared.ts +6 -0
  604. package/src/providers/fetch-provider-catalog.ts +11 -0
  605. package/src/providers/gemini/client.ts +25 -8
  606. package/src/providers/inference/adapter-factory.ts +6 -0
  607. package/src/providers/jev/client.test.ts +270 -0
  608. package/src/providers/jev/client.ts +534 -0
  609. package/src/providers/model-catalog.ts +59 -3
  610. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  611. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  612. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  613. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  614. package/src/providers/openai/chat-completions-provider.ts +100 -29
  615. package/src/providers/openai/google-thought-signature.ts +0 -6
  616. package/src/providers/openai/responses-provider.ts +23 -1
  617. package/src/providers/ratelimit.ts +1 -3
  618. package/src/providers/search-provider-catalog.ts +24 -0
  619. package/src/providers/server-tool-pairing.ts +16 -7
  620. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  621. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  622. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  623. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  624. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  625. package/src/providers/types.ts +5 -0
  626. package/src/routes/control.ts +7 -5
  627. package/src/runtime/AGENTS.md +6 -4
  628. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  629. package/src/runtime/actor-trust-resolver.ts +3 -3
  630. package/src/runtime/agent-wake.ts +32 -5
  631. package/src/runtime/approval-message-composer.ts +3 -69
  632. package/src/runtime/assistant-event-hub.ts +73 -75
  633. package/src/runtime/assistant-event-publish-options.ts +7 -0
  634. package/src/runtime/assistant-event-targeting.ts +79 -0
  635. package/src/runtime/assistant-stream-state.ts +13 -100
  636. package/src/runtime/background-job-runner.ts +5 -5
  637. package/src/runtime/channel-retry-sweep.ts +1 -0
  638. package/src/runtime/guardian-action-message-composer.ts +8 -161
  639. package/src/runtime/guardian-action-service.ts +2 -17
  640. package/src/runtime/guardian-reply-router.ts +29 -40
  641. package/src/runtime/http-server.ts +2 -2
  642. package/src/runtime/http-types.ts +6 -0
  643. package/src/runtime/message-composer-types.ts +3 -61
  644. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  645. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  646. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  647. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  648. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  649. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  650. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  651. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  652. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  653. package/src/runtime/pending-interactions.ts +29 -0
  654. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  655. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  656. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  657. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  658. package/src/runtime/routes/browser-context.ts +31 -11
  659. package/src/runtime/routes/browser-routes.ts +43 -4
  660. package/src/runtime/routes/btw-routes.ts +9 -4
  661. package/src/runtime/routes/channel-route-shared.ts +1 -9
  662. package/src/runtime/routes/client-routes.ts +108 -0
  663. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  664. package/src/runtime/routes/conversation-routes.ts +125 -14
  665. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  666. package/src/runtime/routes/credential-routes.ts +1 -2
  667. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  668. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  669. package/src/runtime/routes/documents-routes.ts +51 -0
  670. package/src/runtime/routes/email-routes.ts +3 -7
  671. package/src/runtime/routes/events-routes.ts +75 -54
  672. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  673. package/src/runtime/routes/identity-routes.ts +1 -5
  674. package/src/runtime/routes/image-generation-routes.ts +8 -14
  675. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  676. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  677. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  678. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  679. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  680. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  681. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  682. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  683. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  684. package/src/runtime/routes/migration-routes.ts +73 -2
  685. package/src/runtime/routes/notification-routes.ts +10 -0
  686. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  687. package/src/runtime/routes/oauth-providers.ts +11 -0
  688. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  689. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  690. package/src/runtime/routes/plugins-routes.ts +6 -0
  691. package/src/runtime/routes/roadmap-routes.ts +47 -8
  692. package/src/runtime/routes/schedule-routes.ts +2 -0
  693. package/src/runtime/routes/secret-routes.ts +99 -119
  694. package/src/runtime/routes/settings-routes.ts +10 -4
  695. package/src/runtime/routes/watcher-routes.ts +1 -3
  696. package/src/runtime/trust-class.test.ts +7 -6
  697. package/src/runtime/trust-class.ts +9 -17
  698. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  699. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  700. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  701. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  702. package/src/schedule/run-script.ts +2 -2
  703. package/src/schedule/scheduler.ts +47 -11
  704. package/src/schedule/tool-surface-readiness.ts +65 -0
  705. package/src/schedule/worker-control.ts +15 -5
  706. package/src/schedule/worker-mcp.ts +85 -0
  707. package/src/schedule/worker.ts +9 -31
  708. package/src/security/secure-keys.ts +74 -112
  709. package/src/skills/clawhub.ts +3 -8
  710. package/src/skills/managed-store.ts +1 -5
  711. package/src/stt/types.ts +4 -0
  712. package/src/subagent/notify.ts +54 -34
  713. package/src/subagent/parent-notification.ts +6 -0
  714. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  715. package/src/telemetry/phone-call-funnel.ts +144 -0
  716. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  717. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  718. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  719. package/src/tools/browser/browser-execution.ts +29 -60
  720. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  721. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  722. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  723. package/src/tools/client-os.ts +15 -0
  724. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  725. package/src/tools/computer-use/target.ts +42 -0
  726. package/src/tools/document/document-tool.ts +13 -18
  727. package/src/tools/execution-target.ts +14 -16
  728. package/src/tools/host-terminal/host-shell.ts +16 -7
  729. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  730. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  731. package/src/tools/network/web-fetch.ts +274 -7
  732. package/src/tools/network/web-search.ts +468 -6
  733. package/src/tools/permission-checker.ts +3 -3
  734. package/src/tools/policy-context.ts +3 -1
  735. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  736. package/src/tools/shared/shell-output.test.ts +10 -0
  737. package/src/tools/shared/shell-output.ts +14 -2
  738. package/src/tools/skills/load.ts +2 -1
  739. package/src/tools/skills/sandbox-runner.ts +24 -4
  740. package/src/tools/skills/scaffold-managed.ts +2 -1
  741. package/src/tools/skills/skill-script-runner.ts +7 -0
  742. package/src/tools/skills/skill-tool-factory.ts +15 -1
  743. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  744. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  745. package/src/tools/terminal/safe-env.ts +39 -24
  746. package/src/tools/terminal/sanitized-bash.ts +15 -2
  747. package/src/tools/terminal/shell-launch.test.ts +162 -0
  748. package/src/tools/terminal/shell.test.ts +29 -0
  749. package/src/tools/terminal/shell.ts +15 -7
  750. package/src/tools/tool-approval-handler.ts +5 -1
  751. package/src/tools/tool-types.ts +3 -3
  752. package/src/tools/types.ts +8 -2
  753. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  754. package/src/util/abort-reasons.ts +18 -26
  755. package/src/util/browser-human-verification.ts +14 -0
  756. package/src/util/canonicalize-identity.test.ts +21 -0
  757. package/src/util/canonicalize-identity.ts +16 -0
  758. package/src/util/errors.ts +13 -3
  759. package/src/util/host-process.test.ts +17 -1
  760. package/src/util/host-process.ts +24 -0
  761. package/src/util/provider-error-patterns.ts +3 -0
  762. package/src/util/unicode.ts +29 -0
  763. package/src/util/worker-process.ts +77 -35
  764. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  765. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  766. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  767. package/src/workspace/migrations/registry.ts +2 -0
  768. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  769. package/src/api/events/heartbeat-alert.ts +0 -20
  770. package/src/daemon/message-types/web-activity.ts +0 -65
  771. package/src/desktop/desktop-dependencies.test.ts +0 -131
  772. package/src/tools/computer-use/definitions.ts +0 -607
@@ -1,5 +1,6 @@
1
1
  import { Buffer } from "node:buffer";
2
2
  import { randomUUID } from "node:crypto";
3
+ import { setTimeout as delay } from "node:timers/promises";
3
4
 
4
5
  import {
5
6
  type BargeInGuard,
@@ -26,17 +27,29 @@ import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
26
27
  import {
27
28
  createControlMarkerHoldback,
28
29
  TASK_STOP_MARKER,
30
+ TASK_UPDATE_SILENT_MARKER,
29
31
  } from "../calls/voice-control-protocol.js";
32
+ import type { VoiceEscalationTarget } from "../calls/voice-escalation-target.js";
30
33
  import {
31
34
  createFrontDoorLegCoordinator,
32
35
  type FrontDoorLegCoordinator,
33
36
  type SpokenEscalationBridge,
34
37
  } from "../calls/voice-leg-coordinator.js";
38
+ import {
39
+ getVoiceMetricsAggregateFields,
40
+ type VoiceEndpointAction,
41
+ type VoiceEndpointSource,
42
+ type VoiceMetricsClock,
43
+ VoiceMetricsCollector,
44
+ type VoiceMetricsEvent,
45
+ type VoiceTurnSeedMarks,
46
+ } from "../calls/voice-metrics.js";
35
47
  import {
36
48
  createProgressCadence,
37
49
  type ProgressCadence,
38
50
  } from "../calls/voice-progress-cadence.js";
39
51
  import type {
52
+ VoiceTurnCallbacks,
40
53
  VoiceTurnHandle,
41
54
  VoiceTurnOptions,
42
55
  } from "../calls/voice-session-bridge.js";
@@ -56,6 +69,10 @@ import {
56
69
  VoiceFrontModelConfigSchema,
57
70
  } from "../config/schemas/voice.js";
58
71
  import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
72
+ import type {
73
+ ConversationModeSessionCoordinator,
74
+ ModeSessionSourceHandle,
75
+ } from "../daemon/conversation-mode-session.js";
59
76
  import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
60
77
  import type { TrustContext } from "../daemon/trust-context-types.js";
61
78
  import {
@@ -95,6 +112,7 @@ import type {
95
112
  SttStreamServerEvent,
96
113
  } from "../stt/types.js";
97
114
  import { getSubagentManager } from "../subagent/index.js";
115
+ import type { SubagentParentNotification } from "../subagent/parent-notification.js";
98
116
  import {
99
117
  liveVoiceEndScreen,
100
118
  liveVoiceSilenceReason,
@@ -116,6 +134,12 @@ import {
116
134
  dismissesUiSurface,
117
135
  revealsUiSurface,
118
136
  } from "./activity-label.js";
137
+ import { CameraModeSessionProducer } from "./camera-mode-session.js";
138
+ import {
139
+ type ContinuationJudgement,
140
+ createContinuationJudge,
141
+ type LiveVoiceContinuationJudge,
142
+ } from "./continuation-judge.js";
119
143
  import {
120
144
  buildDuplexContinuationLabel,
121
145
  createContinuationLabeler,
@@ -128,15 +152,7 @@ import type {
128
152
  } from "./live-voice-archive.js";
129
153
  import type { LiveVoiceCredentialReadiness } from "./live-voice-credential-preflight.js";
130
154
  import {
131
- getLiveVoiceMetricsAggregateFields,
132
- type LiveVoiceMetricsClock,
133
- LiveVoiceMetricsCollector,
134
- type LiveVoiceMetricsEvent,
135
- type LiveVoiceTurnSeedMarks,
136
- type VoiceEndpointAction,
137
- type VoiceEndpointSource,
138
- } from "./live-voice-metrics.js";
139
- import {
155
+ type LiveVoicePhotoResult,
140
156
  persistAmbientSightFrame,
141
157
  persistLiveVoicePhoto,
142
158
  } from "./live-voice-photo.js";
@@ -151,6 +167,7 @@ import type {
151
167
  LiveVoiceTtsOptions,
152
168
  LiveVoiceTtsResult,
153
169
  } from "./live-voice-tts.js";
170
+ import { MicrophoneIdleGate } from "./microphone-idle-gate.js";
154
171
  import {
155
172
  type LiveVoiceClientAttachFrameFrame,
156
173
  type LiveVoiceClientAttachImageFrame,
@@ -167,12 +184,15 @@ import {
167
184
  isLookSessionControl,
168
185
  LOOK_FOLLOW_UP_CONTENT,
169
186
  LOOK_FRAME_REASON,
187
+ type LookFollowUp,
170
188
  lookFollowUpNote,
171
189
  type LookSessionControl,
172
190
  progressConfigForCadence,
173
191
  requestedSessionControl,
174
192
  sessionControlTeaching,
175
193
  } from "./session-controls.js";
194
+ import { type VoiceTaskOutcome, VoiceTaskOutcomes } from "./task-outcomes.js";
195
+ import { VoiceInputDiagnostics } from "./voice-input-diagnostics.js";
176
196
 
177
197
  const log = getLogger("live-voice-session");
178
198
 
@@ -263,25 +283,22 @@ const PROVIDER_TURN_END_FALLBACK_MARGIN_MS = 1_000;
263
283
  // a segment is at most ~180 chars of speech (~10 s of 24 kHz mono PCM
264
284
  // ≈ 480 KB), so one buffered segment is an acceptable bound.
265
285
  const TTS_MAX_OPEN_SYNTHESIS_JOBS = 2;
286
+ const TTS_MAX_PLAYBACK_LEAD_MS = 500;
266
287
  // Audible silence required before a finished background continuation's result
267
288
  // is spoken into a live call. Long enough that the announcement lands in a real
268
289
  // lull rather than on the heels of the turn that just ended; short enough that
269
290
  // the user is not left wondering whether the work survived.
270
291
  const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
271
- // How many times a pending announcement re-arms itself against a client
272
- // playback tail that outlasted its timer. Each re-arm waits out the whole
273
- // remaining tail, so one covers a reply of any length; the rest cover a fresh
274
- // reply's audio landing between two checks. Beyond the cap the announcement
275
- // falls back to the stash rather than chasing a call that keeps talking.
276
- const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
292
+ // How long a barge-in's continuation waits for the interrupting words before
293
+ // the continuation judge gives up on them and the continuation proceeds.
294
+ // Covers the caller finishing the sentence and the utterance releasing.
295
+ const BARGE_IN_INTERRUPTION_WAIT_MS = 4_000;
277
296
  const FOREGROUND_TASK_RESUME_SILENCE_MS = 1_500;
278
297
  const FOREGROUND_TASK_MAX_SUSPENDED_MS = 120_000;
279
298
  const FOREGROUND_TASK_MAX_INTERVENING_TURNS = 3;
280
- // `content` of an announcement turn. The answer never rides here — it goes in
281
- // the model-facing control prompt (buildLiveDeliveryNote), so the only thing
282
- // persisted on the user side is this marker, and it persists hidden.
299
+ // Hidden content for continuation announcements; findings travel in the control prompt.
283
300
  export const CONTINUATION_DELIVERY_CONTENT =
284
- "(background work finished — deliver it now)";
301
+ "(background work finished: deliver it now)";
285
302
  // How long a look control waits for the fresh frame the client takes for it.
286
303
  // Long enough for a camera that has to open and warm up before its first keep;
287
304
  // past it the look is dropped rather than answered into a later, unrelated
@@ -364,10 +381,14 @@ export interface LiveVoiceSessionOptions {
364
381
  */
365
382
  resolveCredentialReadiness?: LiveVoiceCredentialReadinessResolver | null;
366
383
  startVoiceTurn?: LiveVoiceTurnStarter;
384
+ acquireModeSessionResidency?: (conversationId: string) => Promise<{
385
+ coordinator: ConversationModeSessionCoordinator;
386
+ release: () => void;
387
+ }>;
367
388
  streamTtsAudio?: LiveVoiceTtsStreamer | null;
368
389
  archiveAudio?: LiveVoiceSessionAudioArchiver | null;
369
390
  emitMetrics?: boolean;
370
- metricsClock?: LiveVoiceMetricsClock;
391
+ metricsClock?: VoiceMetricsClock;
371
392
  /**
372
393
  * Mirrors phase changes to the iOS Live Activity. Injectable so tests can
373
394
  * assert what a session reports without reaching the platform.
@@ -448,6 +469,13 @@ export interface LiveVoiceSessionOptions {
448
469
  * Absent, the continuation carries the deterministic transcript label.
449
470
  */
450
471
  labelBackgroundContinuation?: LiveVoiceContinuationLabeler;
472
+ /**
473
+ * Decides from the caller's interruption whether the interrupted request
474
+ * should still continue in the background. The factory wires the
475
+ * TypeSafe-backed judge; tests inject a stub. Absent, every eligible
476
+ * barge-in continues, as it does without a TypeSafe profile.
477
+ */
478
+ judgeBackgroundContinuation?: LiveVoiceContinuationJudge;
451
479
  /**
452
480
  * Returns the pending teardown promise for a conversation's most recent
453
481
  * turn. The barge-in path awaits it before forking the background
@@ -493,6 +521,7 @@ type LiveVoiceUtterancePhase =
493
521
  // manual mode and a session-shared instance in persistent server-VAD mode.
494
522
  interface UtteranceCycle {
495
523
  phase: LiveVoiceUtterancePhase;
524
+ preDialBargeInGuard: BargeInGuard | null;
496
525
  released: boolean;
497
526
  assistantTurnStarted: boolean;
498
527
  // The whole cycle (turn included) finalized; the record can no longer
@@ -510,12 +539,8 @@ interface UtteranceCycle {
510
539
  // server_vad has the turn detector for the same question, and never sets
511
540
  // this — its ingress is handleServerVadAudio.
512
541
  manualAudioCaptured: boolean;
513
- // server_vad capture routed speech (not just pre-roll silence) into this
514
- // cycle. Distinguishes an eagerly re-armed cycle holding only leading
515
- // silence from one already carrying the user's utterance: the
516
- // stale-language interception in handleServerVadAudio may retire the
517
- // former, never the latter. turnId cannot answer this, because a
518
- // silence-only pre-roll flush assigns it too.
542
+ // Local VAD or the provider detected speech. A language change may retire
543
+ // only a silence-only cycle; pre-roll can assign turnId before speech.
519
544
  speechRouted: boolean;
520
545
  pendingAudioChunks: Buffer[];
521
546
  pendingAudioBytes: number;
@@ -601,6 +626,8 @@ type UtteranceStartResult =
601
626
  // client in job-list order.
602
627
  interface TtsSegmentJob {
603
628
  readonly text: string;
629
+ readonly isReply: boolean;
630
+ audioSent: boolean;
604
631
  // Per-segment language-hint override, preferred over the turn's language.
605
632
  // Set on fixed phrases whose localized table lacks the turn's language:
606
633
  // the English fallback text carries "en" so an enforcing provider never
@@ -635,6 +662,9 @@ interface ActiveAssistantTurn {
635
662
  language: string | undefined;
636
663
  abortController: AbortController;
637
664
  handle: VoiceTurnHandle | null;
665
+ modeSessionSource: ModeSessionSourceHandle | undefined;
666
+ modeSessionRequestIds: string[];
667
+ modeSessionDeliveryTurnId: string | null;
638
668
  // When the turn launched, for narration's turnElapsedMs.
639
669
  launchedAtMs: number;
640
670
  // Tool-activity log and spoken progress narration for the turn
@@ -659,6 +689,10 @@ interface ActiveAssistantTurn {
659
689
  // whole frame rather than its wording. What the CLIENT believes, as against
660
690
  // `pendingApproval`, which is what is true.
661
691
  publishedApprovalRequestId: string | null;
692
+ // Structured activity the client most recently received. TTS can
693
+ // temporarily move system surfaces to speaking without changing this
694
+ // logical activity, while a tool or approval frame clears it.
695
+ publishedActivityKind: "escalation" | null;
662
696
  // Set while the turn is blocked on a decision the user has to make, and null
663
697
  // when it is not. Suppresses progress narration, whose entire vocabulary
664
698
  // ("still on it", "almost there") describes work in flight and would be
@@ -681,6 +715,7 @@ interface ActiveAssistantTurn {
681
715
  // A tts_audio frame actually went out to the client — latches on the first
682
716
  // forwarded chunk so the firstTtsAudio metric is marked exactly once per turn.
683
717
  ttsAudioStarted: boolean;
718
+ ttsFailed: boolean;
684
719
  finalized: boolean;
685
720
  // Unified front-door speculative dispatch: the leg is in flight but its
686
721
  // leading verdict (hold vs commit) has not arrived. The thinking
@@ -718,19 +753,12 @@ interface ActiveAssistantTurn {
718
753
  // merges it with this turn's utterance instead of treating that utterance as
719
754
  // a fresh follow-up. Null for an ordinary (non-barge-in) turn.
720
755
  interruptedRequest: string | null;
721
- // The completed background continuation's answer, folded into this turn's
722
- // control prompt as context so the model can deliver or reference it in reply
723
- // to the user's own utterance. Null when no continuation result is pending.
724
- continuationResult: string | null;
725
- // Set only on an announcement turn: the interrupted request and the
726
- // continuation's answer, which the turn exists solely to deliver. The turn has
727
- // no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
728
- // the answer rides the control prompt (buildLiveDeliveryNote).
729
- continuationDelivery: ContinuationDelivery | null;
730
- // Set only on the turn that answers a look: which look it answers. The turn
731
- // has no user utterance behind it; the instruction rides the control prompt
732
- // (lookFollowUpNote).
733
- lookFollowUp: LookSessionControl | null;
756
+ contextOutcomes: VoiceTaskOutcome[];
757
+ taskOutcome: VoiceTaskOutcome | null;
758
+ taskDeliveryContext: string | null;
759
+ notificationHandledSilently: boolean;
760
+ // The fresh view and original caller request this hidden turn resumes.
761
+ lookFollowUp: LookFollowUp | null;
734
762
  // The turn's content is an internal instruction rather than user speech (the
735
763
  // greeting that opens a session, say). The row still persists and the model
736
764
  // still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
@@ -754,15 +782,8 @@ interface ActiveAssistantTurn {
754
782
  // slot while this turn was in flight. Nothing will ever send it, so it is
755
783
  // collected once the discard has removed the row still linking it.
756
784
  staleFrameId: string | null;
757
- // The queued announcement this turn consumed alongside `continuationResult`
758
- // — the two are routes for one answer, so launching consumes both. Held here
759
- // so a rolled-back speculative dispatch can re-arm it (see
760
- // restorePendingTurnContext). Null on the announcement turn itself, which
761
- // owns its delivery outright.
762
- consumedAnnouncement: ContinuationDelivery | null;
763
785
  // `detachStopGeneration` at the moment this turn consumed the pending
764
- // context. A bump since then means a stop/interrupt/supersede deliberately
765
- // dropped the continuation, so a rollback must not resurrect it.
786
+ // context. A stop invalidates that context so a rollback cannot restore it.
766
787
  pendingContextStopGeneration: number;
767
788
  // Counts assistant text deltas seen this turn. A narration generation
768
789
  // captures it at launch and discards its result if it moved: text the model
@@ -777,6 +798,9 @@ interface ActiveAssistantTurn {
777
798
  // The front-door leg's coordinator: whether it handed the turn off to the
778
799
  // escalated leg. Null until the front-door leg starts.
779
800
  frontDoor: FrontDoorLegCoordinator | null;
801
+ // Final post-hook inference target for an escalated leg. Retained so the
802
+ // neutral waiting phase can be restored after bridge audio drains.
803
+ escalationTarget: VoiceEscalationTarget | null;
780
804
  ttsBuffer: string;
781
805
  // What the caller actually hears this turn, summed over the model's own
782
806
  // segments (acks and progress narration do not count). Logged at tts_done
@@ -802,6 +826,19 @@ interface ActiveAssistantTurn {
802
826
  assistantAudioSampleRate?: number;
803
827
  }
804
828
 
829
+ function sameModeSessionSource(
830
+ a: ModeSessionSourceHandle | undefined,
831
+ b: ModeSessionSourceHandle,
832
+ ): boolean {
833
+ return (
834
+ a?.id === b.id &&
835
+ a.mode === b.mode &&
836
+ a.sourceId === b.sourceId &&
837
+ a.generation === b.generation &&
838
+ a.activation === b.activation
839
+ );
840
+ }
841
+
805
842
  // Base control prompt for every live-voice turn. Opens with the spoken-reply
806
843
  // rules shared with the phone path (spoken-reply-rules.ts): this is the only
807
844
  // place the model learns that its text is spoken, since the system prompt has
@@ -849,14 +886,10 @@ function buildInterruptionMergeNote(interruptedRequest: string): string {
849
886
  return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together. If they corrected or redirected the unfinished work, continue executing it with that correction instead of merely acknowledging it. If they clearly want the unfinished task abandoned, use the task-stop control.`;
850
887
  }
851
888
 
852
- // System-level guidance appended to the NEXT turn's control prompt after a
853
- // background continuation finished the reply the user interrupted. Folds the
854
- // completed answer in as context so the model can deliver or reference it in
855
- // its reply to whatever the user just said — this turn answers the user, so the
856
- // result is offered only if it fits. Reaches the model only; it is not a user
857
- // message and never renders as a transcript bubble.
858
- function buildResurfaceContextNote(continuationResult: string): string {
859
- return `Earlier the user interrupted you, and in the background you finished the reply they cut off. What you worked out was: "${continuationResult}". If their current message relates to it, use it to answer; otherwise you may briefly offer it or leave it aside, and do not repeat it verbatim if it no longer fits.`;
889
+ function buildResurfaceContextNote(
890
+ outcomes: readonly VoiceTaskOutcome[],
891
+ ): string {
892
+ return `Answer the user's current message. These pending background reports are context, not new requests. Use their findings if relevant; otherwise leave them for a separate announcement. Reading them does not acknowledge delivery. Pending results:\n${outcomes.map((outcome) => outcome.message).join("\n\n")}`;
860
893
  }
861
894
 
862
895
  // Appended to the turn that follows a barge-in whose interrupted work was
@@ -879,19 +912,11 @@ function describeInterruptedRequest(request: string): string {
879
912
  : "their earlier request";
880
913
  }
881
914
 
882
- // A look control waiting on its fresh frame: which look, when it was asked
883
- // for, and the wait's bound.
884
- interface PendingLook {
885
- action: LookSessionControl;
915
+ // A bounded wait to resume the caller request from a fresh view.
916
+ interface PendingLook extends LookFollowUp {
886
917
  armedAtMs: number;
887
918
  timer: ReturnType<typeof setTimeout>;
888
- }
889
-
890
- // A finished background continuation waiting to be delivered: the request it
891
- // took over and the answer it produced.
892
- interface ContinuationDelivery {
893
- request: string;
894
- answer: string;
919
+ frameLanded: boolean;
895
920
  }
896
921
 
897
922
  interface OwnedForegroundTask {
@@ -914,17 +939,6 @@ interface SuspendedForegroundTask {
914
939
 
915
940
  type ForegroundTaskState = OwnedForegroundTask | SuspendedForegroundTask;
916
941
 
917
- // Appended to an announcement turn's control prompt: the turn the session
918
- // starts on its own when a background continuation finishes while the call is
919
- // live and nobody is speaking. There is no user utterance behind it, so this
920
- // note is the whole instruction and the answer rides here rather than in the
921
- // turn's `content`. Reaches the model only; it is not a user message and never
922
- // renders as a transcript bubble.
923
- function buildLiveDeliveryNote(request: string, answer: string): string {
924
- const what = describeInterruptedRequest(request);
925
- return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
926
- }
927
-
928
942
  function buildForegroundTaskResumePrompt(request: string): string {
929
943
  const task =
930
944
  request.length > 0 ? JSON.stringify(request) : "the unfinished task";
@@ -934,9 +948,8 @@ function buildForegroundTaskResumePrompt(request: string): string {
934
948
  // Assemble a leg's model-facing control prompt: the base live-voice rules, the
935
949
  // screen-reveal and setup-flow teaching (both withheld from the front-door leg,
936
950
  // see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
937
- // context, completed-continuation context, and/or the announcement instruction.
938
- // A turn can carry several (a barge-in follow-up that also has a continuation
939
- // result waiting); the notes are model-only and never render as user bubbles.
951
+ // context, pending task outcomes, and the announcement instruction. These
952
+ // notes are model-only and never render as user bubbles.
940
953
  function buildVoiceControlPrompt(
941
954
  turn: ActiveAssistantTurn,
942
955
  leg: { frontDoor?: boolean },
@@ -961,8 +974,8 @@ function buildVoiceControlPrompt(
961
974
  if (turn.interruptedRequest) {
962
975
  prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
963
976
  }
964
- if (turn.continuationResult) {
965
- prompt = `${prompt}\n\n${buildResurfaceContextNote(turn.continuationResult)}`;
977
+ if (turn.contextOutcomes.length > 0) {
978
+ prompt = `${prompt}\n\n${buildResurfaceContextNote(turn.contextOutcomes)}`;
966
979
  }
967
980
  if (turn.handedOffRequest) {
968
981
  prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
@@ -970,32 +983,24 @@ function buildVoiceControlPrompt(
970
983
  if (turn.lookFollowUp !== null) {
971
984
  prompt = `${prompt}\n\n${lookFollowUpNote(turn.lookFollowUp)}`;
972
985
  }
973
- if (turn.continuationDelivery) {
974
- prompt = `${prompt}\n\n${buildLiveDeliveryNote(
975
- turn.continuationDelivery.request,
976
- turn.continuationDelivery.answer,
977
- )}`;
986
+ if (turn.taskOutcome !== null) {
987
+ prompt += `\n\nThis turn is an internal background task update, not new words from the user. ${turn.taskDeliveryContext} Use the latest conversation context to decide what is useful now. For routine progress, findings covered by completed playback, cancelled work, or findings meant only for internal use, output only ${TASK_UPDATE_SILENT_MARKER}. An interrupted announcement or a displayed card does not establish delivery. Otherwise briefly speak the useful outcome or blocker first, before optional tool calls or a visual summary. Do not just announce completion, read raw worker status, or repeat completed actions.`;
988
+ if (turn.taskOutcome.source === "continuation") {
989
+ prompt += `\n\n${turn.taskOutcome.message}`;
990
+ }
978
991
  }
979
992
  return prompt;
980
993
  }
981
994
 
982
- // Delivered into the conversation when a continuation finishes AFTER the voice
983
- // session ended. There is no next voice turn to fold into, so this lands as a
984
- // normal turn in the thread — where the user actually goes looking for the
985
- // work. Framed as a system-style report rather than a user request so the
986
- // reply reads as "here is what I finished", not a fresh instruction.
987
- function buildClosedSessionDeliveryPrompt(
988
- interruptedRequest: string,
989
- answer: string,
990
- ): string {
991
- const what = describeInterruptedRequest(interruptedRequest);
992
- return `[Background work finished] The voice call ended before the completed result for ${what} could be delivered. Tell the user briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete. What you produced was:\n\n${answer}`;
995
+ function buildContinuationResult(request: string, answer: string): string {
996
+ return `[Background work finished] The completed result for ${describeInterruptedRequest(request)} is ready. Give the user the useful findings without re-running the work. Result:\n\n${answer}`;
993
997
  }
994
998
 
995
999
  // A fresh cycle: nothing captured, nothing released, no metrics turn open.
996
1000
  function createUtteranceCycle(): UtteranceCycle {
997
1001
  return {
998
1002
  phase: "pending",
1003
+ preDialBargeInGuard: null,
999
1004
  released: false,
1000
1005
  assistantTurnStarted: false,
1001
1006
  completed: false,
@@ -1160,6 +1165,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1160
1165
  private readonly archiveAudio: LiveVoiceSessionAudioArchiver | null;
1161
1166
  private readonly spawnBackgroundContinuation: LiveVoiceBackgroundContinuationSpawner | null;
1162
1167
  private readonly labelBackgroundContinuation: LiveVoiceContinuationLabeler | null;
1168
+ private readonly judgeBackgroundContinuation: LiveVoiceContinuationJudge | null;
1169
+ // The latest barge-in's interrupting words, resolved when the turn that
1170
+ // carries them dispatches (or null past BARGE_IN_INTERRUPTION_WAIT_MS). The
1171
+ // continuation judge reads it; a newer barge-in settles the older one null.
1172
+ private bargeInInterruption: {
1173
+ detachSeq: number;
1174
+ promise: Promise<string | null>;
1175
+ settle: (words: string | null) => void;
1176
+ } | null = null;
1177
+ // The continuation judge's pending verdict on a foreground task a barge-in
1178
+ // suspended. The resume waits on it, so a task the caller called off is
1179
+ // cleared instead of resumed.
1180
+ private foregroundTaskJudgement: {
1181
+ epoch: number;
1182
+ settled: Promise<void>;
1183
+ } | null = null;
1163
1184
  // Reads the interrupted turn's teardown promise so the barge-in path can wait
1164
1185
  // for it to settle before forking the continuation.
1165
1186
  private readonly getTurnTeardown:
@@ -1184,9 +1205,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1184
1205
  // continuation completing (before OR after it) can't surface a stale answer.
1185
1206
  private detachSequence = 0;
1186
1207
  private readonly emitMetrics: boolean;
1187
- private readonly metrics: LiveVoiceMetricsCollector;
1208
+ private readonly metrics: VoiceMetricsCollector;
1188
1209
  private readonly createTurnId: () => string;
1189
1210
  private readonly conversationId: string;
1211
+ private readonly acquireModeSessionResidency?: (
1212
+ conversationId: string,
1213
+ ) => Promise<{
1214
+ coordinator: ConversationModeSessionCoordinator;
1215
+ release: () => void;
1216
+ }>;
1217
+ private releaseModeSessionResidency?: () => void;
1218
+ private cameraModeSessions?: CameraModeSessionProducer;
1219
+ private sightFrameSequence: Promise<void> = Promise.resolve();
1190
1220
  /**
1191
1221
  * Mirrors phase changes to the iOS Live Activity through the platform, for
1192
1222
  * the case the client cannot cover: an app backgrounded long enough for iOS
@@ -1197,6 +1227,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1197
1227
  private currentUtterance: UtteranceCycle | null = null;
1198
1228
  private outboundFrames: Promise<void> = Promise.resolve();
1199
1229
  private activeAssistantTurn: ActiveAssistantTurn | null = null;
1230
+ private readonly pendingAssistantStarts = new Set<Promise<void>>();
1200
1231
  private sessionEndMetricsEmitted = false;
1201
1232
  /**
1202
1233
  * Protocol error code of the failure that killed the session, latched by
@@ -1256,6 +1287,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1256
1287
  // what echoEnergyEma is for). Raises the base gate in a noisy room so the
1257
1288
  // room itself stops reading as speech.
1258
1289
  private readonly roomNoiseFloor = new RoomNoiseFloor();
1290
+ private readonly inputDiagnostics = new VoiceInputDiagnostics();
1291
+ private readonly microphoneIdleGate: MicrophoneIdleGate;
1292
+ private diagnosticEchoCorrelation: number | null = null;
1293
+ private diagnosticTtsSampleRate: number | null = null;
1259
1294
  private readonly echoBargeInMargin: number;
1260
1295
  private readonly echoEmaHalfLifeMs: number;
1261
1296
  private readonly echoDrainSlackMs: number;
@@ -1294,6 +1329,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1294
1329
  // The shared sustained-speech accounting (gap tolerance, duty-cycle
1295
1330
  // ceiling, threshold).
1296
1331
  guard: BargeInGuard;
1332
+ armedAtMs: number;
1333
+ resets: number;
1297
1334
  } | null = null;
1298
1335
  // Estimated wall-clock ms until the client finishes draining the
1299
1336
  // assistant audio sent so far. The server clears the turn right after
@@ -1308,23 +1345,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1308
1345
  // Consumed (and cleared) when that turn launches; cleared if the barge-in
1309
1346
  // utterance is discarded, so it can never attach to a later, unrelated turn.
1310
1347
  private pendingInterruptedRequest: string | null = null;
1311
- // Set when a background continuation finishes the reply a barge-in cut off:
1312
- // its final answer, folded into the next turn the user starts as context.
1313
- // Consumed (and cleared) when that turn launches; cleared on a hard stop
1314
- // (abortDetachedRuns) so a stale result never surfaces later, and cleared by
1315
- // an announcement that delivers the same answer itself — an announcement the
1316
- // user cuts short hands it straight back.
1317
- private pendingContinuationResult: string | null = null;
1318
- // The same finished continuation, queued for an announcement turn: the
1319
- // session speaks the result on its own once the call has been quiet for
1320
- // `continuationAnnounceSilenceMs`. This and `pendingContinuationResult` are
1321
- // armed together and are two routes for ONE answer — whichever fires first
1322
- // clears the other, so the user hears it exactly once. An announcement whose
1323
- // turn fails to start, or whose turn a barge-in cuts short, hands the answer
1324
- // back to the stash, so a lost delivery costs the announcement, not the
1325
- // answer.
1326
- private pendingAnnouncement: ContinuationDelivery | null = null;
1327
- private announcementTimer: ReturnType<typeof setTimeout> | null = null;
1348
+ private readonly taskOutcomes = new VoiceTaskOutcomes();
1349
+ private taskAnnouncementTimer: ReturnType<typeof setTimeout> | null = null;
1350
+ private taskAnnouncementsDeferred = false;
1351
+ private taskOutcomesClosing = false;
1328
1352
  // Host-backed work stays on the parent conversation across barge-ins. An
1329
1353
  // owned task belongs to the tool-capable turn doing the work; a suspended
1330
1354
  // task is waiting for either a merged escalation or a synthetic resume.
@@ -1338,8 +1362,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1338
1362
  // A look control sent to a client that declared `lookFrames`, waiting for
1339
1363
  // the fresh frame the client takes for it. The session answers the look on a
1340
1364
  // turn of its own once that frame is in the conversation (see
1341
- // answerLookWhenFloorIsFree). Cleared when the frame lands, when the wait
1342
- // runs out, and when the session closes.
1365
+ // answerLookWhenFloorIsFree). Retained through the floor wait so committed
1366
+ // caller clarifications can join the request until the follow-up launches.
1343
1367
  private pendingLook: PendingLook | null = null;
1344
1368
  private lookFollowUpTimer: ReturnType<typeof setTimeout> | null = null;
1345
1369
  // Set when a continuation actually spawns: the request it took over, so the
@@ -1370,10 +1394,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1370
1394
  // The ring holds speech parked during the release→turn-start window;
1371
1395
  // protected from silent-chunk eviction until it flushes.
1372
1396
  private vadPreRollHasSpeech = false;
1397
+ private vadPreRollBargeInGuard: BargeInGuard | null = null;
1373
1398
  // Detector turn-end that fired while its speech sat parked in the ring;
1374
1399
  // replayed once the parked speech flushes into the next armed utterance.
1375
1400
  private vadPendingTurnEnd: "silence" | "max-duration" | null = null;
1376
- private readonly metricsClock: LiveVoiceMetricsClock;
1401
+ private readonly metricsClock: VoiceMetricsClock;
1377
1402
  // Persistent mode: a server-VAD session keeps one streaming transcriber for
1378
1403
  // the whole session when the stream can be sealed per utterance without
1379
1404
  // closing it. Either the provider is finalize-capable (release flushes via
@@ -1406,6 +1431,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1406
1431
  private readonly finalizeGraceMs: number;
1407
1432
  // Rotates through the progress fallback phrases across the session's turns.
1408
1433
  private progressPhraseCounter = 0;
1434
+ private modeSessionDeliveryCounter = 0;
1409
1435
  // Progress-only phrasing service. It never makes routing decisions and
1410
1436
  // never emits an answer.
1411
1437
  private readonly progressNarrator: VoiceProgressNarrator | null;
@@ -1416,17 +1442,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1416
1442
  // Complete Flux tunables (the constructor schema-parses the partial option
1417
1443
  // once, so every field carries its `liveVoice.flux` schema default).
1418
1444
  private readonly fluxConfig: LiveVoiceFluxConfig;
1419
- /**
1420
- * Per-session latch: the provider's committed end-of-turn owns the turn
1421
- * boundary instead of the silence boundary's front-door hold verdict. Set
1422
- * when the config flag is on AND the resolved streaming provider declares
1423
- * `turnDetection: "provider"` in the STT catalog AND the session runs server
1424
- * VAD. Push-to-talk is excluded deliberately: there the client's release IS
1425
- * the boundary, and answering while the caller still holds the button is not
1426
- * turn detection, it is a bug. False leaves every other code path exactly as
1427
- * it is, with no provider turn detection in the picture.
1428
- */
1429
- private providerTurnEndActive = false;
1445
+ // Provider turn starts own hands-free interruption independently of the
1446
+ // end-of-turn setting. Manual sessions keep their client-owned boundaries.
1447
+ private providerTurnStartActive = false;
1448
+ private latestProviderTurnStartIndex: number | null = null;
1449
+
1450
+ private get providerTurnEndActive(): boolean {
1451
+ return this.providerTurnStartActive && this.fluxConfig.turnEnd.enabled;
1452
+ }
1430
1453
  // Wall-clock of the newest above-gate audio chunk, tracked in every
1431
1454
  // server-VAD session. It is the local VAD's speech-stop mark: the one anchor
1432
1455
  // the reported end-of-turn latency is measured from whichever decider
@@ -1464,6 +1487,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1464
1487
  options: LiveVoiceSessionOptions = {},
1465
1488
  ) {
1466
1489
  this.context = context;
1490
+ this.microphoneIdleGate = new MicrophoneIdleGate(
1491
+ context.startFrame.audio.sampleRate,
1492
+ );
1467
1493
  this.resolveTranscriber =
1468
1494
  options.resolveTranscriber ?? defaultResolveStreamingTranscriber;
1469
1495
  this.resolveCredentialReadiness =
@@ -1475,6 +1501,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1475
1501
  options.spawnBackgroundContinuation ?? null;
1476
1502
  this.labelBackgroundContinuation =
1477
1503
  options.labelBackgroundContinuation ?? null;
1504
+ this.judgeBackgroundContinuation =
1505
+ options.judgeBackgroundContinuation ?? null;
1478
1506
  this.getTurnTeardown = options.getTurnTeardown ?? null;
1479
1507
  this.detachTeardownSettleTimeoutMs =
1480
1508
  options.detachTeardownSettleTimeoutMs ??
@@ -1493,11 +1521,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1493
1521
  this.createTurnId = options.createTurnId ?? randomUUID;
1494
1522
  this.conversationId =
1495
1523
  context.startFrame.conversationId ?? context.sessionId;
1524
+ this.acquireModeSessionResidency = options.acquireModeSessionResidency;
1496
1525
  this.liveActivityReporter =
1497
1526
  options.liveActivityReporter ??
1498
1527
  new LiveActivityReporter(this.conversationId);
1499
1528
  this.metricsClock = options.metricsClock ?? Date.now;
1500
- this.metrics = new LiveVoiceMetricsCollector({
1529
+ this.metrics = new VoiceMetricsCollector({
1501
1530
  sessionId: context.sessionId,
1502
1531
  conversationId: this.conversationId,
1503
1532
  ...(options.metricsClock ? { clock: options.metricsClock } : {}),
@@ -1611,6 +1640,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1611
1640
  // surfaces as a non-recoverable error frame instead of a start rejection.
1612
1641
  this.state = "active";
1613
1642
  this.reachedActive = true;
1643
+ if (this.turnDetector) {
1644
+ this.logInputDiagnostic("voice_input_started");
1645
+ }
1614
1646
  void this.armUtterance().catch(() => {});
1615
1647
  this.metrics.markReady();
1616
1648
  await this.sendFrame({
@@ -1622,8 +1654,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1622
1654
  // is a property of the daemon and not of what this client asked for.
1623
1655
  // A client reading it back absent is talking to one that predates it.
1624
1656
  textInput: true,
1657
+ ...(this.acquireModeSessionResidency ? { sightSessions: true } : {}),
1625
1658
  ...(this.audioInput ? {} : { audioInput: false }),
1626
1659
  });
1660
+ this.scheduleTaskAnnouncement();
1627
1661
  }
1628
1662
 
1629
1663
  async handleClientFrame(frame: LiveVoiceClientFrame): Promise<void> {
@@ -1654,8 +1688,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1654
1688
  case "attach_frame":
1655
1689
  this.parkTurnFrame(frame);
1656
1690
  return;
1691
+ case "sight_start":
1692
+ await this.sequenceSightFrame(frame);
1693
+ return;
1694
+ case "sight_end":
1695
+ await this.sequenceSightFrame(frame);
1696
+ return;
1657
1697
  case "sight_frame":
1658
- this.persistSightFrame(frame);
1698
+ await this.sequenceSightFrame(frame);
1659
1699
  return;
1660
1700
  case "text":
1661
1701
  await this.handleTextTurn(frame);
@@ -1663,6 +1703,35 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1663
1703
  }
1664
1704
  }
1665
1705
 
1706
+ private sequenceSightFrame(
1707
+ frame: Extract<
1708
+ LiveVoiceClientFrame,
1709
+ { type: "sight_start" | "sight_end" | "sight_frame" }
1710
+ >,
1711
+ ): Promise<void> {
1712
+ const operation = this.sightFrameSequence.then(async () => {
1713
+ if (this.isClosed || this.state === "failed") {
1714
+ if (frame.type === "sight_frame") {
1715
+ this.reclaimParkedFrame(frame.attachmentId);
1716
+ }
1717
+ return;
1718
+ }
1719
+ switch (frame.type) {
1720
+ case "sight_start":
1721
+ await this.startSightSession(frame.cameraEpoch, frame.source);
1722
+ return;
1723
+ case "sight_end":
1724
+ this.endSightSession(frame.cameraEpoch);
1725
+ return;
1726
+ case "sight_frame":
1727
+ this.persistSightFrame(frame);
1728
+ return;
1729
+ }
1730
+ });
1731
+ this.sightFrameSequence = operation.catch(() => {});
1732
+ return operation;
1733
+ }
1734
+
1666
1735
  /**
1667
1736
  * Run a turn the user typed instead of spoke.
1668
1737
  *
@@ -1769,6 +1838,24 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1769
1838
  */
1770
1839
  private persistSightFrame(frame: LiveVoiceClientSightFrameFrame): void {
1771
1840
  const receivedAtMs = Date.now();
1841
+ const cameraToken =
1842
+ frame.cameraEpoch === undefined
1843
+ ? undefined
1844
+ : this.cameraModeSessions?.beginFrame(frame.cameraEpoch, frame.source);
1845
+ if (frame.cameraEpoch !== undefined && !cameraToken) {
1846
+ deleteOrphanAttachments([frame.attachmentId]);
1847
+ if (!this.isClosed) {
1848
+ void this.sendFrame({
1849
+ type: "error",
1850
+ code: LiveVoiceProtocolErrorCode.InvalidFrame,
1851
+ message: "That camera run is no longer active.",
1852
+ frameType: "sight_frame",
1853
+ attachmentId: frame.attachmentId,
1854
+ recoverable: true,
1855
+ });
1856
+ }
1857
+ return;
1858
+ }
1772
1859
  // One line per keep, written when the row lands, carrying the whole
1773
1860
  // timeline: the client leg the frame reported, the daemon leg the persist
1774
1861
  // measured, and the distance from the speech onset the frame may have
@@ -1778,11 +1865,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1778
1865
  this.lastSpeechStartedAtMs === null
1779
1866
  ? null
1780
1867
  : receivedAtMs - this.lastSpeechStartedAtMs;
1781
- void persistAmbientSightFrame(
1782
- this.conversationId,
1783
- frame.attachmentId,
1784
- "voice",
1785
- ).then((result) => {
1868
+ const settleFrame = (
1869
+ result: LiveVoicePhotoResult,
1870
+ rejection?: unknown,
1871
+ ): void => {
1872
+ if (rejection !== undefined) {
1873
+ log.warn(
1874
+ { err: rejection, attachmentId: frame.attachmentId },
1875
+ "Sight frame persistence rejected unexpectedly",
1876
+ );
1877
+ }
1878
+ try {
1879
+ this.cameraModeSessions?.finishFrame(
1880
+ cameraToken,
1881
+ result.ok && result.messageId
1882
+ ? { messageId: result.messageId, at: receivedAtMs }
1883
+ : undefined,
1884
+ );
1885
+ } catch (err) {
1886
+ log.warn(
1887
+ { err, attachmentId: frame.attachmentId },
1888
+ "Could not record camera session ownership",
1889
+ );
1890
+ }
1786
1891
  log.info(
1787
1892
  {
1788
1893
  attachmentId: frame.attachmentId,
@@ -1815,9 +1920,76 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1815
1920
  recoverable: true,
1816
1921
  });
1817
1922
  }
1923
+ };
1924
+ void persistAmbientSightFrame(
1925
+ this.conversationId,
1926
+ frame.attachmentId,
1927
+ "voice",
1928
+ undefined,
1929
+ undefined,
1930
+ cameraToken?.owner,
1931
+ ).then(
1932
+ (result) => settleFrame(result),
1933
+ (err) => settleFrame({ ok: false }, err),
1934
+ );
1935
+ }
1936
+
1937
+ private async startSightSession(
1938
+ cameraEpoch: number,
1939
+ source: "live" | "ambient" | undefined,
1940
+ ): Promise<void> {
1941
+ let acquiredProducer = false;
1942
+ if (!this.cameraModeSessions && this.acquireModeSessionResidency) {
1943
+ try {
1944
+ const residency = await this.acquireModeSessionResidency(
1945
+ this.conversationId,
1946
+ );
1947
+ if (this.isClosed || this.state === "failed") {
1948
+ residency.release();
1949
+ return;
1950
+ }
1951
+ this.cameraModeSessions = new CameraModeSessionProducer(
1952
+ residency.coordinator,
1953
+ this.context.sessionId,
1954
+ );
1955
+ this.releaseModeSessionResidency = residency.release;
1956
+ acquiredProducer = true;
1957
+ } catch (err) {
1958
+ log.warn({ err, cameraEpoch }, "Could not prepare camera session");
1959
+ }
1960
+ }
1961
+ let handle: ModeSessionSourceHandle | undefined;
1962
+ try {
1963
+ handle = this.cameraModeSessions?.start(cameraEpoch, source ?? "live");
1964
+ } catch (err) {
1965
+ log.warn({ err, cameraEpoch }, "Could not start camera session");
1966
+ }
1967
+ if (!handle && acquiredProducer) {
1968
+ this.cameraModeSessions = undefined;
1969
+ const release = this.releaseModeSessionResidency;
1970
+ this.releaseModeSessionResidency = undefined;
1971
+ release?.();
1972
+ }
1973
+ if (handle || this.isClosed) {
1974
+ return;
1975
+ }
1976
+ void this.sendFrame({
1977
+ type: "error",
1978
+ code: LiveVoiceProtocolErrorCode.InvalidFrame,
1979
+ message: "Could not start that camera run.",
1980
+ frameType: "sight_start",
1981
+ recoverable: true,
1818
1982
  });
1819
1983
  }
1820
1984
 
1985
+ private endSightSession(cameraEpoch: number): void {
1986
+ try {
1987
+ this.cameraModeSessions?.end(cameraEpoch);
1988
+ } catch (err) {
1989
+ log.warn({ err, cameraEpoch }, "Could not end camera session");
1990
+ }
1991
+ }
1992
+
1821
1993
  /**
1822
1994
  * Park a camera frame for the next turn to carry, or give up the parked one.
1823
1995
  *
@@ -1926,6 +2098,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1926
2098
  if (frame.bargeInMinSpeechMs !== undefined) {
1927
2099
  this.bargeInMinSpeechMs = frame.bargeInMinSpeechMs;
1928
2100
  }
2101
+ this.logInputDiagnostic("voice_input_config_updated");
1929
2102
  }
1930
2103
 
1931
2104
  async handleBinaryAudio(chunk: Uint8Array): Promise<void> {
@@ -1937,12 +2110,31 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1937
2110
  return;
1938
2111
  }
1939
2112
 
2113
+ try {
2114
+ this.cameraModeSessions?.close(
2115
+ reason === "client_end"
2116
+ ? { status: "completed", endReason: "camera_session_ended" }
2117
+ : { status: "interrupted", endReason: `camera_${reason}` },
2118
+ );
2119
+ } catch (err) {
2120
+ log.warn({ err, reason }, "Could not close camera session");
2121
+ }
2122
+ this.releaseModeSessionResidency?.();
2123
+ this.releaseModeSessionResidency = undefined;
2124
+
2125
+ if (this.turnDetector) {
2126
+ this.logInputDiagnostic("voice_input_closed", {
2127
+ reason,
2128
+ summary: this.inputDiagnostics.flush(),
2129
+ });
2130
+ }
2131
+
1940
2132
  // Recorded first, and independently of `shouldEmitSessionEndMetrics`
1941
2133
  // below: that flag governs the client-facing `metrics` frame, and a
1942
2134
  // failed session (which suppresses the frame) is the one whose end
1943
2135
  // telemetry matters most. Session duration downstream is this row's
1944
2136
  // `recorded_at` minus the started row's, so it must be written before the
1945
- // teardown below, which awaits a pending continuation and can run long.
2137
+ // teardown below, which waits for the active turn and can run long.
1946
2138
  const failed = this.state === "failed";
1947
2139
  // Only a session that never dispatched a turn gets a silence reason; for
1948
2140
  // every other session the question is meaningless.
@@ -1985,9 +2177,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1985
2177
 
1986
2178
  const shouldEmitSessionEndMetrics = this.state !== "failed";
1987
2179
  this.state = "closed";
2180
+ this.taskOutcomesClosing = true;
2181
+ this.taskOutcomes.interruptPlayback(Date.now());
2182
+ const turnTeardown = this.getTurnTeardown?.(this.conversationId);
1988
2183
  this.clearForegroundTask("session_closed");
1989
2184
  // Retire the island before the teardown below starts awaiting things. A
1990
- // close can take a while (a pending continuation is delivered first), and
2185
+ // close can take a while waiting for turn teardown, and
1991
2186
  // an activity left asserting "Speaking…" through it is exactly the stale
1992
2187
  // claim this reporter exists to prevent.
1993
2188
  this.liveActivityReporter.end();
@@ -1998,13 +2193,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1998
2193
  // again on its next session, so the staged row and bytes go with the call.
1999
2194
  this.reclaimParkedFrame(this.pendingTurnAttachmentId);
2000
2195
  this.pendingTurnAttachmentId = null;
2001
- // There is no longer anyone on the call to speak to, so a queued
2002
- // announcement cannot be spoken — but the answer behind it is finished
2003
- // work the user asked for, so it takes the same conversation route a
2004
- // continuation finishing after this point takes. Then the queue is
2005
- // cleared: the announcement is dead either way.
2006
- await this.deliverPendingContinuationToConversation();
2007
- this.clearContinuationAnnouncement();
2008
2196
  this.clearPendingLook();
2009
2197
  this.stopSessionTranscriber();
2010
2198
  // Detached continuations outlive the call. A deliberate `interrupt()`
@@ -2012,6 +2200,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2012
2200
  // turn to fold into, a continuation that finishes after this point delivers
2013
2201
  // into the conversation instead (see detachInterruptedTurn).
2014
2202
  await this.cancelAssistantTurn("session_closed");
2203
+ await this.deliverTaskOutcomesToConversation(turnTeardown);
2015
2204
  if (shouldEmitSessionEndMetrics) {
2016
2205
  await this.emitSessionEndMetrics();
2017
2206
  }
@@ -2073,11 +2262,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2073
2262
  // in exactly the configuration roles exist for (live voice on flux,
2074
2263
  // global on base deepgram), which reinstates the race this seed
2075
2264
  // prevents.
2076
- this.setProviderTurnEndActive(
2077
- this.fluxConfig.turnEnd.enabled &&
2078
- supportsProviderTurnDetection(
2079
- sttCatalogKeyForRole(stt, "liveVoice"),
2080
- ) &&
2265
+ this.latestProviderTurnStartIndex = null;
2266
+ this.setProviderTurnDetectionActive(
2267
+ supportsProviderTurnDetection(sttCatalogKeyForRole(stt, "liveVoice")) &&
2081
2268
  this.turnDetector !== null,
2082
2269
  );
2083
2270
  const transcriber = await this.resolveTranscriber({
@@ -2093,7 +2280,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2093
2280
  if (!transcriber) {
2094
2281
  // No stream answered, so no end-of-turn ever will: the guess above
2095
2282
  // must not outlive the dial that disproved it.
2096
- this.setProviderTurnEndActive(false);
2283
+ this.setProviderTurnDetectionActive(false);
2097
2284
  return {
2098
2285
  status: "unavailable",
2099
2286
  message: unavailableTranscriberMessage(),
@@ -2107,11 +2294,23 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2107
2294
  // change under a live session, so this normally confirms the guess; it
2108
2295
  // clears it when the dial fell back to another provider or resolved
2109
2296
  // one the config did not name.
2110
- this.setProviderTurnEndActive(
2111
- this.fluxConfig.turnEnd.enabled &&
2112
- supportsProviderTurnDetection(transcriber.providerId) &&
2297
+ const expectedProviderSpeechStart = this.providerTurnStartActive;
2298
+ this.setProviderTurnDetectionActive(
2299
+ supportsProviderTurnDetection(transcriber.providerId) &&
2113
2300
  this.turnDetector !== null,
2114
2301
  );
2302
+ // A non-Flux fallback cannot confirm the onset suppressed during the
2303
+ // dial. Preserve its guard result even if local silence already released.
2304
+ const preDialBargeInGuard = utterance.preDialBargeInGuard;
2305
+ utterance.preDialBargeInGuard = null;
2306
+ if (
2307
+ expectedProviderSpeechStart &&
2308
+ !this.providerTurnStartActive &&
2309
+ utterance.speechRouted &&
2310
+ !utterance.completed
2311
+ ) {
2312
+ this.handleSpeechStart("local", preDialBargeInGuard);
2313
+ }
2115
2314
  if (
2116
2315
  this.turnDetector &&
2117
2316
  (typeof transcriber.finalizeUtterance === "function" ||
@@ -2155,7 +2354,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2155
2354
  }
2156
2355
  // The dial threw, so the pre-dial guess is disproved the same way the
2157
2356
  // unavailable case disproves it: nothing will send an end-of-turn.
2158
- this.setProviderTurnEndActive(false);
2357
+ this.setProviderTurnDetectionActive(false);
2159
2358
  return {
2160
2359
  status: "error",
2161
2360
  message: `Live voice transcription could not be started: ${errorMessage(
@@ -2194,6 +2393,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2194
2393
  if (transcriber && this.sharedTranscriber === transcriber) {
2195
2394
  this.sharedTranscriber = null;
2196
2395
  this.sharedTranscriberLanguage = undefined;
2396
+ this.microphoneIdleGate.reset();
2197
2397
  }
2198
2398
  }
2199
2399
 
@@ -2254,6 +2454,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2254
2454
  // server_vad capability; manual sessions keep single-utterance semantics
2255
2455
  // (no speculative post-turn transcriber).
2256
2456
  private scheduleRearmAfterTurn(): void {
2457
+ this.scheduleTaskAnnouncement();
2257
2458
  if (!this.turnDetector) {
2258
2459
  return;
2259
2460
  }
@@ -2372,16 +2573,112 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2372
2573
  return;
2373
2574
  }
2374
2575
 
2375
- for (const classified of this.classifyVadEnergy(chunk)) {
2576
+ const receivedAtMs = Date.now();
2577
+ const sampleRate = this.context.startFrame.audio.sampleRate;
2578
+ const meanAmplitude = pcm16MeanAmplitude(chunk);
2579
+ const before = {
2580
+ baseThreshold: this.effectiveBaseThreshold(),
2581
+ roomNoiseFloor: this.roomNoiseFloor.floor,
2582
+ echoEnergy: this.echoEnergyEma,
2583
+ echoOnsetLapsed: this.echoOnsetLapsed,
2584
+ echoWindowMs: this.echoWindowTotalAudioMs,
2585
+ echoGuardCarryover: this.echoWindowGuardCarryover,
2586
+ echoReferenceMs: pcm16DurationMs(
2587
+ this.echoReferenceAudio.byteLength,
2588
+ sampleRate,
2589
+ ),
2590
+ playbackRemainingMs: Math.max(
2591
+ 0,
2592
+ this.assistantPlaybackTailUntilMs - receivedAtMs,
2593
+ ),
2594
+ playbackEchoPossible: this.isAssistantPlaybackEchoPossible(),
2595
+ };
2596
+ this.diagnosticEchoCorrelation = null;
2597
+ const classifiedChunks = this.classifyVadEnergy(chunk, meanAmplitude);
2598
+ const durations = { speechMs: 0, silenceMs: 0, echoMs: 0 };
2599
+ for (const classified of classifiedChunks) {
2600
+ durations[`${classified.classification}Ms`] += pcm16DurationMs(
2601
+ classified.chunk.byteLength,
2602
+ sampleRate,
2603
+ );
2604
+ }
2605
+ const summary = this.inputDiagnostics.observe({
2606
+ receivedAtMs,
2607
+ chunkMs: pcm16DurationMs(chunk.byteLength, sampleRate),
2608
+ meanAmplitude,
2609
+ ...before,
2610
+ ...durations,
2611
+ echoProbeMs: pcm16DurationMs(
2612
+ this.echoProbeChunks.reduce(
2613
+ (bytes, entry) => bytes + entry.byteLength,
2614
+ 0,
2615
+ ),
2616
+ sampleRate,
2617
+ ),
2618
+ echoCorrelation: this.diagnosticEchoCorrelation,
2619
+ });
2620
+ if (summary) {
2621
+ this.logInputDiagnostic("voice_input_window", { summary });
2622
+ }
2623
+
2624
+ for (const classified of classifiedChunks) {
2376
2625
  await this.handleClassifiedVadAudio(detector, classified);
2377
2626
  }
2378
2627
  }
2379
2628
 
2629
+ private logInputDiagnostic(
2630
+ event: string,
2631
+ details: Record<string, unknown> = {},
2632
+ ): void {
2633
+ if (!this.turnDetector) {
2634
+ return;
2635
+ }
2636
+ const turn = this.activeAssistantTurn;
2637
+ const task = this.foregroundTaskState;
2638
+ log.info(
2639
+ {
2640
+ event,
2641
+ sessionId: this.context.sessionId,
2642
+ conversationId: this.conversationId,
2643
+ speechGeneration: this.vadSpeechGeneration,
2644
+ inputTurnId: this.currentUtterance?.turnId ?? null,
2645
+ assistantTurnId: turn?.turnId ?? null,
2646
+ assistantCompleted: turn?.assistantCompleted ?? null,
2647
+ ttsAudioStarted: turn?.ttsAudioStarted ?? false,
2648
+ foregroundTaskPhase: task?.phase ?? null,
2649
+ hostToolStarted: task?.hostToolStarted ?? null,
2650
+ inputSampleRate: this.context.startFrame.audio.sampleRate,
2651
+ ttsSampleRate: this.diagnosticTtsSampleRate,
2652
+ configuredThreshold:
2653
+ this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD,
2654
+ effectiveThreshold: this.effectiveBaseThreshold(),
2655
+ roomNoiseFloor: this.roomNoiseFloor.floor,
2656
+ noiseFloorMargin: this.noiseFloorMargin,
2657
+ echoMargin: this.echoBargeInMargin,
2658
+ echoEnergy: this.echoEnergyEma,
2659
+ echoOnsetLapsed: this.echoOnsetLapsed,
2660
+ playbackRemainingMs: Math.max(
2661
+ 0,
2662
+ this.assistantPlaybackTailUntilMs - Date.now(),
2663
+ ),
2664
+ echoDrainSlackMs: this.echoDrainSlackMs,
2665
+ bargeInMinSpeechMs: this.bargeInMinSpeechMs,
2666
+ microphoneGateClosed:
2667
+ this.providerTurnEndActive && this.microphoneIdleGate.closed,
2668
+ ...details,
2669
+ },
2670
+ "Live voice input diagnostics",
2671
+ );
2672
+ }
2673
+
2380
2674
  private async handleClassifiedVadAudio(
2381
2675
  detector: MediaTurnDetector,
2382
2676
  classified: VadClassifiedChunk,
2383
2677
  ): Promise<void> {
2384
- const { chunk, classification: energyClassification } = classified;
2678
+ const { classification: energyClassification } = classified;
2679
+ let { chunk } = classified;
2680
+ const inputCycle = this.currentUtterance;
2681
+ this.trackPreDialBargeInGuard(inputCycle, classified);
2385
2682
  const hasSpeech = energyClassification === "speech";
2386
2683
  detector.onMediaChunk(hasSpeech);
2387
2684
  this.trackBargeInGuard(energyClassification, chunk);
@@ -2389,11 +2686,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2389
2686
  this.localSpeechStopAtMs = Date.now();
2390
2687
  }
2391
2688
 
2392
- // Playback echo is neither user audio nor useful pre-roll. Dropping it
2393
- // prevents the assistant's reply from reaching transcription as a ghost
2394
- // follow-up turn.
2689
+ // Flux needs elapsed audio through pauses. Confirmed playback echo keeps
2690
+ // its duration but carries no speaker content into transcription.
2395
2691
  if (energyClassification === "echo") {
2396
- return;
2692
+ if (!this.providerTurnEndActive) {
2693
+ return;
2694
+ }
2695
+ chunk = Buffer.alloc(chunk.byteLength);
2397
2696
  }
2398
2697
  // Measured past the echo gate, so a greeting heard through the speaker
2399
2698
  // cannot stand in for the user on a silent close.
@@ -2402,11 +2701,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2402
2701
  this.peakChunkAmplitude = meanAmplitude;
2403
2702
  }
2404
2703
 
2405
- // Idle mic: hold silent chunks in the bounded pre-roll instead of
2406
- // collecting or streaming them; flushed on speech onset so the
2407
- // transcriber still gets leading context ahead of the first syllable.
2408
- if (!hasSpeech && !detector.isActive) {
2409
- this.pushVadPreRoll(chunk, false);
2704
+ if (this.providerTurnEndActive) {
2705
+ const wasClosed = this.microphoneIdleGate.closed;
2706
+ chunk = this.microphoneIdleGate.process(chunk, hasSpeech);
2707
+ if (wasClosed !== this.microphoneIdleGate.closed) {
2708
+ this.logInputDiagnostic("voice_input_microphone_gate_changed");
2709
+ }
2710
+ if (chunk.byteLength === 0) {
2711
+ return;
2712
+ }
2713
+ }
2714
+
2715
+ // Locally endpointed streams keep idle audio in pre-roll.
2716
+ if (!hasSpeech && !detector.isActive && !this.providerTurnEndActive) {
2717
+ this.pushVadPreRoll(chunk, "silence");
2410
2718
  return;
2411
2719
  }
2412
2720
 
@@ -2437,7 +2745,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2437
2745
  // keeps its old-language stream and the language change applies from
2438
2746
  // the following utterance.
2439
2747
  utterance.finalTranscriptSegments.length === 0 &&
2440
- utterance.latestPartialText === null &&
2748
+ (utterance.latestPartialText?.trim() ?? "").length === 0 &&
2441
2749
  this.sharedStreamLanguageIsStale()
2442
2750
  ) {
2443
2751
  this.retireSharedTranscriberForRedial(sharedForLanguage);
@@ -2448,13 +2756,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2448
2756
  if (utterance.released || utterance.completed) {
2449
2757
  // Parked speech makes silent chunks arm-worthy too: the parked
2450
2758
  // utterance must flush without requiring more speech.
2451
- if (!hasSpeech && !this.vadPreRollHasSpeech) {
2759
+ if (
2760
+ !hasSpeech &&
2761
+ !this.vadPreRollHasSpeech &&
2762
+ !this.providerTurnEndActive
2763
+ ) {
2452
2764
  return;
2453
2765
  }
2454
2766
  if (!this.canArmNextUtterance(utterance)) {
2455
2767
  // Speech in the release→turn-start window: hold it in the pre-roll
2456
2768
  // ring so it flushes into the next utterance once it arms.
2457
- this.pushVadPreRoll(chunk, hasSpeech);
2769
+ this.pushVadPreRoll(chunk, energyClassification);
2458
2770
  return;
2459
2771
  }
2460
2772
  // Sets currentUtterance synchronously; the transcriber resolves async
@@ -2466,6 +2778,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2466
2778
  }
2467
2779
  }
2468
2780
 
2781
+ if (utterance !== inputCycle) {
2782
+ this.trackPreDialBargeInGuard(utterance, classified);
2783
+ }
2784
+
2469
2785
  // Speech is now reaching the cycle, either in this chunk or parked in
2470
2786
  // the pre-roll about to flush; read the flag before takeVadPreRoll
2471
2787
  // resets it.
@@ -2489,8 +2805,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2489
2805
  * onset is neither learned as echo nor lost. Once seeded, the EMA follows
2490
2806
  * confirmed echo while speech above the learned margin remains frozen out.
2491
2807
  */
2492
- private classifyVadEnergy(chunk: Buffer): VadClassifiedChunk[] {
2493
- const meanAmplitude = pcm16MeanAmplitude(chunk);
2808
+ private classifyVadEnergy(
2809
+ chunk: Buffer,
2810
+ meanAmplitude: number,
2811
+ ): VadClassifiedChunk[] {
2494
2812
  const chunkMs = pcm16DurationMs(
2495
2813
  chunk.byteLength,
2496
2814
  this.context.startFrame.audio.sampleRate,
@@ -2612,12 +2930,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2612
2930
  chunk.byteLength,
2613
2931
  Math.ceil((sampleRate * ECHO_CORRELATION_PROBE_MS * 2) / 1_000),
2614
2932
  );
2615
- return (
2616
- pcm16MaxNormalizedCorrelation(
2617
- chunk.subarray(0, probeByteLength),
2618
- this.echoReferenceAudio,
2619
- ) >= ECHO_CORRELATION_THRESHOLD
2933
+ const correlation = pcm16MaxNormalizedCorrelation(
2934
+ chunk.subarray(0, probeByteLength),
2935
+ this.echoReferenceAudio,
2620
2936
  );
2937
+ this.diagnosticEchoCorrelation = correlation;
2938
+ return correlation >= ECHO_CORRELATION_THRESHOLD;
2621
2939
  }
2622
2940
 
2623
2941
  private updateEchoEnergy(meanAmplitude: number, chunkMs: number): void {
@@ -2682,6 +3000,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2682
3000
  }
2683
3001
 
2684
3002
  private appendEchoReference(chunk: LiveVoiceTtsAudioChunk): void {
3003
+ this.diagnosticTtsSampleRate = chunk.sampleRate ?? null;
2685
3004
  if (
2686
3005
  chunk.contentType.split(";", 1)[0]?.trim().toLowerCase() !==
2687
3006
  "audio/pcm" ||
@@ -2704,7 +3023,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2704
3023
  utterance: UtteranceCycle,
2705
3024
  chunk: Buffer,
2706
3025
  ): Promise<void> {
2707
- this.collectUserAudio(utterance, chunk);
3026
+ // Continuous idle input belongs to the STT stream, not an unbounded
3027
+ // recording of the room before the caller starts a request.
3028
+ if (utterance.speechRouted) {
3029
+ this.collectUserAudio(utterance, chunk);
3030
+ }
2708
3031
  if (this.vadSpeechStartPending) {
2709
3032
  this.vadSpeechStartPending = false;
2710
3033
  this.markSpeechStart(utterance);
@@ -2724,7 +3047,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2724
3047
  return utterance.completed || utterance.assistantTurnStarted;
2725
3048
  }
2726
3049
 
2727
- private pushVadPreRoll(chunk: Buffer, hasSpeech: boolean): void {
3050
+ private pushVadPreRoll(
3051
+ chunk: Buffer,
3052
+ classification: VadEnergyClassification,
3053
+ ): void {
3054
+ const hasSpeech = classification === "speech";
3055
+ if (hasSpeech && !this.vadPreRollBargeInGuard) {
3056
+ this.vadPreRollBargeInGuard = createBargeInGuard(this.bargeInMinSpeechMs);
3057
+ }
3058
+ this.vadPreRollBargeInGuard?.track(
3059
+ classification,
3060
+ pcm16DurationMs(
3061
+ chunk.byteLength,
3062
+ this.context.startFrame.audio.sampleRate,
3063
+ ),
3064
+ );
2728
3065
  // A full ring never lets idle silence evict parked speech.
2729
3066
  if (
2730
3067
  !hasSpeech &&
@@ -2744,6 +3081,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2744
3081
 
2745
3082
  private takeVadPreRoll(): Buffer[] {
2746
3083
  this.vadPreRollHasSpeech = false;
3084
+ this.vadPreRollBargeInGuard = null;
2747
3085
  return this.vadPreRollChunks.splice(0);
2748
3086
  }
2749
3087
 
@@ -2753,11 +3091,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2753
3091
  // Read before takeVadPreRoll resets it: a ring holding parked speech
2754
3092
  // makes this cycle speech-bearing, a silence-only ring does not.
2755
3093
  const preRollHadSpeech = this.vadPreRollHasSpeech;
3094
+ const preRollBargeInGuard = this.vadPreRollBargeInGuard;
2756
3095
  for (const chunk of this.takeVadPreRoll()) {
2757
3096
  this.collectUserAudio(utterance, chunk);
2758
3097
  this.bufferPendingUtteranceAudio(utterance, chunk);
2759
3098
  }
2760
3099
  if (preRollHadSpeech) {
3100
+ utterance.preDialBargeInGuard = preRollBargeInGuard;
2761
3101
  utterance.speechRouted = true;
2762
3102
  this.detectedSpeech = true;
2763
3103
  }
@@ -2783,10 +3123,19 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2783
3123
  chunk: Buffer,
2784
3124
  ): Promise<void> {
2785
3125
  try {
2786
- utterance.transcriber?.sendAudio(
2787
- chunk,
2788
- this.context.startFrame.audio.mimeType,
2789
- );
3126
+ if (utterance.transcriber) {
3127
+ utterance.transcriber.sendAudio(
3128
+ chunk,
3129
+ this.context.startFrame.audio.mimeType,
3130
+ );
3131
+ this.inputDiagnostics.recordSttSubmission(
3132
+ Date.now(),
3133
+ pcm16DurationMs(
3134
+ chunk.byteLength,
3135
+ this.context.startFrame.audio.sampleRate,
3136
+ ),
3137
+ );
3138
+ }
2790
3139
  await this.drainOutboundFrames();
2791
3140
  } catch (err) {
2792
3141
  await this.sendFrame({
@@ -2810,15 +3159,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2810
3159
  }
2811
3160
  }
2812
3161
 
2813
- // VAD speech onset. Contract: speech_started tells the client to flush
2814
- // tail playback immediately; barge-in then cancels any in-flight,
2815
- // non-finalized turn — including a pre-TTS "thinking" turn whose reply is
2816
- // still being generated, so a user can cut in before the assistant starts
2817
- // talking (JARVIS-1266). Speaking over a thinking or audibly speaking turn
2818
- // is deferred behind the same sustained-speech guard, so a cough or noise
2819
- // blip cannot kill an unspoken reply or clip a spoken one; sustained speech
2820
- // aborts the turn. Onset while listening keeps the instant speech_started
2821
- // (turn-taking latency is untouched).
3162
+ // Local onset tracks audio routing and silence-boundary freshness even
3163
+ // when the provider owns the decision to interrupt playback.
2822
3164
  private handleVadSpeechStart(): void {
2823
3165
  if (this.isClosed || this.state === "failed") {
2824
3166
  return;
@@ -2826,7 +3168,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2826
3168
 
2827
3169
  this.vadSpeechStartPending = true;
2828
3170
  // Speech resumed: an endpoint decision still in flight is stale (the
2829
- // generation bump defers it), and a pending hold replay is moot — the
3171
+ // generation bump defers it), and a pending hold replay is moot: the
2830
3172
  // utterance keeps accumulating and the detector fires a fresh turn-end.
2831
3173
  this.vadSpeechGeneration += 1;
2832
3174
  this.clearEndpointExtensionTimer();
@@ -2834,9 +3176,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2834
3176
  // spoke through. The next silence boundary arms a fresh one.
2835
3177
  this.clearProviderTurnEndTimer();
2836
3178
 
3179
+ if (!this.providerTurnStartActive) {
3180
+ this.handleSpeechStart("local");
3181
+ }
3182
+ }
3183
+
3184
+ private handleSpeechStart(
3185
+ source: "local" | "provider",
3186
+ preDialGuard: BargeInGuard | null = null,
3187
+ ): void {
2837
3188
  // Speech resumed while a speculative leg was awaiting its verdict: the
2838
3189
  // pause was mid-thought after all. Discard silently (no frames were ever
2839
- // sent for it) and let the utterance keep accumulating — this is the
3190
+ // sent for it) and let the utterance keep accumulating. This is the
2840
3191
  // hold outcome decided by the caller's own voice instead of the model.
2841
3192
  const speculative = this.activeAssistantTurn;
2842
3193
  if (speculative?.speculativePending) {
@@ -2849,20 +3200,37 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2849
3200
  // assistant starts talking.
2850
3201
  const bargeableTurn = turn && !turn.finalized ? turn : null;
2851
3202
  // The client can still be draining audible playback after tts_done
2852
- // (the turn is already cleared server-side) — that tail deserves the
2853
- // same guard, or a noise blip clips the reply's last words.
3203
+ // (the turn is already cleared server-side). Local onset still needs
3204
+ // the guard, or a noise blip clips the reply's last words.
2854
3205
  const drainingPlayback = this.isAssistantPlaybackEchoPossible();
2855
3206
 
2856
- if ((bargeableTurn || drainingPlayback) && this.bargeInMinSpeechMs > 0) {
2857
- // Onset audio keeps flowing into the cycle/pre-roll while the guard
2858
- // accumulates (trackBargeInGuard), so no speech is lost either way.
2859
- this.pendingBargeIn = {
2860
- turn: bargeableTurn,
2861
- guard: createBargeInGuard(this.bargeInMinSpeechMs),
2862
- };
2863
- return;
3207
+ if (
3208
+ source === "local" &&
3209
+ (bargeableTurn || drainingPlayback) &&
3210
+ this.bargeInMinSpeechMs > 0
3211
+ ) {
3212
+ const guard = preDialGuard ?? createBargeInGuard(this.bargeInMinSpeechMs);
3213
+ if (!guard.fired) {
3214
+ // A released pre-dial run below the threshold has already expired.
3215
+ if (preDialGuard && this.currentUtterance?.released) {
3216
+ return;
3217
+ }
3218
+ this.pendingBargeIn = {
3219
+ turn: bargeableTurn,
3220
+ guard,
3221
+ armedAtMs: Date.now(),
3222
+ resets: 0,
3223
+ };
3224
+ this.logInputDiagnostic("voice_input_barge_in_armed");
3225
+ return;
3226
+ }
2864
3227
  }
2865
3228
 
3229
+ this.logInputDiagnostic("voice_input_speech_started", {
3230
+ source,
3231
+ interruptsTurn: bargeableTurn !== null,
3232
+ trace: this.inputDiagnostics.snapshot(Date.now()),
3233
+ });
2866
3234
  this.pendingBargeIn = null;
2867
3235
  this.assistantPlaybackTailUntilMs = 0;
2868
3236
  this.sendSpeechStarted();
@@ -2871,11 +3239,36 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2871
3239
  }
2872
3240
  }
2873
3241
 
3242
+ private trackPreDialBargeInGuard(
3243
+ utterance: UtteranceCycle | null,
3244
+ { chunk, classification }: VadClassifiedChunk,
3245
+ ): void {
3246
+ if (
3247
+ !this.providerTurnStartActive ||
3248
+ utterance?.phase !== "pending" ||
3249
+ utterance.released ||
3250
+ utterance.transcriber !== null
3251
+ ) {
3252
+ return;
3253
+ }
3254
+ const guard = (utterance.preDialBargeInGuard ??= createBargeInGuard(
3255
+ this.bargeInMinSpeechMs,
3256
+ ));
3257
+ guard.track(
3258
+ classification,
3259
+ pcm16DurationMs(
3260
+ chunk.byteLength,
3261
+ this.context.startFrame.audio.sampleRate,
3262
+ ),
3263
+ );
3264
+ }
3265
+
2874
3266
  /**
2875
3267
  * Tell the client the caller started speaking, and remember when, so a
2876
3268
  * camera frame that follows can be logged against the onset it answers.
2877
3269
  */
2878
3270
  private sendSpeechStarted(): void {
3271
+ this.taskOutcomes.interruptPlayback(Date.now());
2879
3272
  this.lastSpeechStartedAtMs = Date.now();
2880
3273
  void this.sendFrame({ type: "speech_started" });
2881
3274
  }
@@ -2899,6 +3292,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2899
3292
  );
2900
3293
  const step = pending.guard.track(classification, chunkMs);
2901
3294
  if (step === "reset") {
3295
+ pending.resets += 1;
2902
3296
  if (this.echoWindowGuardCarryover) {
2903
3297
  this.echoWindowGuardCarryover = false;
2904
3298
  this.echoEnergyEma = 0;
@@ -2909,6 +3303,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2909
3303
  if (step !== "fired") {
2910
3304
  return;
2911
3305
  }
3306
+ this.logInputDiagnostic("voice_input_barge_in_fired", {
3307
+ armedAtMs: pending.armedAtMs,
3308
+ guardResets: pending.resets,
3309
+ accumulatedSpeechMs: pending.guard.speechMs,
3310
+ trace: this.inputDiagnostics.snapshot(Date.now()),
3311
+ });
2912
3312
  this.pendingBargeIn = null;
2913
3313
  this.assistantPlaybackTailUntilMs = 0;
2914
3314
  this.sendSpeechStarted();
@@ -2919,6 +3319,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2919
3319
  }
2920
3320
 
2921
3321
  private bargeIn(turn: ActiveAssistantTurn): void {
3322
+ this.taskOutcomes.interruptPlayback(Date.now());
2922
3323
  // Abort synchronously so no tts_audio frame can follow turn_cancelled,
2923
3324
  // and settle the cancelled turn's metrics so the next utterance's marks
2924
3325
  // do not collide with it in the collector. turn_cancelled flushes
@@ -2933,38 +3334,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2933
3334
  .trim();
2934
3335
  this.pendingInterruptedRequest =
2935
3336
  interruptedRequest.length > 0 ? interruptedRequest : null;
2936
- // A fresh interruption supersedes every earlier detached run, not just its
2937
- // stashed result: abort still-running continuations (and skip pending
2938
- // detaches) before this barge-in's own continuation can launch, so two
2939
- // full-ability background writers never share the workspace. The stashed
2940
- // result is dropped with them so the barge-in follow-up (or any later)
2941
- // turn can't consume an older answer before this barge-in's own
2942
- // continuation completes. This barge-in's detach snapshots the stop
2943
- // generation AFTER this bump (below), so it is unaffected. The abort is
2944
- // signal-level, with the same accepted residual as the foreground-wins
2945
- // gate (see the tool_use_start handler): a tool call already executing in
2946
- // the superseded run is not awaited. The replacement's detach still waits
2947
- // out the interrupted TURN's teardown before forking, which bounds the
2948
- // overlap to that one abandoned call.
3337
+ // Supersede running continuations before starting another writer. Finished
3338
+ // outcomes stay in the delivery queue. Signal cancellation does not await
3339
+ // a tool call already executing in the superseded run.
2949
3340
  this.abortDetachedRuns({ reason: "superseded_by_new_barge_in" });
2950
- // An announcement turn is the one interruption with nothing to continue:
2951
- // the work is already finished, which is why detachInterruptedTurn skips it
2952
- // as `announcement_turn`. The analog of "the interrupted work survives" is
2953
- // therefore the stash — the answer goes back into it so the user's next real
2954
- // turn still carries it, and the announcement (cut short, at best partly
2955
- // spoken) is not the delivery it was supposed to be. The restore sits after
2956
- // the abort above, which clears the stash, so nothing in this sequence
2957
- // re-drops it. Only a barge-in restores: interrupt/close/a superseding new
2958
- // continuation are resets the user asked for, and their answer stays gone.
2959
- if (turn.continuationDelivery !== null) {
2960
- this.pendingContinuationResult = turn.continuationDelivery.answer;
2961
- }
2962
3341
  // Order this barge-in among concurrent detaches SYNCHRONOUSLY, in barge
2963
3342
  // order — the actual detach runs after an async teardown chain, and those
2964
3343
  // chains can interleave, so bumping there could assign sequences out of
2965
3344
  // order and let an older continuation re-stash. A higher sequence here
2966
3345
  // immediately invalidates every earlier still-running continuation.
2967
3346
  const detachSeq = ++this.detachSequence;
3347
+ this.openBargeInInterruption(detachSeq);
3348
+ if (keepForegroundTaskOnParent) {
3349
+ this.judgeSuspendedForegroundTask(detachSeq);
3350
+ }
2968
3351
  this.clearFillerTimers(turn);
2969
3352
  // Tagged reason: provider catch-sites classify untagged caller aborts as
2970
3353
  // retryable transport failures (ERROR log + futile retry against the
@@ -3052,14 +3435,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3052
3435
  }
3053
3436
  const skipReason = this.isClosed
3054
3437
  ? "session_closed"
3055
- : // Barging in over an announcement means the user is answering it, not
3056
- // asking for it to be finished in the background — there is no pending
3057
- // request behind an announcement turn to continue. Its answer is
3058
- // already finished, so bargeIn returns it to the stash instead.
3059
- turn.continuationDelivery !== null
3438
+ : // Announcement work is finished; only its delivery remains pending.
3439
+ turn.taskOutcome !== null
3060
3440
  ? "announcement_turn"
3061
- : // Nor over the answer to a look: there is no request behind it
3062
- // either, and the user talking over it is them moving on.
3441
+ : // Look follow-ups use a transient view; a barge-in leaves any
3442
+ // unfinished host work with the parent conversation.
3063
3443
  turn.lookFollowUp !== null
3064
3444
  ? "look_follow_up"
3065
3445
  : // The model already finished generating (barge-in during TTS playback
@@ -3100,6 +3480,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3100
3480
  interruptedRequest,
3101
3481
  AbortSignal.any([controller.signal, labelAbort.signal]),
3102
3482
  );
3483
+ // Judge, in parallel with the teardown wait and the label, whether the
3484
+ // caller's interruption still wants this request finished.
3485
+ const judgementPromise = this.requestContinuationJudgement(
3486
+ interruptedRequest,
3487
+ detachSeq,
3488
+ AbortSignal.any([controller.signal, labelAbort.signal]),
3489
+ );
3103
3490
  void (async () => {
3104
3491
  try {
3105
3492
  // Wait for the interrupted turn's teardown to settle its partial into
@@ -3141,6 +3528,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3141
3528
  return;
3142
3529
  }
3143
3530
  }
3531
+ const judgement = await judgementPromise;
3532
+ if (judgement !== null && judgement.outcome !== "unavailable") {
3533
+ log.info(
3534
+ {
3535
+ turnId: turn.turnId,
3536
+ outcome: judgement.outcome,
3537
+ noul: judgement.noul,
3538
+ latencyMs: judgement.latencyMs,
3539
+ },
3540
+ "Voice duplex continuation judged",
3541
+ );
3542
+ }
3543
+ if (judgement !== null && !judgement.keep) {
3544
+ log.info(
3545
+ {
3546
+ turnId: turn.turnId,
3547
+ skipReason: "caller_moved_on",
3548
+ waitedMs: Date.now() - detachStartedAtMs,
3549
+ },
3550
+ "Voice duplex continuation skipped",
3551
+ );
3552
+ return;
3553
+ }
3144
3554
  // A closed session is NOT a reason to skip: the work outlives the
3145
3555
  // call, and its result is delivered into the conversation below.
3146
3556
  // Only an explicit invalidation (stop/interrupt/supersede) stops it.
@@ -3192,48 +3602,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3192
3602
  !controller.signal.aborted &&
3193
3603
  this.detachStopGeneration === stopGeneration &&
3194
3604
  detachSeq === this.detachSequence;
3195
- // Two destinations, decided by whether a next voice turn still exists.
3196
- // Live session: stash for the next turn the user starts, AND queue an
3197
- // announcement that speaks it into audible silence if the user says
3198
- // nothing — the two are routes for one answer and cancel each other,
3199
- // so it is delivered exactly once. Session closed: there is no next
3200
- // turn and nobody to speak to, so deliver into the conversation — the
3201
- // thread is exactly where the user goes looking for the work.
3202
- const deliverToConversation =
3203
- notInvalidated && this.isClosed && answer.length > 0;
3204
- if (notInvalidated && !this.isClosed) {
3205
- this.pendingContinuationResult = answer.length > 0 ? answer : null;
3206
- if (answer.length > 0) {
3207
- this.pendingAnnouncement = { request: interruptedRequest, answer };
3208
- this.scheduleContinuationAnnouncement();
3605
+ let accepted = false;
3606
+ if (notInvalidated && answer.length > 0) {
3607
+ const outcome: VoiceTaskOutcome = {
3608
+ taskId: `voice-continuation:${turn.turnId}`,
3609
+ message: buildContinuationResult(interruptedRequest, answer),
3610
+ metadata: {},
3611
+ source: "continuation",
3612
+ };
3613
+ accepted = this.receiveTaskOutcome(outcome);
3614
+ if (!accepted) {
3615
+ await Promise.allSettled(this.pendingAssistantStarts);
3616
+ await this.getTurnTeardown?.(this.conversationId);
3617
+ const { injectMessageIntoParent } =
3618
+ await import("../subagent/notify.js");
3619
+ injectMessageIntoParent(
3620
+ this.conversationId,
3621
+ outcome.message,
3622
+ outcome.metadata,
3623
+ { bypassLiveVoice: true },
3624
+ );
3209
3625
  }
3210
3626
  }
3211
- if (deliverToConversation) {
3212
- const { injectMessageIntoParent } =
3213
- await import("../subagent/notify.js");
3214
- injectMessageIntoParent(
3215
- this.conversationId,
3216
- buildClosedSessionDeliveryPrompt(interruptedRequest, answer),
3217
- );
3218
- }
3219
3627
  log.info(
3220
3628
  {
3221
3629
  turnId: turn.turnId,
3222
3630
  ranMs: Date.now() - detachStartedAtMs,
3223
3631
  resultChars: answer.length,
3224
- // Where the answer went. "stashed" = queued for the next voice
3225
- // turn and for an announcement into silence, whichever comes first
3226
- // (the announcement logs "announced" when it wins);
3227
- // "conversation" = the call had ended, so it was delivered into the
3228
- // thread; "dropped" = a stop/interrupt or a newer barge-in landed
3229
- // while it ran.
3230
3632
  resultDestination: !notInvalidated
3231
3633
  ? "dropped"
3232
3634
  : answer.length === 0
3233
3635
  ? "empty"
3234
- : this.isClosed
3235
- ? "conversation"
3236
- : "stashed",
3636
+ : accepted
3637
+ ? "queued"
3638
+ : "conversation",
3237
3639
  },
3238
3640
  "Voice duplex continuation finished",
3239
3641
  );
@@ -3259,6 +3661,105 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3259
3661
  })();
3260
3662
  }
3261
3663
 
3664
+ // Opens the wait for this barge-in's interrupting words, settling any older
3665
+ // barge-in's wait with null: its continuation was already superseded.
3666
+ private openBargeInInterruption(detachSeq: number): void {
3667
+ this.bargeInInterruption?.settle(null);
3668
+ let settle!: (words: string | null) => void;
3669
+ const promise = new Promise<string | null>((resolve) => {
3670
+ let settled = false;
3671
+ const timer = setTimeout(() => {
3672
+ finish(null);
3673
+ }, BARGE_IN_INTERRUPTION_WAIT_MS);
3674
+ const finish = (words: string | null): void => {
3675
+ if (settled) {
3676
+ return;
3677
+ }
3678
+ settled = true;
3679
+ clearTimeout(timer);
3680
+ if (this.bargeInInterruption?.detachSeq === detachSeq) {
3681
+ this.bargeInInterruption = null;
3682
+ }
3683
+ resolve(words);
3684
+ };
3685
+ settle = finish;
3686
+ });
3687
+ this.bargeInInterruption = { detachSeq, promise, settle };
3688
+ }
3689
+
3690
+ // Asks the continuation judge whether the foreground task this barge-in
3691
+ // suspended should still be finished. A drop clears the task, so the
3692
+ // automatic resume never fires; a keep leaves the resume as it was. The
3693
+ // interrupting turn may claim the task first, in which case it owns the
3694
+ // outcome and the verdict is only logged.
3695
+ private judgeSuspendedForegroundTask(detachSeq: number): void {
3696
+ const task = this.foregroundTaskState;
3697
+ if (task?.phase !== "suspended" || !this.judgeBackgroundContinuation) {
3698
+ return;
3699
+ }
3700
+ const epoch = task.epoch;
3701
+ const entry: { epoch: number; settled: Promise<void> } = {
3702
+ epoch,
3703
+ settled: Promise.resolve(),
3704
+ };
3705
+ entry.settled = this.requestContinuationJudgement(
3706
+ task.request,
3707
+ detachSeq,
3708
+ new AbortController().signal,
3709
+ ).then((judgement) => {
3710
+ if (this.foregroundTaskJudgement === entry) {
3711
+ this.foregroundTaskJudgement = null;
3712
+ }
3713
+ if (judgement === null || judgement.outcome === "unavailable") {
3714
+ return;
3715
+ }
3716
+ log.info(
3717
+ {
3718
+ epoch,
3719
+ outcome: judgement.outcome,
3720
+ noul: judgement.noul,
3721
+ latencyMs: judgement.latencyMs,
3722
+ },
3723
+ "Voice foreground task judged",
3724
+ );
3725
+ const current = this.foregroundTaskState;
3726
+ if (
3727
+ !judgement.keep &&
3728
+ current?.phase === "suspended" &&
3729
+ current.epoch === epoch
3730
+ ) {
3731
+ this.clearForegroundTask("caller_moved_on");
3732
+ }
3733
+ });
3734
+ this.foregroundTaskJudgement = entry;
3735
+ }
3736
+
3737
+ // The continuation judge's verdict for one barge-in, or null when no judge
3738
+ // is wired. The judge only waits on the interruption when it can run.
3739
+ private async requestContinuationJudgement(
3740
+ interruptedRequest: string,
3741
+ detachSeq: number,
3742
+ signal: AbortSignal,
3743
+ ): Promise<ContinuationJudgement | null> {
3744
+ const judge = this.judgeBackgroundContinuation;
3745
+ if (!judge) {
3746
+ return null;
3747
+ }
3748
+ const waiter = this.bargeInInterruption;
3749
+ const interruption =
3750
+ waiter?.detachSeq === detachSeq ? waiter.promise : Promise.resolve(null);
3751
+ try {
3752
+ return await judge({
3753
+ parentConversationId: this.conversationId,
3754
+ interruptedRequest,
3755
+ interruption,
3756
+ signal,
3757
+ });
3758
+ } catch {
3759
+ return null;
3760
+ }
3761
+ }
3762
+
3262
3763
  // Resolves the model-phrased label for a continuation, or null when there
3263
3764
  // is nothing to name it from, no labeler is wired, or the call missed
3264
3765
  // (timeout, abort, provider error, model declined). Never rejects: the
@@ -3294,7 +3795,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3294
3795
  const currentRequest = turn.utterance.finalTranscriptSegments
3295
3796
  .join(" ")
3296
3797
  .trim();
3297
- return currentRequest || turn.interruptedRequest || "";
3798
+ return (
3799
+ currentRequest ||
3800
+ turn.lookFollowUp?.callerUtterance ||
3801
+ turn.interruptedRequest ||
3802
+ ""
3803
+ );
3298
3804
  }
3299
3805
 
3300
3806
  private createForegroundTaskOwnership(
@@ -3315,6 +3821,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3315
3821
 
3316
3822
  private noteEscalatedForegroundTask(turn: ActiveAssistantTurn): void {
3317
3823
  if (
3824
+ turn.taskOutcome !== null ||
3318
3825
  this.activeAssistantTurn?.token !== turn.token ||
3319
3826
  turn.discardRequested ||
3320
3827
  turn.abortController.signal.aborted
@@ -3339,6 +3846,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3339
3846
  turn: ActiveAssistantTurn,
3340
3847
  effectiveToolName: string,
3341
3848
  ): void {
3849
+ if (turn.taskOutcome !== null) {
3850
+ return;
3851
+ }
3342
3852
  const current = this.foregroundTaskState;
3343
3853
  if (current?.phase === "owned" && current.ownerToken === turn.token) {
3344
3854
  if (!current.hostToolStarted) {
@@ -3608,6 +4118,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3608
4118
  private async resumeForegroundTask(
3609
4119
  task: SuspendedForegroundTask,
3610
4120
  ): Promise<void> {
4121
+ // A verdict still out on this task decides whether it resumes at all.
4122
+ const judgement = this.foregroundTaskJudgement;
4123
+ if (judgement?.epoch === task.epoch) {
4124
+ await judgement.settled;
4125
+ }
3611
4126
  const current = this.foregroundTaskState;
3612
4127
  if (
3613
4128
  current?.phase !== "suspended" ||
@@ -3635,178 +4150,139 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3635
4150
  }
3636
4151
 
3637
4152
  // Abort every background continuation this session started and drop its
3638
- // handle. Called on a hard stop (client interrupt / session close), when a
3639
- // newer barge-in supersedes the detached runs, and on a foreground-wins
3640
- // abort; the continuation's own `.finally` removes it from the set too.
3641
- // `keepPendingResult` is the foreground-wins variant: the foreground turn is
3642
- // claiming the workspace, so running continuations must die (and pending
3643
- // detaches must be skipped), but an already-completed continuation's stashed
3644
- // answer stays — it cannot race anything, and the next turn's "use only if
3645
- // relevant" framing makes a stale one harmless.
4153
+ // handle. Finished outcomes have independent delivery state.
3646
4154
  private abortDetachedRuns(opts?: {
3647
- keepPendingResult?: boolean;
3648
- // Why the runs are being invalidated, for the log line below. Every
3649
- // caller passes one: an unexplained dead continuation is the single
3650
- // hardest thing to debug about this feature.
3651
4155
  reason?: string;
3652
- // The foreground tool whose start tripped the contention gate, if any.
3653
4156
  toolName?: string;
3654
4157
  }): void {
3655
4158
  const aborted = this.detachControllers.size;
3656
- const hadPendingResult = this.pendingContinuationResult !== null;
3657
- if (aborted > 0 || hadPendingResult) {
4159
+ if (aborted > 0) {
3658
4160
  log.debug(
3659
4161
  {
3660
4162
  conversationId: this.conversationId,
3661
4163
  reason: opts?.reason ?? "unspecified",
3662
4164
  ...(opts?.toolName ? { toolName: opts.toolName } : {}),
3663
4165
  abortedRuns: aborted,
3664
- droppedPendingResult: hadPendingResult && !opts?.keepPendingResult,
3665
4166
  },
3666
4167
  "Voice duplex continuations invalidated",
3667
4168
  );
3668
4169
  }
3669
- // Bump the generation so a barge-in whose async teardown is still in flight
3670
- // (its detach not yet spawned) sees the stop and skips the continuation.
3671
4170
  this.detachStopGeneration += 1;
3672
4171
  for (const controller of this.detachControllers) {
3673
4172
  controller.abort();
3674
4173
  }
3675
4174
  this.detachControllers.clear();
3676
- // A hard stop also drops any completed continuation's result still waiting
3677
- // to fold into the next turn, so it can't surface after the user reset.
3678
- if (!opts?.keepPendingResult) {
3679
- this.pendingContinuationResult = null;
4175
+ }
4176
+
4177
+ receiveSubagentNotification(
4178
+ notification: SubagentParentNotification,
4179
+ ): boolean {
4180
+ return this.receiveTaskOutcome({ ...notification, source: "subagent" });
4181
+ }
4182
+
4183
+ private receiveTaskOutcome(notification: VoiceTaskOutcome): boolean {
4184
+ if (this.isClosed && this.taskOutcomesClosing) {
4185
+ this.taskOutcomes.enqueue(notification);
4186
+ return true;
4187
+ }
4188
+ if (this.isClosed || this.state === "failed" || !this.startVoiceTurn) {
4189
+ return false;
3680
4190
  }
3681
- // The announcement dies either way: even the foreground-wins variant (which
3682
- // keeps the stash) has a foreground turn claiming the floor, so speaking up
3683
- // on our own is exactly what must not happen.
3684
- this.clearContinuationAnnouncement();
4191
+ this.taskOutcomes.acknowledgePlayback(Date.now());
4192
+ this.taskOutcomes.enqueue(notification);
4193
+ this.taskAnnouncementsDeferred = false;
4194
+ this.scheduleTaskAnnouncement();
4195
+ return true;
3685
4196
  }
3686
4197
 
3687
- private clearContinuationAnnouncement(): void {
3688
- this.pendingAnnouncement = null;
3689
- if (this.announcementTimer) {
3690
- clearTimeout(this.announcementTimer);
3691
- this.announcementTimer = null;
4198
+ private clearTaskAnnouncementTimer(): void {
4199
+ if (this.taskAnnouncementTimer !== null) {
4200
+ clearTimeout(this.taskAnnouncementTimer);
4201
+ this.taskAnnouncementTimer = null;
3692
4202
  }
3693
4203
  }
3694
4204
 
3695
- /**
3696
- * Hang-up delivery for a continuation that already finished: the call is
3697
- * ending, so neither route that was armed for its answer can still run — the
3698
- * announcement has nobody to speak to and the stash has no next voice turn to
3699
- * fold into. Deliver it into the conversation instead, exactly as a
3700
- * continuation finishing after the close does, because that thread is where
3701
- * the user goes looking for the work ("barge in, hear the answer, close the
3702
- * room, go look for the result").
3703
- *
3704
- * Best-effort and consume-once: the fields are emptied whether or not the
3705
- * injection lands, so a close cannot deliver twice.
3706
- */
3707
- private async deliverPendingContinuationToConversation(): Promise<void> {
3708
- const activeTurn = this.activeAssistantTurn;
3709
- const activeAnnouncement = activeTurn?.continuationDelivery ?? null;
3710
- const announcement = this.pendingAnnouncement ?? activeAnnouncement;
3711
- const answer = announcement?.answer ?? this.pendingContinuationResult;
3712
- const request = announcement?.request ?? "";
3713
- this.pendingAnnouncement = null;
3714
- this.pendingContinuationResult = null;
3715
- if (activeTurn !== null) {
3716
- activeTurn.continuationDelivery = null;
3717
- }
3718
- if (answer === null || answer.length === 0) {
4205
+ private scheduleTaskAnnouncement(): void {
4206
+ this.taskOutcomes.acknowledgePlayback(Date.now());
4207
+ if (
4208
+ !this.taskOutcomes.hasPending ||
4209
+ this.taskAnnouncementsDeferred ||
4210
+ this.state === "initializing" ||
4211
+ this.isClosed ||
4212
+ this.state === "failed" ||
4213
+ this.activeAssistantTurn !== null
4214
+ ) {
4215
+ this.clearTaskAnnouncementTimer();
4216
+ return;
4217
+ }
4218
+ this.taskAnnouncementTimer = this.scheduleFloorCheck(
4219
+ this.taskAnnouncementTimer,
4220
+ this.continuationAnnounceSilenceMs,
4221
+ (blockedBy) => {
4222
+ this.taskAnnouncementTimer = null;
4223
+ this.taskOutcomes.acknowledgePlayback(Date.now());
4224
+ if (blockedBy === null) {
4225
+ void this.announceTaskOutcome();
4226
+ } else if (
4227
+ blockedBy !== "turn_active" &&
4228
+ blockedBy !== "session_unavailable"
4229
+ ) {
4230
+ this.scheduleTaskAnnouncement();
4231
+ }
4232
+ },
4233
+ );
4234
+ }
4235
+
4236
+ private async announceTaskOutcome(): Promise<void> {
4237
+ const notification = this.taskOutcomes.next();
4238
+ if (notification === undefined) {
3719
4239
  return;
3720
4240
  }
4241
+ let started = false;
3721
4242
  try {
3722
- const { injectMessageIntoParent } = await import("../subagent/notify.js");
3723
- injectMessageIntoParent(
3724
- this.conversationId,
3725
- buildClosedSessionDeliveryPrompt(request, answer),
3726
- );
3727
- log.info(
4243
+ started = await this.launchAssistantTurn(
4244
+ createSyntheticUtterance(),
4245
+ notification.source === "continuation"
4246
+ ? CONTINUATION_DELIVERY_CONTENT
4247
+ : notification.message,
3728
4248
  {
3729
- conversationId: this.conversationId,
3730
- resultChars: answer.length,
3731
- resultDestination: "conversation",
4249
+ taskOutcome: notification,
4250
+ hiddenPrompt: true,
4251
+ initialLeg: "escalated",
3732
4252
  },
3733
- "Voice duplex continuation delivered on session close",
3734
4253
  );
3735
4254
  } catch (err) {
3736
4255
  log.warn(
3737
- { err, conversationId: this.conversationId },
3738
- "Voice duplex continuation delivery on session close failed",
4256
+ { err, taskId: notification.taskId },
4257
+ "Voice task announcement failed to start",
3739
4258
  );
3740
4259
  }
4260
+ if (!started) {
4261
+ // A failed launch waits for the next user turn or update before retrying.
4262
+ this.taskAnnouncementsDeferred = true;
4263
+ this.clearTaskAnnouncementTimer();
4264
+ }
3741
4265
  }
3742
4266
 
3743
- /**
3744
- * Live-voice is non-interactive (JARVIS-1291) and the session never starts a
3745
- * turn on its own — with one narrow exception, this: a background
3746
- * continuation the user asked for has finished, the call is still live, and
3747
- * nobody is speaking. Arms the silence timer that decides whether that
3748
- * exception applies. Re-arming resets the wait, so a run of arms collapses
3749
- * into one announcement.
3750
- *
3751
- * The silence the timer waits out starts when the audio the client already
3752
- * has queued runs out, so a continuation finishing at the tail end of a long
3753
- * spoken reply waits for that reply to actually finish playing.
3754
- *
3755
- * `retry` marks the single re-arm a user-speech blocker gets. An active
3756
- * assistant turn keeps the announcement queued without polling; releasing
3757
- * the turn re-arms the timer against the actual playback tail.
3758
- */
3759
- private scheduleContinuationAnnouncement(
3760
- retry = false,
3761
- drainRearms = 0,
3762
- ): void {
3763
- this.announcementTimer = this.scheduleFloorCheck(
3764
- this.announcementTimer,
3765
- this.continuationAnnounceSilenceMs,
3766
- () => {
3767
- this.announcementTimer = null;
3768
- const blockedBy = this.continuationAnnouncementBlocker();
3769
- if (blockedBy === null) {
3770
- void this.announceContinuation().catch((err: unknown) => {
3771
- log.warn({ err }, "Voice duplex continuation announcement failed");
3772
- });
3773
- return;
3774
- }
3775
- log.debug(
3776
- {
3777
- conversationId: this.conversationId,
3778
- blockedBy,
3779
- retry,
3780
- drainRearms,
3781
- },
3782
- "Voice duplex continuation announcement deferred",
3783
- );
3784
- // A tail that grew while the timer ran (the reply was still streaming
3785
- // TTS when it was armed) is a wait of known length rather than a busy
3786
- // call, so it re-arms against the new deadline without spending the
3787
- // single retry the busy-call blockers get. Otherwise a long reply's
3788
- // playback alone outlasts the budget and the announcement is dropped
3789
- // even though the call goes idle right after. The tail only extends
3790
- // while a turn is active, which blocks earlier, so the re-arms converge;
3791
- // the cap bounds the interleaving where a fresh turn's audio lands
3792
- // between two checks.
3793
- if (
3794
- blockedBy === "playback_draining" &&
3795
- drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
3796
- ) {
3797
- this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
3798
- return;
3799
- }
3800
- if (blockedBy === "turn_active") {
3801
- return;
3802
- }
3803
- if (retry || this.pendingAnnouncement === null) {
3804
- this.pendingAnnouncement = null;
3805
- return;
3806
- }
3807
- this.scheduleContinuationAnnouncement(true, drainRearms);
3808
- },
3809
- );
4267
+ private async deliverTaskOutcomesToConversation(
4268
+ turnTeardown?: Promise<void>,
4269
+ ): Promise<void> {
4270
+ this.clearTaskAnnouncementTimer();
4271
+ // Voice aborts discard the parent queue. Deliver only after its turn settles.
4272
+ await Promise.allSettled(this.pendingAssistantStarts);
4273
+ await turnTeardown;
4274
+ await this.getTurnTeardown?.(this.conversationId);
4275
+ const { injectMessageIntoParent } = await import("../subagent/notify.js");
4276
+ const notifications = this.taskOutcomes.drain(Date.now());
4277
+ this.taskOutcomesClosing = false;
4278
+ for (const notification of notifications) {
4279
+ injectMessageIntoParent(
4280
+ this.conversationId,
4281
+ notification.message,
4282
+ notification.metadata,
4283
+ { cronRunId: notification.cronRunId, bypassLiveVoice: true },
4284
+ );
4285
+ }
3810
4286
  }
3811
4287
 
3812
4288
  /**
@@ -3853,9 +4329,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3853
4329
  }
3854
4330
 
3855
4331
  /**
3856
- * Publish whatever the turn's activity line should be right now: the
3857
- * decision it is waiting on if it is waiting, and its newest running tool if
3858
- * it is not.
4332
+ * Publish whatever the turn's activity should be right now: the decision it
4333
+ * is waiting on, its newest running tool, or the structured escalation state
4334
+ * underneath those temporary overlays.
3859
4335
  *
3860
4336
  * The single entry point for every caller that would otherwise reach for
3861
4337
  * `currentActivityLabel` directly. A turn can start and finish other ops
@@ -3873,7 +4349,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3873
4349
  );
3874
4350
  return;
3875
4351
  }
3876
- this.publishActivity(turn, this.currentActivityLabel(turn));
4352
+ const label = this.currentActivityLabel(turn);
4353
+ if (
4354
+ label.length === 0 &&
4355
+ turn.escalationTarget !== null &&
4356
+ !turn.assistantCompleted
4357
+ ) {
4358
+ if (turn.publishedActivityKind !== "escalation") {
4359
+ this.publishEscalationActivity(turn, turn.escalationTarget);
4360
+ }
4361
+ return;
4362
+ }
4363
+ this.publishActivity(turn, label);
3877
4364
  }
3878
4365
 
3879
4366
  /**
@@ -3951,14 +4438,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3951
4438
  * conversation may wait on one.
3952
4439
  *
3953
4440
  * Callers whose line depends on turn state go through
3954
- * {@link refreshActivity}; this is called directly only to clear the line
3955
- * outright, which a cancelled or finished turn does regardless of what it
3956
- * was waiting on.
4441
+ * {@link refreshActivity}. Structured transition events call this directly
4442
+ * because their meaning is not captured by the label alone.
3957
4443
  */
3958
4444
  private publishActivity(
3959
4445
  turn: ActiveAssistantTurn,
3960
4446
  label: string,
3961
4447
  approvalRequestId?: string,
4448
+ detail?: Pick<
4449
+ Extract<LiveVoiceServerFramePayload, { type: "activity" }>,
4450
+ "kind" | "profile" | "profileSource"
4451
+ >,
3962
4452
  ): void {
3963
4453
  // De-duplicated on the request id as well as the wording. The two move
3964
4454
  // independently: a wait can be entered and left without the tool line
@@ -3966,6 +4456,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3966
4456
  // retires the approval — leaving the island's buttons up with nothing
3967
4457
  // behind them.
3968
4458
  if (
4459
+ detail === undefined &&
4460
+ turn.publishedActivityKind === null &&
3969
4461
  turn.activityLabel === label &&
3970
4462
  turn.publishedApprovalRequestId === (approvalRequestId ?? null)
3971
4463
  ) {
@@ -3973,29 +4465,36 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3973
4465
  }
3974
4466
  turn.activityLabel = label;
3975
4467
  turn.publishedApprovalRequestId = approvalRequestId ?? null;
4468
+ turn.publishedActivityKind = detail?.kind ?? null;
3976
4469
  void this.sendFrame(
3977
4470
  {
3978
4471
  type: "activity",
3979
4472
  turnId: turn.turnId,
3980
4473
  label,
3981
4474
  ...(approvalRequestId !== undefined ? { approvalRequestId } : {}),
4475
+ ...detail,
3982
4476
  },
3983
4477
  () => !this.isClosed,
3984
4478
  );
3985
4479
  }
3986
4480
 
4481
+ private publishEscalationActivity(
4482
+ turn: ActiveAssistantTurn,
4483
+ target: VoiceEscalationTarget,
4484
+ ): void {
4485
+ this.publishActivity(turn, "", undefined, {
4486
+ kind: "escalation",
4487
+ profile: target.profile,
4488
+ profileSource: target.source,
4489
+ });
4490
+ }
4491
+
3987
4492
  private clearActiveAssistantTurn(token: symbol): void {
3988
4493
  if (this.activeAssistantTurn?.token !== token) {
3989
4494
  return;
3990
4495
  }
3991
4496
  this.activeAssistantTurn = null;
3992
- if (
3993
- this.pendingAnnouncement !== null &&
3994
- !this.isClosed &&
3995
- this.state !== "failed"
3996
- ) {
3997
- this.scheduleContinuationAnnouncement();
3998
- }
4497
+ this.scheduleTaskAnnouncement();
3999
4498
  if (
4000
4499
  this.foregroundTaskState?.phase === "suspended" &&
4001
4500
  this.foregroundTaskState.resumePending &&
@@ -4006,18 +4505,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4006
4505
  }
4007
4506
  }
4008
4507
 
4009
- // Why the session must not speak up right now, or null when the call is
4010
- // genuinely idle: live, no turn running, and no user speech anywhere in
4011
- // flight (onset latch, barge-in guard, detector, or a cycle that has already
4012
- // captured something). The server-VAD re-arm keeps an empty cycle armed
4013
- // between turns, so an untouched `currentUtterance` IS the idle state.
4014
- private continuationAnnouncementBlocker(): string | null {
4015
- if (this.pendingAnnouncement === null) {
4016
- return "nothing_pending";
4017
- }
4018
- return this.sessionTurnFloorBlocker();
4019
- }
4020
-
4021
4508
  /**
4022
4509
  * Why the session may not start a turn of its own right now, or null when it
4023
4510
  * may.
@@ -4101,7 +4588,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4101
4588
  * that would not open, a share the desktop refused) cannot turn up minutes
4102
4589
  * later as a reply to nothing.
4103
4590
  */
4104
- private awaitLookFrame(action: LookSessionControl): void {
4591
+ private awaitLookFrame(
4592
+ action: LookSessionControl,
4593
+ turn: ActiveAssistantTurn,
4594
+ ): void {
4105
4595
  if (!this.lookFrames) {
4106
4596
  return;
4107
4597
  }
@@ -4116,7 +4606,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4116
4606
  "Live voice look dropped: no frame arrived",
4117
4607
  );
4118
4608
  }, LOOK_FRAME_WAIT_MS);
4119
- this.pendingLook = { action, armedAtMs: Date.now(), timer };
4609
+ this.pendingLook = {
4610
+ action,
4611
+ callerUtterance: this.foregroundTaskRequestForTurn(turn),
4612
+ armedAtMs: Date.now(),
4613
+ timer,
4614
+ frameLanded: false,
4615
+ };
4616
+ }
4617
+
4618
+ private appendPendingLookRequest(content: string): void {
4619
+ if (this.pendingLook !== null && content.trim().length > 0) {
4620
+ this.pendingLook.callerUtterance =
4621
+ `${this.pendingLook.callerUtterance}\n${content}`.trim();
4622
+ }
4120
4623
  }
4121
4624
 
4122
4625
  private clearPendingLook(): void {
@@ -4138,11 +4641,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4138
4641
  */
4139
4642
  private lookFrameLanded(): void {
4140
4643
  const pending = this.pendingLook;
4141
- if (pending === null || this.isClosed) {
4644
+ if (pending === null || pending.frameLanded || this.isClosed) {
4142
4645
  return;
4143
4646
  }
4144
4647
  clearTimeout(pending.timer);
4145
- this.pendingLook = null;
4648
+ pending.frameLanded = true;
4146
4649
  this.answerLookWhenFloorIsFree(pending, this.turnsLaunched, 0);
4147
4650
  }
4148
4651
 
@@ -4163,11 +4666,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4163
4666
  turnsAtFrame: number,
4164
4667
  rearms: number,
4165
4668
  ): void {
4669
+ if (this.pendingLook !== look) {
4670
+ return;
4671
+ }
4166
4672
  if (this.lookFollowUpTimer !== null) {
4167
4673
  clearTimeout(this.lookFollowUpTimer);
4168
4674
  this.lookFollowUpTimer = null;
4169
4675
  }
4170
- const { action, armedAtMs } = look;
4676
+ const { action, callerUtterance, armedAtMs } = look;
4171
4677
  // A turn launched since the frame landed read it already. Speech that
4172
4678
  // never became a turn (a cough, noise that transcribed to nothing) is only
4173
4679
  // waited out.
@@ -4176,10 +4682,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4176
4682
  ? "turn_since_look"
4177
4683
  : this.sessionTurnFloorBlocker();
4178
4684
  if (blockedBy === null) {
4685
+ this.clearPendingLook();
4179
4686
  void this.launchAssistantTurn(
4180
4687
  createSyntheticUtterance(),
4181
4688
  LOOK_FOLLOW_UP_CONTENT,
4182
- { lookFollowUp: action, hiddenPrompt: true },
4689
+ { lookFollowUp: { action, callerUtterance }, hiddenPrompt: true },
4183
4690
  ).catch((err: unknown) => {
4184
4691
  log.warn(
4185
4692
  { err, conversationId: this.conversationId, action },
@@ -4191,6 +4698,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4191
4698
  const waitable =
4192
4699
  blockedBy !== "turn_since_look" && blockedBy !== "session_unavailable";
4193
4700
  if (!waitable || Date.now() - armedAtMs >= LOOK_ANSWER_DEADLINE_MS) {
4701
+ this.clearPendingLook();
4194
4702
  log.info(
4195
4703
  { conversationId: this.conversationId, action, blockedBy, rearms },
4196
4704
  "Live voice look follow-up skipped",
@@ -4207,77 +4715,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4207
4715
  );
4208
4716
  }
4209
4717
 
4210
- // Speak the finished continuation's result on a turn the session starts
4211
- // itself. The turn rides a synthetic cycle rather than the armed VAD one —
4212
- // `currentUtterance` belongs to the capture loop, and overwriting it would
4213
- // collide with the user's next real utterance.
4214
- private async announceContinuation(): Promise<void> {
4215
- const pending = this.pendingAnnouncement;
4216
- if (!pending) {
4217
- return;
4218
- }
4219
- this.pendingAnnouncement = null;
4220
- // The two routes for one answer are mutually exclusive: while this turn is
4221
- // in flight the stash must not deliver the same answer again. It comes back
4222
- // only if this turn provably never started (below) or a barge-in cuts it
4223
- // short before it delivers (see bargeIn).
4224
- this.pendingContinuationResult = null;
4225
- // Pins the invalidation state this hand-off was decided in: a hard stop
4226
- // landing while the turn starts owns the drop, and must not be undone by
4227
- // the restore below.
4228
- const stopGeneration = this.detachStopGeneration;
4229
- let started: boolean;
4230
- try {
4231
- started = await this.launchAssistantTurn(
4232
- createSyntheticUtterance(),
4233
- CONTINUATION_DELIVERY_CONTENT,
4234
- { continuationDelivery: pending },
4235
- );
4236
- } catch (err) {
4237
- // A throw before the leg's own error handling (an outbound frame write
4238
- // rejecting, say) leaves the same state a false return does: nothing
4239
- // persisted, nothing spoken, and the stash already emptied above. Take
4240
- // the identical fallback rather than letting the answer die with the
4241
- // rejection.
4242
- started = false;
4243
- log.warn(
4244
- { err, conversationId: this.conversationId },
4245
- "Voice duplex continuation announcement threw while starting",
4246
- );
4247
- }
4248
- if (started) {
4249
- log.info(
4250
- {
4251
- conversationId: this.conversationId,
4252
- resultChars: pending.answer.length,
4253
- resultDestination: "announced",
4254
- },
4255
- "Voice duplex continuation announced",
4256
- );
4257
- return;
4258
- }
4259
- // The turn never reached the bridge (the conversation is busy from another
4260
- // surface, or the bridge threw), so nothing was persisted or spoken. Hand
4261
- // the answer back to the stash — the user's next turn is the fallback route
4262
- // that exists for exactly this. A newer continuation's answer already in
4263
- // the stash wins over this older one rather than being overwritten.
4264
- const restashed =
4265
- !this.isClosed &&
4266
- this.detachStopGeneration === stopGeneration &&
4267
- this.pendingContinuationResult === null;
4268
- if (restashed) {
4269
- this.pendingContinuationResult = pending.answer;
4270
- }
4271
- log.warn(
4272
- {
4273
- conversationId: this.conversationId,
4274
- resultChars: pending.answer.length,
4275
- resultDestination: restashed ? "stashed" : "dropped",
4276
- },
4277
- "Voice duplex continuation announcement could not start",
4278
- );
4279
- }
4280
-
4281
4718
  // VAD closed the utterance — the analog of ptt_release: emit
4282
4719
  // utterance_end, then run the standard release path.
4283
4720
  private handleVadUtteranceEnd(reason: "silence" | "max-duration"): void {
@@ -4294,6 +4731,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4294
4731
  this.vadSpeechStartPending = false;
4295
4732
  // The detector turn is over: an untripped guard was noise, not
4296
4733
  // barge-in — leave playback untouched.
4734
+ if (this.pendingBargeIn) {
4735
+ this.logInputDiagnostic("voice_input_barge_in_expired", {
4736
+ reason,
4737
+ armedAtMs: this.pendingBargeIn.armedAtMs,
4738
+ guardResets: this.pendingBargeIn.resets,
4739
+ accumulatedSpeechMs: this.pendingBargeIn.guard.speechMs,
4740
+ });
4741
+ }
4297
4742
  this.pendingBargeIn = null;
4298
4743
  if (reason === "max-duration") {
4299
4744
  // A max-duration boundary always releases: drop any pending hold
@@ -4372,7 +4817,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4372
4817
  // end-of-turn is coming and the utterance replays this boundary on the
4373
4818
  // silence path.
4374
4819
  // `waitMsOverride` collapses that wait when the caller already knows no
4375
- // end-of-turn is coming (see setProviderTurnEndActive).
4820
+ // end-of-turn is coming (see setProviderTurnDetectionActive).
4376
4821
  private armProviderTurnEndFallbackTimer(
4377
4822
  utterance: UtteranceCycle,
4378
4823
  waitMsOverride?: number,
@@ -4405,32 +4850,22 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4405
4850
  }
4406
4851
 
4407
4852
  /**
4408
- * Flips the provider end-of-turn latch, unwinding an optimistic arm.
4409
- *
4410
- * `beginUtterance` arms the latch from the configured provider before the
4411
- * dial resolves, so a silence boundary can already have deferred to the
4412
- * provider by the time the resolved one says otherwise. That deferred
4413
- * boundary is parked on the fail-open deadline, a whole end-of-turn budget
4414
- * away, waiting for an event that will now never arrive. Collapse the wait to
4415
- * zero rather than burn the budget: the deadline body re-checks the cycle
4416
- * and replays the silence boundary, which with the latch down takes the
4417
- * ordinary hold path. Replaying through the deadline instead of calling
4418
- * `handleVadUtteranceEnd` directly keeps the release off the arming
4419
- * caller's stack, which is still mid-dial.
4420
- *
4421
- * A cleared latch with no deadline armed needs no unwind: either no
4422
- * boundary ever deferred, or the caller resumed speaking and
4423
- * `handleVadSpeechStart` already dropped the deadline, leaving the detector
4424
- * owning the next boundary. The cycle's `turnBoundaryGeneration` stamp is
4425
- * left as it is: `isStaleProviderTurnEnd` is only ever consulted from
4426
- * `handleProviderTurnEnd`, which returns immediately once the latch is down.
4853
+ * The pre-dial configuration can defer a silence boundary to a provider
4854
+ * that resolves without turn detection. Collapse its fallback deadline
4855
+ * rather than wait for an event that cannot arrive. Replaying through the
4856
+ * timer keeps release off the resolver's stack and rechecks cycle ownership.
4427
4857
  */
4428
- private setProviderTurnEndActive(active: boolean): void {
4429
- if (active === this.providerTurnEndActive) {
4858
+ private setProviderTurnDetectionActive(active: boolean): void {
4859
+ const wasEndActive = this.providerTurnEndActive;
4860
+ this.providerTurnStartActive = active;
4861
+ if (active) {
4862
+ this.pendingBargeIn = null;
4863
+ }
4864
+ if (wasEndActive === this.providerTurnEndActive) {
4430
4865
  return;
4431
4866
  }
4432
- this.providerTurnEndActive = active;
4433
- if (active || this.providerTurnEndTimer === null) {
4867
+ this.microphoneIdleGate.reset();
4868
+ if (this.providerTurnEndActive || this.providerTurnEndTimer === null) {
4434
4869
  return;
4435
4870
  }
4436
4871
  const utterance = this.currentUtterance;
@@ -4441,18 +4876,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4441
4876
  }
4442
4877
 
4443
4878
  /**
4444
- * Record the provider turn a cycle is currently inside. Providers without
4445
- * turn numbering send no index, which leaves the cycle on the local speech
4446
- * generation as its only staleness signal (see isStaleProviderTurnEnd).
4879
+ * Turn indices belong to the stream, so duplicates cannot interrupt the
4880
+ * reply to an earlier cycle. The cycle also keeps its own index for matching
4881
+ * end-of-turn events; unnumbered events use local speech generation there.
4447
4882
  */
4448
- private recordProviderTurnStart(
4883
+ private handleProviderTurnStart(
4449
4884
  utterance: UtteranceCycle,
4450
4885
  turnIndex: number | undefined,
4451
4886
  ): void {
4452
- if (turnIndex === undefined) {
4887
+ this.logInputDiagnostic("voice_input_provider_turn_start", {
4888
+ inputTurnId: this.ensureTurnId(utterance),
4889
+ turnIndex: turnIndex ?? null,
4890
+ sttProvider: utterance.dialedSttProvider,
4891
+ localDetectorActive: this.turnDetector?.isActive ?? false,
4892
+ });
4893
+ if (
4894
+ this.currentUtterance !== utterance ||
4895
+ (utterance.released && utterance.assistantTurnStarted) ||
4896
+ utterance.completed ||
4897
+ (turnIndex !== undefined &&
4898
+ this.latestProviderTurnStartIndex !== null &&
4899
+ turnIndex <= this.latestProviderTurnStartIndex)
4900
+ ) {
4453
4901
  return;
4454
4902
  }
4455
- utterance.openProviderTurnIndex = turnIndex;
4903
+ if (turnIndex !== undefined) {
4904
+ this.latestProviderTurnStartIndex = turnIndex;
4905
+ utterance.openProviderTurnIndex = turnIndex;
4906
+ }
4907
+ if (this.providerTurnStartActive) {
4908
+ utterance.speechRouted = true;
4909
+ this.detectedSpeech = true;
4910
+ this.markSpeechStart(utterance);
4911
+ this.handleSpeechStart("provider");
4912
+ }
4456
4913
  }
4457
4914
 
4458
4915
  /**
@@ -4520,8 +4977,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4520
4977
  */
4521
4978
  private async handleProviderTurnEnd(
4522
4979
  utterance: UtteranceCycle,
4523
- turnIndex: number | undefined,
4980
+ event: Extract<SttStreamServerEvent, { type: "turn-end" }>,
4524
4981
  ): Promise<void> {
4982
+ const { turnIndex } = event;
4983
+ this.logInputDiagnostic("voice_input_provider_turn_end", {
4984
+ inputTurnId: utterance.turnId,
4985
+ sttProvider: utterance.dialedSttProvider,
4986
+ turnIndex: turnIndex ?? null,
4987
+ confidence: event.confidence ?? null,
4988
+ trigger: event.trigger ?? null,
4989
+ audioWindowEndSeconds: event.audioWindowEndSeconds ?? null,
4990
+ transcriptChars: event.text.length,
4991
+ localDetectorActive: this.turnDetector?.isActive ?? false,
4992
+ msSinceLocalSpeechStop:
4993
+ this.localSpeechStopAtMs === null
4994
+ ? null
4995
+ : this.msSinceLocalSpeechStop(),
4996
+ providerTurnEndActive: this.providerTurnEndActive,
4997
+ providerTurnEndTimedOut: utterance.providerTurnEndTimedOut,
4998
+ });
4525
4999
  if (
4526
5000
  !this.providerTurnEndActive ||
4527
5001
  this.currentUtterance !== utterance ||
@@ -4572,12 +5046,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4572
5046
  // closing the stream.
4573
5047
  utterance.providerClosedTurn = true;
4574
5048
  await this.releaseUtterance();
4575
- // Leave the local detector idle, which is where every other commit path
4576
- // leaves it. A provider can close a turn while the trailing-silence
4577
- // countdown is still running, and barge-in fires from the detector's speech
4578
- // ONSET: a detector left mid-turn reports no onset, so the caller could
4579
- // not interrupt the reply they just triggered. The forced boundary
4580
- // reaches an already-released utterance and returns.
5049
+ // The provider can commit before the local silence countdown expires.
5050
+ // Reset the detector so the next speech run gets its own local boundary.
5051
+ // This forced boundary reaches an already-released utterance and returns.
4581
5052
  this.turnDetector?.forceEnd();
4582
5053
  }
4583
5054
 
@@ -4642,6 +5113,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4642
5113
  // are skipped; the thinking frame and timers still apply.
4643
5114
  const alreadyReleased = utterance.released;
4644
5115
  turn.speculativePending = false;
5116
+ // Caller context is final once the speculative turn commits.
5117
+ if (!turn.hiddenPrompt && turn.speculativeContent !== null) {
5118
+ this.appendPendingLookRequest(turn.speculativeContent);
5119
+ if (turn.interruptedRequest !== null) {
5120
+ this.bargeInInterruption?.settle(turn.speculativeContent);
5121
+ }
5122
+ }
4645
5123
  // Finals can land between the speculative dispatch and this verdict.
4646
5124
  // Fill the language only when dispatch had none: the model request was
4647
5125
  // already issued with the dispatch language, so overwriting here would
@@ -4723,11 +5201,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4723
5201
  reason: string,
4724
5202
  ): void {
4725
5203
  turn.speculativePending = false;
4726
- // The dispatch is being unwound, so the barge-in merge note, the finished
4727
- // continuation's answer, its queued announcement, and any camera frame it
4728
- // claimed all go back to the session. The turn that would have delivered
4729
- // them is gone, and the utterance they belong to is about to be sent by
4730
- // another.
5204
+ // Return claimed user context and camera frames for the replay.
4731
5205
  this.restorePendingTurnContext(turn);
4732
5206
  // Latched before the handle check: when the discard beats the bridge
4733
5207
  // handle's resolution (startVoiceTurn still persisting), the handle's
@@ -4737,6 +5211,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4737
5211
  this.activeAssistantTurn = null;
4738
5212
  }
4739
5213
  this.clearFillerTimers(turn);
5214
+ this.releaseModeSessionRequests(turn);
5215
+ this.releaseModeSessionDelivery(turn);
4740
5216
  turn.abortController.abort(
4741
5217
  createAbortReason("voice_session_aborted", `live-voice-${reason}`),
4742
5218
  );
@@ -5037,12 +5513,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5037
5513
  // completion signal has no cycle to advance here.
5038
5514
  return;
5039
5515
  case "turn-start":
5040
- // Barge-in is deliberately untouched: local VAD still owns it,
5041
- // because a provider roundtrip cannot beat a local energy gate on an
5042
- // interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
5043
- // index is recorded so a later end-of-turn can be told apart from one
5044
- // this turn superseded (see isStaleProviderTurnEnd).
5045
- this.recordProviderTurnStart(utterance, event.turnIndex);
5516
+ this.handleProviderTurnStart(utterance, event.turnIndex);
5046
5517
  return;
5047
5518
  case "eager-turn-end":
5048
5519
  case "turn-resumed":
@@ -5052,7 +5523,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5052
5523
  // follow-up.
5053
5524
  return;
5054
5525
  case "turn-end":
5055
- await this.handleProviderTurnEnd(utterance, event.turnIndex);
5526
+ await this.handleProviderTurnEnd(utterance, event);
5056
5527
  return;
5057
5528
  case "error":
5058
5529
  await this.sendTranscriberErrorFrame(event);
@@ -5183,14 +5654,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5183
5654
  return;
5184
5655
  }
5185
5656
  case "turn-start": {
5186
- // Barge-in is deliberately untouched: local VAD still owns it,
5187
- // because a provider roundtrip cannot beat a local energy gate on an
5188
- // interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
5189
- // index is recorded so a later end-of-turn can be told apart from one
5190
- // this turn superseded (see isStaleProviderTurnEnd).
5191
5657
  const target = this.pendingTranscriptCycle();
5192
5658
  if (target) {
5193
- this.recordProviderTurnStart(target, event.turnIndex);
5659
+ this.handleProviderTurnStart(target, event.turnIndex);
5194
5660
  }
5195
5661
  return;
5196
5662
  }
@@ -5206,7 +5672,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5206
5672
  // transcript owns the turn the provider just closed.
5207
5673
  const target = this.pendingTranscriptCycle();
5208
5674
  if (target) {
5209
- await this.handleProviderTurnEnd(target, event.turnIndex);
5675
+ await this.handleProviderTurnEnd(target, event);
5210
5676
  }
5211
5677
  return;
5212
5678
  }
@@ -5224,8 +5690,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5224
5690
  }
5225
5691
  // The shared stream closed under the session: fall back to the
5226
5692
  // per-cycle path — the next arm resolves a fresh transcriber.
5227
- this.sharedTranscriber = null;
5228
- this.sharedTranscriberLanguage = undefined;
5693
+ this.releaseSharedTranscriber(transcriber);
5229
5694
  const drained = this.drainFinalizeQueueFor(transcriber);
5230
5695
  const current = this.currentUtterance;
5231
5696
  if (
@@ -5378,6 +5843,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5378
5843
  }
5379
5844
 
5380
5845
  private async interrupt(): Promise<void> {
5846
+ this.taskOutcomes.interruptPlayback(Date.now());
5847
+ this.taskAnnouncementsDeferred = true;
5848
+ this.clearTaskAnnouncementTimer();
5381
5849
  if (this.isClosed || this.state === "failed") {
5382
5850
  return;
5383
5851
  }
@@ -5421,6 +5889,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5421
5889
  // per-cycle) and clears the finalize bookkeeping. Used by interrupt() and
5422
5890
  // close(); persistent mode re-establishes itself on the next arm.
5423
5891
  private stopSessionTranscriber(): void {
5892
+ this.microphoneIdleGate.reset();
5424
5893
  this.clearFinalizeGraceTimer();
5425
5894
  this.finalizeQueue = [];
5426
5895
  const shared = this.sharedTranscriber;
@@ -5505,64 +5974,32 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5505
5974
  await this.launchAssistantTurn(utterance, content);
5506
5975
  }
5507
5976
 
5508
- /**
5509
- * Take the barge-in merge context, the completed-continuation context, the
5510
- * handoff note, and the parked camera frame for the turn being launched
5511
- * (each feeds exactly the next launched turn) and cancel the queued
5512
- * announcement, because the turn this context rides delivers the same answer
5513
- * and would otherwise say it twice.
5514
- *
5515
- * Every real user turn consumes here, speculative or not: a speculative
5516
- * dispatch latches `assistantTurnStarted`, so a turn that skipped this would
5517
- * never pick the context up on any later pass. A rolled-back dispatch hands
5518
- * it all back (restorePendingTurnContext).
5519
- */
5977
+ // User-turn context is consumable; finished outcomes are only peeked.
5520
5978
  private consumePendingTurnContext(): {
5521
5979
  interruptedRequest: string | null;
5522
- continuationResult: string | null;
5523
5980
  handedOffRequest: string | null;
5524
- announcement: ContinuationDelivery | null;
5525
5981
  turnAttachmentId: string | null;
5526
5982
  } {
5527
5983
  const consumed = {
5528
5984
  interruptedRequest: this.pendingInterruptedRequest,
5529
- continuationResult: this.pendingContinuationResult,
5530
5985
  handedOffRequest: this.pendingHandoffRequest,
5531
- announcement: this.pendingAnnouncement,
5532
5986
  turnAttachmentId: this.pendingTurnAttachmentId,
5533
5987
  };
5534
5988
  this.pendingInterruptedRequest = null;
5535
- this.pendingContinuationResult = null;
5536
5989
  this.pendingHandoffRequest = null;
5537
5990
  this.pendingTurnAttachmentId = null;
5538
- this.clearContinuationAnnouncement();
5539
5991
  return consumed;
5540
5992
  }
5541
5993
 
5542
- /**
5543
- * Hand a turn's consumed pending context back to the session because the
5544
- * turn never happened: a speculative dispatch was discarded or held, or the
5545
- * launch never reached the bridge. Without this a hold verdict would destroy
5546
- * a finished continuation's answer outright — the stash was already emptied
5547
- * and no later turn would ever see it.
5548
- *
5549
- * Restores are conservative. A hard stop (interrupt/close/supersede) bumps
5550
- * `detachStopGeneration`, and that drop is deliberate — never undone here.
5551
- * Anything newer already sitting in a slot wins over the older value being
5552
- * returned to it.
5553
- */
5994
+ // Rollbacks return consumed context without changing queued outcome delivery.
5554
5995
  private restorePendingTurnContext(turn: ActiveAssistantTurn): void {
5555
5996
  const interruptedRequest = turn.interruptedRequest;
5556
- const continuationResult = turn.continuationResult;
5557
5997
  const handedOffRequest = turn.handedOffRequest;
5558
- const announcement = turn.consumedAnnouncement;
5559
5998
  const turnAttachmentId = turn.turnAttachmentId;
5560
5999
  // One restore per turn: the rollback paths overlap (a discarded turn whose
5561
6000
  // leg start also fails), and the second pass must be a no-op.
5562
6001
  turn.interruptedRequest = null;
5563
- turn.continuationResult = null;
5564
6002
  turn.handedOffRequest = null;
5565
- turn.consumedAnnouncement = null;
5566
6003
  turn.turnAttachmentId = null;
5567
6004
  // Nobody's to send unless the slot already holds it. A client that
5568
6005
  // retransmits the id this turn claimed parked the SAME frame rather than a
@@ -5593,39 +6030,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5593
6030
  } else {
5594
6031
  turn.staleFrameId = orphanedFrameId;
5595
6032
  }
5596
- if (this.pendingContinuationResult === null) {
5597
- this.pendingContinuationResult = continuationResult;
5598
- }
5599
- // "Still running" is stale the moment a result exists, which is why the
5600
- // detach clears the handoff note when it finishes — don't reinstate it
5601
- // against a continuation that has since completed.
5602
6033
  if (
5603
6034
  this.pendingHandoffRequest === null &&
5604
- this.pendingContinuationResult === null
6035
+ this.detachControllers.size > 0
5605
6036
  ) {
5606
6037
  this.pendingHandoffRequest = handedOffRequest;
5607
6038
  }
5608
- if (announcement !== null && this.pendingAnnouncement === null) {
5609
- this.pendingAnnouncement = announcement;
5610
- this.scheduleContinuationAnnouncement();
5611
- }
5612
6039
  }
5613
6040
 
5614
6041
  // Build the ActiveAssistantTurn for a released utterance and drive its model
5615
6042
  // leg. Resolves true once the leg has a live turn handle, false when the
5616
- // dispatch never reached the bridge — the signal a caller with a fallback
5617
- // route (announceContinuation) needs, since a failed start is otherwise
6043
+ // dispatch never reached the bridge: the signal a caller with a fallback
6044
+ // route (announceTaskOutcome) needs, since a failed start is otherwise
5618
6045
  // handled entirely inside startAssistantLeg.
5619
6046
  private async launchAssistantTurn(
5620
6047
  utterance: UtteranceCycle,
5621
6048
  content: string,
5622
6049
  opts?: {
5623
- // Set on an announcement turn: the finished continuation this turn exists
5624
- // to deliver. Its answer goes in the control prompt, not in `content`.
5625
- continuationDelivery?: ContinuationDelivery | null;
5626
- // Set on the turn that answers a look: which look. Its instruction goes
5627
- // in the control prompt, not in `content`.
5628
- lookFollowUp?: LookSessionControl;
6050
+ taskOutcome?: VoiceTaskOutcome;
6051
+ // The original request accompanies the fresh view in the control prompt.
6052
+ lookFollowUp?: LookFollowUp;
5629
6053
  // Unified front-door: dispatch without releasing the utterance. The
5630
6054
  // thinking frame and floor-holding timers are deferred until the leg's
5631
6055
  // leading verdict commits the turn (see commitSpeculativeTurn); a hold
@@ -5642,16 +6066,70 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5642
6066
  foregroundTaskEpoch?: number;
5643
6067
  },
5644
6068
  ): Promise<boolean> {
6069
+ if (opts?.hiddenPrompt !== true) {
6070
+ this.taskOutcomes.acknowledgePlayback(Date.now());
6071
+ this.taskAnnouncementsDeferred = false;
6072
+ }
5645
6073
  utterance.assistantTurnStarted = true;
5646
- // The announcement turn IS the delivery of the pending continuation, so it
5647
- // must not also consume the stash — feeding it both would deliver the same
5648
- // answer twice. Every other turn is a real user turn and takes the context.
6074
+ // Task announcements own their updates. Pending user-turn context stays
6075
+ // available for the next user turn instead of being consumed by an update.
5649
6076
  const pending =
5650
- opts?.continuationDelivery == null
5651
- ? this.consumePendingTurnContext()
5652
- : null;
6077
+ opts?.taskOutcome == null ? this.consumePendingTurnContext() : null;
6078
+ // A speculative dispatch may hold only the start of the interruption
6079
+ // ("Actually..."): its words reach the continuation judge when it
6080
+ // commits (commitSpeculativeTurn), and a discard restores the wait for
6081
+ // the redispatch.
6082
+ if (
6083
+ pending?.interruptedRequest != null &&
6084
+ opts?.hiddenPrompt !== true &&
6085
+ opts?.speculative !== true
6086
+ ) {
6087
+ this.bargeInInterruption?.settle(content);
6088
+ }
5653
6089
  const token = Symbol("live-voice-assistant-turn");
5654
6090
  const turnId = this.ensureTurnId(utterance);
6091
+ const initialLeg = opts?.initialLeg ?? "front-door";
6092
+ const modeSessionRequestId = randomUUID();
6093
+ const attemptedModeSessionRequestIds: string[] = [modeSessionRequestId];
6094
+ let modeSessionSource: ModeSessionSourceHandle | undefined;
6095
+ let modeSessionRequestIds: string[] = [];
6096
+ let modeSessionDeliveryTurnId: string | null = null;
6097
+ try {
6098
+ modeSessionSource =
6099
+ this.cameraModeSessions?.holdDelivery(modeSessionRequestId);
6100
+ if (!modeSessionSource) {
6101
+ this.releaseModeSessionTurnId(modeSessionRequestId);
6102
+ } else {
6103
+ modeSessionRequestIds.push(modeSessionRequestId);
6104
+ if (initialLeg === "front-door") {
6105
+ const escalatedRequestId = randomUUID();
6106
+ attemptedModeSessionRequestIds.push(escalatedRequestId);
6107
+ const escalatedSource =
6108
+ this.cameraModeSessions?.holdDelivery(escalatedRequestId);
6109
+ if (!sameModeSessionSource(escalatedSource, modeSessionSource)) {
6110
+ throw new Error("Camera session changed during voice admission");
6111
+ }
6112
+ modeSessionRequestIds.push(escalatedRequestId);
6113
+ }
6114
+ const deliveryTurnId = `live-voice-delivery:${this.context.sessionId}:${++this.modeSessionDeliveryCounter}`;
6115
+ attemptedModeSessionRequestIds.push(deliveryTurnId);
6116
+ const deliverySource =
6117
+ this.cameraModeSessions?.holdDelivery(deliveryTurnId);
6118
+ if (!sameModeSessionSource(deliverySource, modeSessionSource)) {
6119
+ throw new Error("Camera session changed during voice admission");
6120
+ }
6121
+ modeSessionDeliveryTurnId = deliveryTurnId;
6122
+ }
6123
+ } catch (err) {
6124
+ log.warn(
6125
+ { err, turnId },
6126
+ "Could not attach camera session ownership to voice turn",
6127
+ );
6128
+ this.releaseModeSessionTurnIds(attemptedModeSessionRequestIds);
6129
+ modeSessionSource = undefined;
6130
+ modeSessionRequestIds = [];
6131
+ modeSessionDeliveryTurnId = null;
6132
+ }
5655
6133
  this.startMetricsTurnIfNeeded(utterance, turnId);
5656
6134
  this.markAssistantDispatch(utterance, turnId);
5657
6135
  const abortController = new AbortController();
@@ -5671,6 +6149,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5671
6149
  language: this.turnLanguageFor(utterance),
5672
6150
  abortController,
5673
6151
  handle: null,
6152
+ modeSessionSource,
6153
+ modeSessionRequestIds,
6154
+ modeSessionDeliveryTurnId: modeSessionSource
6155
+ ? modeSessionDeliveryTurnId
6156
+ : null,
5674
6157
  launchedAtMs: Date.now(),
5675
6158
  progress: createProgressCadence({
5676
6159
  config: progressConfigForCadence(
@@ -5714,8 +6197,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5714
6197
  sessionControlRequested: null,
5715
6198
  activityLabel: "",
5716
6199
  publishedApprovalRequestId: null,
6200
+ publishedActivityKind: null,
5717
6201
  pendingApproval: null,
5718
6202
  ttsAudioStarted: false,
6203
+ ttsFailed: false,
5719
6204
  finalized: false,
5720
6205
  speculativePending: opts?.speculative === true,
5721
6206
  speculativeGeneration: this.vadSpeechGeneration,
@@ -5727,13 +6212,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5727
6212
  discardRequested: false,
5728
6213
  speculativeBuffer: "",
5729
6214
  interruptedRequest: pending?.interruptedRequest ?? null,
5730
- continuationResult: pending?.continuationResult ?? null,
6215
+ contextOutcomes: pending ? this.taskOutcomes.pendingOutcomes() : [],
5731
6216
  handedOffRequest: pending?.handedOffRequest ?? null,
5732
6217
  turnAttachmentId: pending?.turnAttachmentId ?? null,
5733
6218
  staleFrameId: null,
5734
- consumedAnnouncement: pending?.announcement ?? null,
5735
6219
  pendingContextStopGeneration: this.detachStopGeneration,
5736
- continuationDelivery: opts?.continuationDelivery ?? null,
6220
+ taskOutcome: opts?.taskOutcome ?? null,
6221
+ taskDeliveryContext: opts?.taskOutcome
6222
+ ? this.taskOutcomes.deliveryContext(opts.taskOutcome)
6223
+ : null,
6224
+ notificationHandledSilently: false,
5737
6225
  lookFollowUp: opts?.lookFollowUp ?? null,
5738
6226
  hiddenPrompt: opts?.hiddenPrompt === true,
5739
6227
  foregroundTaskEpoch,
@@ -5743,6 +6231,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5743
6231
  : null,
5744
6232
  deltaEpoch: 0,
5745
6233
  frontDoor: null,
6234
+ escalationTarget: null,
5746
6235
  ttsBuffer: "",
5747
6236
  spokenSegments: 0,
5748
6237
  spokenChars: 0,
@@ -5756,6 +6245,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5756
6245
  };
5757
6246
  this.activeAssistantTurn = activeTurn;
5758
6247
  this.turnsLaunched += 1;
6248
+ if (!activeTurn.hiddenPrompt && !activeTurn.speculativePending) {
6249
+ this.appendPendingLookRequest(content);
6250
+ }
5759
6251
 
5760
6252
  // A speculative turn defers the thinking frame and both floor-holding
5761
6253
  // timers to commitSpeculativeTurn: until the verdict arrives, the pause
@@ -5763,6 +6255,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5763
6255
  if (!opts?.speculative) {
5764
6256
  await this.sendFrame({ type: "thinking", turnId });
5765
6257
  if (!this.isActiveAssistantTurn(token)) {
6258
+ this.releaseModeSessionRequests(activeTurn);
6259
+ this.releaseModeSessionDelivery(activeTurn);
5766
6260
  this.restorePendingTurnContext(activeTurn);
5767
6261
  // Nothing was persisted, so a frame the restore could not hand back
5768
6262
  // links to no row and is collectible right here.
@@ -5805,7 +6299,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5805
6299
  activeTurn.progress.arm();
5806
6300
  }
5807
6301
 
5808
- const initialLeg = opts?.initialLeg ?? "front-door";
5809
6302
  const started = await this.startAssistantLeg(activeTurn, {
5810
6303
  content,
5811
6304
  routingLeg: initialLeg,
@@ -5819,6 +6312,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5819
6312
  : {}),
5820
6313
  });
5821
6314
  if (!started) {
6315
+ this.releaseModeSessionRequests(activeTurn);
6316
+ this.releaseModeSessionDelivery(activeTurn);
5822
6317
  // Nothing was persisted or spoken, so the context this turn consumed goes
5823
6318
  // back to the session rather than dying with the failed dispatch. A frame
5824
6319
  // a newer one displaced meanwhile links to no row, so it is collected
@@ -5861,10 +6356,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5861
6356
  attachments?: readonly string[];
5862
6357
  },
5863
6358
  ): Promise<boolean> {
5864
- if (!this.startVoiceTurn) {
6359
+ if (
6360
+ !this.startVoiceTurn ||
6361
+ this.isClosed ||
6362
+ activeTurn.abortController.signal.aborted
6363
+ ) {
5865
6364
  return false;
5866
6365
  }
5867
6366
  const { token, utterance, turnId } = activeTurn;
6367
+ const modeSessionRequestId = activeTurn.modeSessionRequestIds.shift();
5868
6368
 
5869
6369
  // `rawText` accumulates this leg's spoken stream. A front-door leg runs
5870
6370
  // through the shared coordinator, which reads its stream through the
@@ -5934,6 +6434,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5934
6434
  activeTurn.handle?.abort();
5935
6435
  activeTurn.handle = null;
5936
6436
  },
6437
+ overruleLeg: () => {
6438
+ const handle = activeTurn.handle;
6439
+ activeTurn.handle = null;
6440
+ if (handle?.overrule) {
6441
+ handle.overrule();
6442
+ } else {
6443
+ handle?.abort();
6444
+ }
6445
+ },
5937
6446
  speakBridge: (bridge) =>
5938
6447
  this.speakEscalationBridge(activeTurn, bridge),
5939
6448
  startEscalatedLeg: (escalated) => {
@@ -5947,6 +6456,107 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5947
6456
  activeTurn.frontDoor = coordinator;
5948
6457
  }
5949
6458
 
6459
+ // The leg's completion. A front-door answer held for the escalation judge
6460
+ // is spoken or overruled first, so the turn never closes its TTS buffer
6461
+ // under text that has not been released yet.
6462
+ const completeLeg = (
6463
+ msg: Parameters<NonNullable<VoiceTurnCallbacks["message_complete"]>>[0],
6464
+ ): void => {
6465
+ if (
6466
+ coordinator?.awaitingJudge === true &&
6467
+ msg.type === "message_complete"
6468
+ ) {
6469
+ void coordinator.settled().then(() => completeLeg(msg));
6470
+ return;
6471
+ }
6472
+ const current = this.activeAssistantTurn;
6473
+ if (
6474
+ current?.token !== token ||
6475
+ current.assistantCompleted ||
6476
+ // A barged-in turn finalizes through cancelAssistantTurn.
6477
+ current.abortController.signal.aborted ||
6478
+ this.isClosed
6479
+ ) {
6480
+ return;
6481
+ }
6482
+ if (!leg.frontDoor) {
6483
+ this.claimSuspendedForegroundTask(current);
6484
+ }
6485
+ // A speculative leg that finished without a single delta (empty
6486
+ // output, provider hiccup) carries no verdict, so fail open to a
6487
+ // committed turn so the utterance releases and finalizes like a
6488
+ // normal empty completion instead of dangling un-released.
6489
+ if (current.speculativePending) {
6490
+ this.commitSpeculativeTurn(current);
6491
+ }
6492
+ // A front-door leg that stopped mid-bridge hands off now with
6493
+ // whatever arrived. A cancellation mid-bridge falls through to
6494
+ // normal cancelled finalization instead: a dead turn must not
6495
+ // spawn an escalated leg. A front-door leg that handed off is
6496
+ // finished; the escalated leg drives completion, so this leg's
6497
+ // own trailing completion (including the generation_cancelled
6498
+ // from its abort) is a no-op.
6499
+ if (coordinator !== null) {
6500
+ if (msg.type === "message_complete") {
6501
+ coordinator.complete();
6502
+ }
6503
+ if (coordinator.handedOff) {
6504
+ return;
6505
+ }
6506
+ }
6507
+ // A held "[…"-tail that never completed a marker is real text:
6508
+ // force-flush it before assistantCompleted closes the TTS buffer
6509
+ // and completeTtsForTurn signals the drain, so it is spoken and
6510
+ // emitted rather than dropped.
6511
+ if (!leg.frontDoor && msg.type === "message_complete") {
6512
+ flushLegText(rawText, { force: true });
6513
+ }
6514
+ // Read off the leg that finished the reply: a front-door answer
6515
+ // or the escalated leg. A handed-off front-door leg returned
6516
+ // above, so its holding phrase can never end a call.
6517
+ if (msg.type === "message_complete") {
6518
+ current.notificationHandledSilently =
6519
+ current.taskOutcome !== null &&
6520
+ rawText.trim() === TASK_UPDATE_SILENT_MARKER;
6521
+ const request = requestedSessionControl(rawText, this.sessionControls);
6522
+ if (request?.action === "updates") {
6523
+ // The session's own control: nothing to send, and nothing to
6524
+ // wait for, since it shapes turns that have not started yet.
6525
+ this.progressCadence = request.cadence;
6526
+ log.info(
6527
+ { turnId, cadence: request.cadence },
6528
+ "Live voice progress cadence changed",
6529
+ );
6530
+ } else if (request?.action === "task_stop") {
6531
+ this.stopOutstandingInterruptedWork("spoken_task_stop");
6532
+ } else {
6533
+ if (request?.action === "end") {
6534
+ this.clearForegroundTask("call_end_requested");
6535
+ }
6536
+ current.sessionControlRequested = request;
6537
+ }
6538
+ }
6539
+ if (leg.frontDoor) {
6540
+ this.queueForegroundTaskResume(current);
6541
+ }
6542
+ current.assistantCompleted = true;
6543
+ if (msg.type === "generation_cancelled") {
6544
+ void this.finalizeAssistantTurn(
6545
+ current,
6546
+ "cancelled",
6547
+ "generation_cancelled",
6548
+ );
6549
+ return;
6550
+ }
6551
+ current.assistantMessageId = msg.messageId ?? null;
6552
+ this.completeTtsForTurn(token);
6553
+ };
6554
+
6555
+ let finishStart!: () => void;
6556
+ const startup = new Promise<void>((resolve) => {
6557
+ finishStart = resolve;
6558
+ });
6559
+ this.pendingAssistantStarts.add(startup);
5950
6560
  try {
5951
6561
  // Latched before the await, not after: this flag only decides whether the
5952
6562
  // end event carries a silence classification, and the dashboard decides
@@ -5969,7 +6579,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5969
6579
  const handle = await this.startVoiceTurn({
5970
6580
  conversationId: this.conversationId,
5971
6581
  voiceSessionId: this.context.sessionId,
6582
+ ...(activeTurn.modeSessionSource && modeSessionRequestId
6583
+ ? {
6584
+ preacceptedModeSession: {
6585
+ requestId: modeSessionRequestId,
6586
+ source: activeTurn.modeSessionSource,
6587
+ },
6588
+ }
6589
+ : {}),
5972
6590
  ...(activeTurn.hiddenPrompt ? { hiddenSyntheticPrompt: true } : {}),
6591
+ ...(activeTurn.lookFollowUp !== null
6592
+ ? { routingUtterance: activeTurn.lookFollowUp.callerUtterance }
6593
+ : {}),
5973
6594
  userMessageChannel: "vellum",
5974
6595
  assistantMessageChannel: "vellum",
5975
6596
  // Fixed, and NOT the originating client: this pair resolves the turn's
@@ -6008,16 +6629,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6008
6629
  onApprovalsResolved: () => {
6009
6630
  this.clearAwaitingApproval(activeTurn);
6010
6631
  },
6632
+ ...(leg.routingLeg === "escalated"
6633
+ ? {
6634
+ onEscalationTargetResolved: (target) => {
6635
+ if (!this.isActiveAssistantTurn(token)) {
6636
+ return;
6637
+ }
6638
+ activeTurn.escalationTarget = target;
6639
+ this.publishEscalationActivity(activeTurn, target);
6640
+ },
6641
+ }
6642
+ : {}),
6011
6643
  content: leg.content,
6012
6644
  ...(leg.attachments ? { attachments: leg.attachments } : {}),
6013
6645
  isInbound: true,
6014
6646
  launchedAtMs: activeTurn.launchedAtMs,
6015
6647
  signal: activeTurn.abortController.signal,
6016
- // An announcement turn's content is a fixed marker, not user speech:
6017
- // persist it hidden and suppress its echo so nothing renders as a user
6018
- // bubble for a turn the user never started.
6019
- ...(activeTurn.continuationDelivery !== null
6020
- ? { hiddenSyntheticPrompt: true }
6648
+ ...(activeTurn.taskOutcome !== null &&
6649
+ activeTurn.taskOutcome.source === "subagent"
6650
+ ? { subagentNotification: activeTurn.taskOutcome }
6021
6651
  : {}),
6022
6652
  ...(leg.overrideProfile != null
6023
6653
  ? { overrideProfile: leg.overrideProfile }
@@ -6070,90 +6700,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6070
6700
  rawText += msg.text;
6071
6701
  flushLegText(rawText);
6072
6702
  },
6073
- message_complete: (msg) => {
6074
- const current = this.activeAssistantTurn;
6075
- if (
6076
- current?.token !== token ||
6077
- current.assistantCompleted ||
6078
- // A barged-in turn finalizes through cancelAssistantTurn.
6079
- current.abortController.signal.aborted ||
6080
- this.isClosed
6081
- ) {
6082
- return;
6083
- }
6084
- if (!leg.frontDoor) {
6085
- this.claimSuspendedForegroundTask(current);
6086
- }
6087
- // A speculative leg that finished without a single delta (empty
6088
- // output, provider hiccup) carries no verdict — fail open to a
6089
- // committed turn so the utterance releases and finalizes like a
6090
- // normal empty completion instead of dangling un-released.
6091
- if (current.speculativePending) {
6092
- this.commitSpeculativeTurn(current);
6093
- }
6094
- // A front-door leg that stopped mid-bridge hands off now with
6095
- // whatever arrived. A cancellation mid-bridge falls through to
6096
- // normal cancelled finalization instead: a dead turn must not
6097
- // spawn an escalated leg. A front-door leg that handed off is
6098
- // finished; the escalated leg drives completion, so this leg's
6099
- // own trailing completion (including the generation_cancelled
6100
- // from its abort) is a no-op.
6101
- if (coordinator !== null) {
6102
- if (msg.type === "message_complete") {
6103
- coordinator.complete();
6104
- }
6105
- if (coordinator.handedOff) {
6106
- return;
6107
- }
6108
- }
6109
- // A held "[…"-tail that never completed a marker is real text —
6110
- // force-flush it before assistantCompleted closes the TTS buffer
6111
- // and completeTtsForTurn signals the drain, so it is spoken and
6112
- // emitted rather than dropped.
6113
- if (!leg.frontDoor && msg.type === "message_complete") {
6114
- flushLegText(rawText, { force: true });
6115
- }
6116
- // Read off the leg that finished the reply: a front-door answer
6117
- // or the escalated leg. A handed-off front-door leg returned
6118
- // above, so its holding phrase can never end a call.
6119
- if (msg.type === "message_complete") {
6120
- const request = requestedSessionControl(
6121
- rawText,
6122
- this.sessionControls,
6123
- );
6124
- if (request?.action === "updates") {
6125
- // The session's own control: nothing to send, and nothing to
6126
- // wait for, since it shapes turns that have not started yet.
6127
- this.progressCadence = request.cadence;
6128
- log.info(
6129
- { turnId, cadence: request.cadence },
6130
- "Live voice progress cadence changed",
6131
- );
6132
- } else if (request?.action === "task_stop") {
6133
- this.stopOutstandingInterruptedWork("spoken_task_stop");
6134
- } else {
6135
- if (request?.action === "end") {
6136
- this.clearForegroundTask("call_end_requested");
6137
- }
6138
- current.sessionControlRequested = request;
6139
- }
6140
- }
6141
- if (leg.frontDoor) {
6142
- this.queueForegroundTaskResume(current);
6143
- }
6144
- current.assistantCompleted = true;
6145
- if (msg.type === "generation_cancelled") {
6146
- void this.finalizeAssistantTurn(
6147
- current,
6148
- "cancelled",
6149
- "generation_cancelled",
6150
- );
6151
- return;
6152
- }
6153
- current.assistantMessageId = msg.messageId ?? null;
6154
- current.continuationDelivery = null;
6155
- this.completeTtsForTurn(token);
6156
- },
6703
+ message_complete: completeLeg,
6157
6704
  persisted_user_message_id: (messageId) => {
6158
6705
  const current = this.activeAssistantTurn;
6159
6706
  // Only the first leg's user row is the real caller utterance; the
@@ -6212,7 +6759,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6212
6759
  foregroundToolContendsWithContinuation(toolName, detail?.input)
6213
6760
  ) {
6214
6761
  this.abortDetachedRuns({
6215
- keepPendingResult: true,
6216
6762
  reason: "foreground_tool_contends",
6217
6763
  toolName,
6218
6764
  });
@@ -6281,7 +6827,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6281
6827
  })();
6282
6828
  },
6283
6829
  });
6284
-
6285
6830
  const current = this.activeAssistantTurn;
6286
6831
  if (current?.token !== token) {
6287
6832
  // A discard that beat this handle's resolution still owes the
@@ -6324,8 +6869,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6324
6869
  }
6325
6870
 
6326
6871
  current.handle = handle;
6872
+ if (coordinator !== null && handle.escalationJudgement) {
6873
+ coordinator.attachEscalationJudge(handle.escalationJudgement);
6874
+ }
6327
6875
  return true;
6328
6876
  } catch (err) {
6877
+ if (modeSessionRequestId) {
6878
+ this.releaseModeSessionTurnId(modeSessionRequestId);
6879
+ }
6329
6880
  if (!this.isActiveAssistantTurn(token)) {
6330
6881
  return false;
6331
6882
  }
@@ -6338,6 +6889,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6338
6889
  this.rearmForegroundTaskAfterLegFailure(activeTurn);
6339
6890
  this.clearFillerTimers(activeTurn);
6340
6891
  this.clearActiveAssistantTurn(token);
6892
+ this.releaseModeSessionRequests(activeTurn);
6893
+ this.releaseModeSessionDelivery(activeTurn);
6341
6894
  if (foregroundTaskResumeRearmed) {
6342
6895
  this.scheduleForegroundTaskResume();
6343
6896
  }
@@ -6351,6 +6904,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6351
6904
  await this.finalizePendingUtterance(utterance, "assistant_start_error");
6352
6905
  this.scheduleRearmAfterTurn();
6353
6906
  return false;
6907
+ } finally {
6908
+ this.pendingAssistantStarts.delete(startup);
6909
+ finishStart();
6354
6910
  }
6355
6911
  }
6356
6912
 
@@ -6377,15 +6933,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6377
6933
  // otherwise sit buffered until a sentence boundary and leave the
6378
6934
  // caller in silence during the escalated model's call.
6379
6935
  this.flushTtsBuffer(activeTurn.token, true);
6380
- return;
6381
- }
6382
- // The canned bridge is a fixed localized-table phrase, enqueued
6383
- // directly (it is already one complete sentence) so the segment can
6384
- // carry the "en" override when the table lacks the turn's language.
6385
- const speakable = sanitizeForTts(spokenBridge).trim();
6386
- if (speakable.length > 0) {
6387
- this.enqueueTtsSegment(activeTurn.token, speakable, { language });
6936
+ } else {
6937
+ // The canned bridge is a fixed localized-table phrase, enqueued
6938
+ // directly (it is already one complete sentence) so the segment can
6939
+ // carry the "en" override when the table lacks the turn's language.
6940
+ const speakable = sanitizeForTts(spokenBridge).trim();
6941
+ if (speakable.length > 0) {
6942
+ this.enqueueTtsSegment(activeTurn.token, speakable, { language });
6943
+ }
6388
6944
  }
6945
+
6946
+ // Target resolution can land before the bridge's queued audio finishes.
6947
+ // Its activity frame moves the system surface to thinking, then a late
6948
+ // bridge tts_audio frame moves it back to speaking. Restore the neutral
6949
+ // waiting phase at the bridge's emission boundary, ahead of any answer
6950
+ // audio queued behind it.
6951
+ const bridgeDrain = activeTurn.ttsQueue;
6952
+ void bridgeDrain.then(
6953
+ () => {
6954
+ const target = activeTurn.escalationTarget;
6955
+ if (
6956
+ target === null ||
6957
+ !this.isActiveAssistantTurn(activeTurn.token) ||
6958
+ activeTurn.assistantCompleted
6959
+ ) {
6960
+ return;
6961
+ }
6962
+ if (activeTurn.publishedActivityKind === "escalation") {
6963
+ this.publishEscalationActivity(activeTurn, target);
6964
+ return;
6965
+ }
6966
+ this.liveActivityReporter.restoreThinkingPhase();
6967
+ },
6968
+ () => undefined,
6969
+ );
6389
6970
  }
6390
6971
 
6391
6972
  private async cancelAssistantTurn(reason: string): Promise<void> {
@@ -6607,7 +7188,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6607
7188
  isLookSessionControl(sessionControl.action) &&
6608
7189
  currentTurn.lookFollowUp === null
6609
7190
  ) {
6610
- this.awaitLookFrame(sessionControl.action);
7191
+ this.awaitLookFrame(sessionControl.action, currentTurn);
6611
7192
  }
6612
7193
  await this.sendFrame(
6613
7194
  {
@@ -6643,6 +7224,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6643
7224
  if (!currentTurn.abortController.signal.aborted) {
6644
7225
  this.scheduleRearmAfterTurn();
6645
7226
  }
7227
+ })
7228
+ .finally(() => {
7229
+ this.releaseModeSessionDelivery(activeTurn);
6646
7230
  });
6647
7231
  }
6648
7232
 
@@ -6696,6 +7280,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6696
7280
  }
6697
7281
  const job: TtsSegmentJob = {
6698
7282
  text: segment,
7283
+ isReply: options.countsAsFirstSegment ?? true,
7284
+ audioSent: false,
6699
7285
  language: options.language,
6700
7286
  started: false,
6701
7287
  settled: false,
@@ -6813,6 +7399,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6813
7399
  await job.frames;
6814
7400
 
6815
7401
  if (failed && this.isForwardingTts(token)) {
7402
+ currentTurn.ttsFailed = true;
6816
7403
  // Per-segment failure: the turn (and session) continue, so the
6817
7404
  // error is recoverable for the client.
6818
7405
  await this.sendFrame(
@@ -6825,6 +7412,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6825
7412
  () => this.isForwardingTts(token),
6826
7413
  );
6827
7414
  }
7415
+ } catch (err) {
7416
+ const turn = this.activeAssistantTurn;
7417
+ if (turn?.token === token) {
7418
+ turn.ttsFailed = true;
7419
+ }
7420
+ throw err;
6828
7421
  } finally {
6829
7422
  job.settled = true;
6830
7423
  const settledTurn = this.activeAssistantTurn;
@@ -6849,16 +7442,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6849
7442
  if (activeTurn?.token !== token) {
6850
7443
  return;
6851
7444
  }
6852
- // Only retain the assistant TTS audio when it will be archived (see
6853
- // collectUserAudio); the mime/sample-rate are cheap and left unconditional.
6854
- if (this.archiveAudio) {
6855
- activeTurn.assistantAudioChunks.push(
6856
- Buffer.from(chunk.dataBase64, "base64"),
6857
- );
6858
- }
6859
- activeTurn.assistantAudioMimeType = chunk.contentType;
6860
- activeTurn.assistantAudioSampleRate = chunk.sampleRate;
6861
7445
  job.frames = job.frames.then(async () => {
7446
+ // Pace audio before entering the shared outbound queue so control frames
7447
+ // and cancellation remain immediate while prefetched speech waits.
7448
+ const leadMs = this.assistantPlaybackTailUntilMs - Date.now();
7449
+ if (
7450
+ leadMs > TTS_MAX_PLAYBACK_LEAD_MS &&
7451
+ !activeTurn.abortController.signal.aborted
7452
+ ) {
7453
+ try {
7454
+ await delay(leadMs - TTS_MAX_PLAYBACK_LEAD_MS, undefined, {
7455
+ signal: activeTurn.abortController.signal,
7456
+ });
7457
+ } catch (error) {
7458
+ if (activeTurn.abortController.signal.aborted) {
7459
+ return;
7460
+ }
7461
+ throw error;
7462
+ }
7463
+ }
6862
7464
  const sent = await this.sendFrame(
6863
7465
  {
6864
7466
  type: "tts_audio",
@@ -6868,13 +7470,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6868
7470
  },
6869
7471
  () => this.isForwardingTts(token),
6870
7472
  );
6871
- // Skip a frame that wasn't actually written — a backed-up outbound
7473
+ // Skip a frame that wasn't actually written. A backed-up outbound
6872
7474
  // queue hasn't reached the client, so it must not extend the
6873
7475
  // playback-tail estimate or latch first-audio state. Token match keeps
6874
7476
  // a stale turn's late send from latching a newer turn.
6875
7477
  if (!sent) {
6876
7478
  return;
6877
7479
  }
7480
+ if (this.archiveAudio) {
7481
+ activeTurn.assistantAudioChunks.push(
7482
+ Buffer.from(chunk.dataBase64, "base64"),
7483
+ );
7484
+ }
7485
+ activeTurn.assistantAudioMimeType = chunk.contentType;
7486
+ activeTurn.assistantAudioSampleRate = chunk.sampleRate;
7487
+ job.audioSent = true;
6878
7488
  // Extend the client playback-tail estimate by this chunk's PCM
6879
7489
  // duration (chunks queue gaplessly client-side, so the tail grows
6880
7490
  // from whichever is later: now or the current estimate).
@@ -6914,6 +7524,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6914
7524
  this.markUtteranceMetric(utterance, "speechStartAtMs", (turnId) =>
6915
7525
  this.metrics.markSpeechStart(turnId),
6916
7526
  );
7527
+ this.logInputDiagnostic("voice_input_speech_routed", {
7528
+ inputTurnId: utterance.turnId,
7529
+ });
6917
7530
  }
6918
7531
 
6919
7532
  /**
@@ -6956,6 +7569,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6956
7569
  }
6957
7570
 
6958
7571
  private markFirstPartial(utterance: UtteranceCycle): void {
7572
+ if ((utterance.latestPartialText?.trim() ?? "").length === 0) {
7573
+ return;
7574
+ }
6959
7575
  this.markUtteranceMetric(utterance, "firstPartialAtMs", (turnId) =>
6960
7576
  this.metrics.markFirstPartial(turnId),
6961
7577
  );
@@ -6965,6 +7581,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6965
7581
  this.markUtteranceMetric(utterance, "finalTranscriptAtMs", (turnId) =>
6966
7582
  this.metrics.markFinalTranscript(turnId),
6967
7583
  );
7584
+ this.logInputDiagnostic("voice_input_transcript", {
7585
+ inputTurnId: utterance.turnId,
7586
+ sttProvider: utterance.dialedSttProvider,
7587
+ transcriptChars: utterance.finalTranscriptSegments.join(" ").trim()
7588
+ .length,
7589
+ });
6968
7590
  }
6969
7591
 
6970
7592
  // Records the mark on the utterance's metrics turn, or — while a previous
@@ -7105,6 +7727,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
7105
7727
  utterance: UtteranceCycle,
7106
7728
  reason: string,
7107
7729
  ): Promise<void> {
7730
+ this.logInputDiagnostic("voice_input_utterance_discarded", {
7731
+ inputTurnId: utterance.turnId,
7732
+ reason,
7733
+ sttProvider: utterance.dialedSttProvider,
7734
+ transcriptChars: utterance.finalTranscriptSegments.join(" ").trim()
7735
+ .length,
7736
+ hasPartialAtDiscard: utterance.latestPartialText !== null,
7737
+ });
7108
7738
  // An utterance that finalizes here never became a turn (empty transcript,
7109
7739
  // client interrupt, transcriber close, error), so it ends the window a
7110
7740
  // barge-in's merge context was waiting to attach to. Drop that context so
@@ -7112,13 +7742,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
7112
7742
  // finalizes through finalizeAssistantTurn, not here, so this never clears a
7113
7743
  // request that the barge-in follow-up turn is still about to consume.
7114
7744
  this.pendingInterruptedRequest = null;
7115
- // `pendingContinuationResult` is deliberately NOT cleared here. Unlike the
7116
- // merge context (bound to the immediate barge-in follow-up), a completed
7117
- // continuation's answer targets the next REAL turn, and hands-free server-VAD
7118
- // routinely discards noise/empty-transcript utterances between turns; wiping
7119
- // it on each such discard would frequently drop a valid result before the
7120
- // user's next question. It is bounded instead by consume-once, the hard-stop
7121
- // clear (abortDetachedRuns), and the model's "use only if relevant" framing.
7122
7745
  utterance.completed = true;
7123
7746
  const turnId = utterance.turnId;
7124
7747
  if (!turnId) {
@@ -7147,6 +7770,42 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
7147
7770
  }
7148
7771
 
7149
7772
  turn.finalized = true;
7773
+ const reply = turn.ttsJobs
7774
+ .filter((job) => job.isReply && job.audioSent)
7775
+ .map((job) => job.text)
7776
+ .join(" ");
7777
+ if (turn.taskOutcome !== null) {
7778
+ if (
7779
+ status === "completed" &&
7780
+ !turn.ttsFailed &&
7781
+ (turn.notificationHandledSilently || reply.length > 0)
7782
+ ) {
7783
+ this.taskOutcomes.finish(
7784
+ turn.taskOutcome,
7785
+ turn.notificationHandledSilently
7786
+ ? Date.now()
7787
+ : this.assistantPlaybackTailUntilMs,
7788
+ turn.notificationHandledSilently ? "" : reply,
7789
+ );
7790
+ } else {
7791
+ if (
7792
+ reason === "barge_in" ||
7793
+ reason === "interrupt" ||
7794
+ reason === "session_closed"
7795
+ ) {
7796
+ this.taskOutcomes.interrupt(turn.taskOutcome);
7797
+ }
7798
+ if (reason !== "barge_in") {
7799
+ this.taskAnnouncementsDeferred = true;
7800
+ }
7801
+ }
7802
+ } else if (status === "completed" && !turn.ttsFailed && reply.length > 0) {
7803
+ this.taskOutcomes.finishContextReply(
7804
+ turn.contextOutcomes,
7805
+ this.assistantPlaybackTailUntilMs,
7806
+ reply,
7807
+ );
7808
+ }
7150
7809
  this.clearFillerTimers(turn);
7151
7810
  if (
7152
7811
  this.foregroundTaskState?.phase === "owned" &&
@@ -7169,6 +7828,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
7169
7828
  : {}),
7170
7829
  });
7171
7830
  await this.finishMetricsTurn(turn.utterance, status, reason, turn.turnId);
7831
+ this.releaseModeSessionRequests(turn);
7832
+
7833
+ if (status === "cancelled") {
7834
+ this.releaseModeSessionDelivery(turn);
7835
+ }
7172
7836
 
7173
7837
  if (
7174
7838
  (options.clearActive ?? true) &&
@@ -7183,6 +7847,36 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
7183
7847
  }
7184
7848
  }
7185
7849
 
7850
+ private releaseModeSessionDelivery(turn: ActiveAssistantTurn): void {
7851
+ const turnId = turn.modeSessionDeliveryTurnId;
7852
+ if (turnId === null) {
7853
+ return;
7854
+ }
7855
+ turn.modeSessionDeliveryTurnId = null;
7856
+ this.releaseModeSessionTurnId(turnId);
7857
+ }
7858
+
7859
+ private releaseModeSessionRequests(turn: ActiveAssistantTurn): void {
7860
+ this.releaseModeSessionTurnIds(turn.modeSessionRequestIds.splice(0));
7861
+ }
7862
+
7863
+ private releaseModeSessionTurnIds(turnIds: readonly string[]): void {
7864
+ for (const turnId of turnIds) {
7865
+ this.releaseModeSessionTurnId(turnId);
7866
+ }
7867
+ }
7868
+
7869
+ private releaseModeSessionTurnId(turnId: string): void {
7870
+ try {
7871
+ this.cameraModeSessions?.releaseDelivery(turnId);
7872
+ } catch (err) {
7873
+ log.warn(
7874
+ { err, turnId },
7875
+ "Could not release camera session ownership from voice turn",
7876
+ );
7877
+ }
7878
+ }
7879
+
7186
7880
  private async archiveBufferedAudio(input: {
7187
7881
  turnId: string;
7188
7882
  userMessageId: string | null;
@@ -7330,7 +8024,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
7330
8024
  }
7331
8025
 
7332
8026
  private async emitMetricsFrame(
7333
- event: LiveVoiceMetricsEvent,
8027
+ event: VoiceMetricsEvent,
7334
8028
  turnId = this.currentUtterance?.turnId ?? this.context.sessionId,
7335
8029
  ): Promise<void> {
7336
8030
  const metrics = this.metrics.getSnapshot();
@@ -7341,7 +8035,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
7341
8035
  conversationId: this.conversationId,
7342
8036
  turnId,
7343
8037
  metrics,
7344
- ...getLiveVoiceMetricsAggregateFields(metrics, turnId),
8038
+ ...getVoiceMetricsAggregateFields(metrics, turnId),
7345
8039
  });
7346
8040
  }
7347
8041
 
@@ -7473,6 +8167,8 @@ export function createLiveVoiceSession(
7473
8167
  options.spawnBackgroundContinuation ?? defaultSpawnBackgroundContinuation,
7474
8168
  labelBackgroundContinuation:
7475
8169
  options.labelBackgroundContinuation ?? createContinuationLabeler(),
8170
+ judgeBackgroundContinuation:
8171
+ options.judgeBackgroundContinuation ?? createContinuationJudge(),
7476
8172
  getTurnTeardown: options.getTurnTeardown ?? getConversationTurnTeardown,
7477
8173
  // Off by default (see the `liveVoice.archiveAudio` schema): voice turns
7478
8174
  // persist only their transcribed text, so the recorded audio never lands
@@ -7802,7 +8498,7 @@ async function defaultArchiveLiveVoiceAudio(
7802
8498
  : linkLiveVoiceAssistantResponseAudioToMessage(input);
7803
8499
  }
7804
8500
 
7805
- function toSeedMarks(stashed: StashedMetricsMarks): LiveVoiceTurnSeedMarks {
8501
+ function toSeedMarks(stashed: StashedMetricsMarks): VoiceTurnSeedMarks {
7806
8502
  return {
7807
8503
  ...(stashed.firstAudioAtMs !== null
7808
8504
  ? { firstAudioAtMs: stashed.firstAudioAtMs }