@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (826) hide show
  1. package/ARCHITECTURE.md +69 -9
  2. package/docs/flux-turn-detection-spike.md +243 -0
  3. package/docs/stt-provider-onboarding.md +3 -1
  4. package/docs/trusted-contact-access.md +9 -5
  5. package/docs/vellum-doctor.md +195 -0
  6. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  7. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  8. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  9. package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
  10. package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
  11. package/node_modules/@vellumai/environments/package.json +2 -1
  12. package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
  13. package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
  14. package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
  15. package/node_modules/@vellumai/environments/src/index.ts +17 -6
  16. package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
  17. package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
  18. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  19. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  20. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  21. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
  22. package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
  23. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
  24. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
  25. package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
  26. package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
  27. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
  28. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
  29. package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  30. package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  31. package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  32. package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
  33. package/openapi.yaml +619 -47
  34. package/package.json +1 -1
  35. package/scripts/voice-ttft-spike.ts +3 -3
  36. package/src/__tests__/agent-image-optimize.test.ts +4 -4
  37. package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
  38. package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
  39. package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
  40. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
  41. package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
  42. package/src/__tests__/answered-question-persistence.test.ts +240 -0
  43. package/src/__tests__/anthropic-provider.test.ts +165 -30
  44. package/src/__tests__/app-control-flow.test.ts +9 -8
  45. package/src/__tests__/assistant-event-hub.test.ts +48 -0
  46. package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
  47. package/src/__tests__/attachments-store.test.ts +130 -68
  48. package/src/__tests__/attachments.test.ts +36 -36
  49. package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
  50. package/src/__tests__/call-controller.test.ts +120 -0
  51. package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
  52. package/src/__tests__/canned-reply-release.test.ts +86 -0
  53. package/src/__tests__/channel-availability-routes.test.ts +14 -1
  54. package/src/__tests__/channel-guardian.test.ts +59 -29
  55. package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
  56. package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
  57. package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
  58. package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
  59. package/src/__tests__/config-loader-backfill.test.ts +19 -0
  60. package/src/__tests__/config-schema.test.ts +150 -7
  61. package/src/__tests__/consult-deadline.test.ts +65 -0
  62. package/src/__tests__/conversation-agent-loop.test.ts +438 -2
  63. package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
  64. package/src/__tests__/conversation-disk-view.test.ts +4 -4
  65. package/src/__tests__/conversation-error.test.ts +22 -3
  66. package/src/__tests__/conversation-fork-crud.test.ts +5 -1
  67. package/src/__tests__/conversation-fork-referential.test.ts +352 -0
  68. package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
  69. package/src/__tests__/conversation-group-tools.test.ts +32 -2
  70. package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
  71. package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
  72. package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
  73. package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
  74. package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
  75. package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
  76. package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
  77. package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
  78. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
  79. package/src/__tests__/conversation-queue.test.ts +433 -1
  80. package/src/__tests__/conversation-retry-route.test.ts +3 -1
  81. package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
  82. package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
  83. package/src/__tests__/conversation-store.test.ts +26 -10
  84. package/src/__tests__/conversation-summarize-route.test.ts +1 -1
  85. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
  86. package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
  87. package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
  88. package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
  89. package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
  90. package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
  91. package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
  92. package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
  93. package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
  94. package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
  95. package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
  96. package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
  97. package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
  98. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
  99. package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
  100. package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
  101. package/src/__tests__/credential-prompt-route.test.ts +54 -7
  102. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  103. package/src/__tests__/cu-unified-flow.test.ts +89 -28
  104. package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
  105. package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
  106. package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
  107. package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
  108. package/src/__tests__/disk-pressure-guard.test.ts +55 -55
  109. package/src/__tests__/disk-pressure-routes.test.ts +10 -10
  110. package/src/__tests__/disk-pressure-tools.test.ts +2 -2
  111. package/src/__tests__/disk-usage.test.ts +35 -20
  112. package/src/__tests__/document-sync-tags.test.ts +379 -0
  113. package/src/__tests__/document-update-default-surface.test.ts +28 -0
  114. package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
  115. package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
  116. package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
  117. package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
  118. package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
  119. package/src/__tests__/file-list-tool.test.ts +12 -10
  120. package/src/__tests__/file-ops-service.test.ts +151 -44
  121. package/src/__tests__/file-read-container-boundary.test.ts +137 -0
  122. package/src/__tests__/file-read-tool.test.ts +21 -9
  123. package/src/__tests__/file-url-path-guard.test.ts +83 -0
  124. package/src/__tests__/filesystem-tools.test.ts +82 -66
  125. package/src/__tests__/guardian-outbound-http.test.ts +8 -8
  126. package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
  127. package/src/__tests__/helpers/hub-clients.ts +42 -0
  128. package/src/__tests__/helpers/mock-conversation.ts +46 -0
  129. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
  130. package/src/__tests__/history-repair.test.ts +112 -12
  131. package/src/__tests__/host-bash-proxy.test.ts +10 -5
  132. package/src/__tests__/host-bash-routes.test.ts +67 -0
  133. package/src/__tests__/host-browser-proxy.test.ts +2 -2
  134. package/src/__tests__/host-cu-proxy.test.ts +3 -7
  135. package/src/__tests__/host-file-proxy.test.ts +3 -7
  136. package/src/__tests__/host-file-read-tool.test.ts +5 -1
  137. package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
  138. package/src/__tests__/http-conversation-lineage.test.ts +2 -5
  139. package/src/__tests__/http-user-message-parity.test.ts +44 -0
  140. package/src/__tests__/identity-routes.test.ts +13 -13
  141. package/src/__tests__/image-conversion.test.ts +29 -26
  142. package/src/__tests__/image-source-path-reinject.test.ts +2 -2
  143. package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
  144. package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
  145. package/src/__tests__/introduction-card-resolver.test.ts +6 -1
  146. package/src/__tests__/list-messages-attachments.test.ts +56 -32
  147. package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
  148. package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
  149. package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
  150. package/src/__tests__/list-messages-page-latest.test.ts +61 -61
  151. package/src/__tests__/list-messages-provider-error.test.ts +6 -6
  152. package/src/__tests__/list-messages-queued.test.ts +49 -6
  153. package/src/__tests__/list-messages-system-card.test.ts +4 -4
  154. package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
  155. package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
  156. package/src/__tests__/llm-usage-store.test.ts +55 -0
  157. package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
  158. package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
  159. package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
  160. package/src/__tests__/media-stream-output.test.ts +175 -0
  161. package/src/__tests__/media-stream-server-integration.test.ts +36 -0
  162. package/src/__tests__/media-stream-stt-session.test.ts +67 -0
  163. package/src/__tests__/migration-export-http.test.ts +1 -1
  164. package/src/__tests__/migration-import-commit-http.test.ts +1 -1
  165. package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
  166. package/src/__tests__/migration-validate-http.test.ts +1 -1
  167. package/src/__tests__/mtime-cache.test.ts +65 -3
  168. package/src/__tests__/notification-platform-adapter.test.ts +104 -0
  169. package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
  170. package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
  171. package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
  172. package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
  173. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  174. package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
  175. package/src/__tests__/path-policy.test.ts +164 -1
  176. package/src/__tests__/persist-media-references.test.ts +2 -2
  177. package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
  178. package/src/__tests__/persist-unsendable-image.test.ts +23 -23
  179. package/src/__tests__/persona-resolver.test.ts +154 -0
  180. package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
  181. package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
  182. package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
  183. package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
  184. package/src/__tests__/profiler-routes.test.ts +13 -13
  185. package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
  186. package/src/__tests__/proxy-approval-callback.test.ts +3 -3
  187. package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
  188. package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
  189. package/src/__tests__/resolve-trust-class.test.ts +20 -0
  190. package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
  191. package/src/__tests__/run-due-schedules.test.ts +174 -0
  192. package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
  193. package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
  194. package/src/__tests__/schedule-routes.test.ts +730 -8
  195. package/src/__tests__/schedule-store.test.ts +540 -0
  196. package/src/__tests__/schedule-tools.test.ts +103 -0
  197. package/src/__tests__/scheduler-wake.test.ts +5 -0
  198. package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
  199. package/src/__tests__/secure-keys.test.ts +91 -0
  200. package/src/__tests__/server-history-render.test.ts +97 -7
  201. package/src/__tests__/size-guard.test.ts +8 -8
  202. package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
  203. package/src/__tests__/skill-tool-factory.test.ts +247 -0
  204. package/src/__tests__/skills-install-staging.test.ts +15 -6
  205. package/src/__tests__/skills.test.ts +32 -0
  206. package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
  207. package/src/__tests__/slack-inbound-verification.test.ts +4 -1
  208. package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
  209. package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
  210. package/src/__tests__/starter-task-flow.test.ts +5 -4
  211. package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
  212. package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
  213. package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
  214. package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
  215. package/src/__tests__/subagent-disposal.test.ts +10 -0
  216. package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
  217. package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
  218. package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
  219. package/src/__tests__/subagent-manager-notify.test.ts +148 -0
  220. package/src/__tests__/subagent-notify-parent.test.ts +1 -1
  221. package/src/__tests__/subagent-persistence.test.ts +141 -0
  222. package/src/__tests__/subagent-role-registry.test.ts +172 -63
  223. package/src/__tests__/subagent-role-resolution.test.ts +88 -0
  224. package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
  225. package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
  226. package/src/__tests__/subagent-terminal-message.test.ts +88 -1
  227. package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
  228. package/src/__tests__/subagent-tools.test.ts +1901 -108
  229. package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
  230. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
  231. package/src/__tests__/system-prompt.test.ts +82 -7
  232. package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
  233. package/src/__tests__/tool-result-spool.test.ts +74 -18
  234. package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
  235. package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
  236. package/src/__tests__/tools-audio-read.test.ts +10 -10
  237. package/src/__tests__/tools-get-route.test.ts +85 -12
  238. package/src/__tests__/tools-list-cli.test.ts +171 -0
  239. package/src/__tests__/top-level-renderer.test.ts +54 -1
  240. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
  241. package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
  242. package/src/__tests__/turn-tail-chain.test.ts +135 -0
  243. package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
  244. package/src/__tests__/ui-visual-surface.test.ts +5 -4
  245. package/src/__tests__/ui-work-result-surface.test.ts +5 -4
  246. package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
  247. package/src/__tests__/usage-routes.test.ts +62 -0
  248. package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
  249. package/src/__tests__/visible-app-context.test.ts +20 -4
  250. package/src/__tests__/web-search-history.test.ts +96 -0
  251. package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
  252. package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
  253. package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
  254. package/src/acp/failure-error.ts +3 -5
  255. package/src/agent/attachments.ts +26 -24
  256. package/src/agent/history-repair/history-repair.ts +27 -26
  257. package/src/agent/image-optimize.ts +6 -6
  258. package/src/agent/loop.ts +40 -22
  259. package/src/agent/message-types.ts +3 -3
  260. package/src/api/constants/document-tools.ts +55 -0
  261. package/src/api/events/question-answered.ts +53 -0
  262. package/src/api/events/tool-result.ts +7 -0
  263. package/src/api/index.ts +13 -0
  264. package/src/api/responses/conversation-message.ts +11 -0
  265. package/src/api/surfaces-oauth-connect.test.ts +96 -0
  266. package/src/api/surfaces.ts +36 -8
  267. package/src/approvals/guardian-channel-delivery.ts +83 -0
  268. package/src/approvals/guardian-expiry-notifier.ts +13 -11
  269. package/src/approvals/guardian-request-resolvers.ts +222 -135
  270. package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
  271. package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
  272. package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
  273. package/src/calls/call-controller.ts +30 -2
  274. package/src/calls/call-speech-output.ts +12 -4
  275. package/src/calls/call-transport.ts +18 -1
  276. package/src/calls/media-stream-output.ts +53 -5
  277. package/src/calls/media-stream-server.ts +21 -1
  278. package/src/calls/media-stream-stt-session.ts +49 -0
  279. package/src/calls/telephony-synthesis-language.ts +84 -0
  280. package/src/calls/tts-text-sanitizer.ts +12 -5
  281. package/src/calls/voice-session-bridge.ts +86 -17
  282. package/src/calls/voice-triage-escalate.ts +54 -7
  283. package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
  284. package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
  285. package/src/channels/config.ts +20 -2
  286. package/src/channels/gateway-verification-sessions.ts +7 -2
  287. package/src/channels/plugin-channel-declarations.ts +108 -0
  288. package/src/channels/types.ts +37 -1
  289. package/src/cli/AGENTS.md +5 -2
  290. package/src/cli/commands/__tests__/cache.test.ts +7 -4
  291. package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
  292. package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
  293. package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
  294. package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
  295. package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
  296. package/src/cli/commands/__tests__/plugins.test.ts +692 -0
  297. package/src/cli/commands/__tests__/schedules.test.ts +136 -37
  298. package/src/cli/commands/conversations.ts +7 -3
  299. package/src/cli/commands/credentials.help.ts +7 -0
  300. package/src/cli/commands/inference-profiles.ts +9 -3
  301. package/src/cli/commands/inference.help.ts +16 -2
  302. package/src/cli/commands/mcp.help.ts +17 -6
  303. package/src/cli/commands/mcp.ts +9 -0
  304. package/src/cli/commands/memory/memory-retrospective.ts +18 -0
  305. package/src/cli/commands/notifications.ts +3 -6
  306. package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
  307. package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
  308. package/src/cli/commands/oauth/connect.test.ts +26 -1
  309. package/src/cli/commands/oauth/connect.ts +4 -1
  310. package/src/cli/commands/oauth/index.help.ts +3 -1
  311. package/src/cli/commands/oauth/providers.ts +3 -0
  312. package/src/cli/commands/plugins.help.ts +25 -2
  313. package/src/cli/commands/plugins.ts +289 -10
  314. package/src/cli/commands/schedules.help.ts +14 -12
  315. package/src/cli/commands/schedules.ts +19 -5
  316. package/src/cli/commands/stt.help.ts +27 -2
  317. package/src/cli/commands/tools.help.ts +4 -2
  318. package/src/cli/commands/tools.ts +38 -0
  319. package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
  320. package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
  321. package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
  322. package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
  323. package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
  324. package/src/cli/lib/install-from-github.ts +61 -0
  325. package/src/cli/lib/install-from-platform.ts +13 -2
  326. package/src/cli/lib/plugin-surfaces.ts +124 -4
  327. package/src/cli/lib/upgrade-plugin.ts +60 -0
  328. package/src/cli.ts +1 -1
  329. package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
  330. package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
  331. package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
  332. package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
  333. package/src/config/__tests__/profile-tool-support.test.ts +131 -0
  334. package/src/config/bundled-skills/AGENTS.md +1 -1
  335. package/src/config/bundled-skills/acp/SKILL.md +6 -7
  336. package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
  337. package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
  338. package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
  339. package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
  340. package/src/config/bundled-skills/schedule/SKILL.md +5 -1
  341. package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
  342. package/src/config/bundled-skills/settings/TOOLS.json +3 -3
  343. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
  344. package/src/config/bundled-skills/subagent/SKILL.md +53 -22
  345. package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
  346. package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
  347. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
  348. package/src/config/bundled-skills/visualize/SKILL.md +1 -1
  349. package/src/config/call-site-defaults.ts +4 -5
  350. package/src/config/default-profile-catalog.ts +9 -5
  351. package/src/config/feature-flag-registry.json +25 -17
  352. package/src/config/llm-resolver.ts +45 -15
  353. package/src/config/loader.ts +75 -1
  354. package/src/config/profile-tool-support.ts +110 -0
  355. package/src/config/sanitize-for-transfer.ts +11 -0
  356. package/src/config/schema.ts +4 -0
  357. package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
  358. package/src/config/schemas/__tests__/stt.test.ts +20 -5
  359. package/src/config/schemas/call-site-catalog.ts +4 -4
  360. package/src/config/schemas/calls.ts +0 -4
  361. package/src/config/schemas/channels.ts +13 -0
  362. package/src/config/schemas/live-voice.ts +82 -23
  363. package/src/config/schemas/llm.ts +1 -1
  364. package/src/config/schemas/memory-retrospective.ts +25 -0
  365. package/src/config/schemas/plugin-updates.ts +59 -0
  366. package/src/config/schemas/services.ts +8 -4
  367. package/src/config/schemas/stt.ts +18 -5
  368. package/src/config/schemas/timeouts.ts +1 -1
  369. package/src/config/schemas/tts.ts +63 -0
  370. package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
  371. package/src/contacts/guardian-delivery-reader.ts +29 -2
  372. package/src/context/compactor.ts +4 -4
  373. package/src/context/post-turn-tool-result-truncation.ts +32 -16
  374. package/src/context/strip-injections.ts +42 -0
  375. package/src/context/tool-result-spool.ts +32 -18
  376. package/src/credential-execution/ces-runtime.ts +3 -0
  377. package/src/credential-execution/client.ts +22 -1
  378. package/src/credential-execution/process-manager.test.ts +99 -0
  379. package/src/credential-execution/process-manager.ts +62 -16
  380. package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
  381. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
  382. package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
  383. package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
  384. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
  385. package/src/daemon/channel-ui-capability.ts +4 -4
  386. package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
  387. package/src/daemon/conversation-agent-loop.ts +318 -148
  388. package/src/daemon/conversation-attachments.ts +1 -1
  389. package/src/daemon/conversation-error.ts +8 -1
  390. package/src/daemon/conversation-initial-prompt.ts +16 -38
  391. package/src/daemon/conversation-lifecycle.ts +154 -32
  392. package/src/daemon/conversation-messaging.ts +106 -43
  393. package/src/daemon/conversation-process.ts +131 -33
  394. package/src/daemon/conversation-queue-manager.ts +10 -0
  395. package/src/daemon/conversation-runtime-assembly.ts +9 -4
  396. package/src/daemon/conversation-store.ts +52 -3
  397. package/src/daemon/conversation-surface-state.ts +9 -0
  398. package/src/daemon/conversation-surfaces.ts +403 -210
  399. package/src/daemon/conversation-tool-setup.ts +56 -7
  400. package/src/daemon/conversation-turn-finalize.ts +134 -89
  401. package/src/daemon/conversation.ts +111 -27
  402. package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
  403. package/src/daemon/disk-pressure-guard.ts +3 -3
  404. package/src/daemon/doordash-steps.ts +5 -5
  405. package/src/daemon/external-plugins-bootstrap.ts +7 -1
  406. package/src/daemon/handlers/config-channels.ts +48 -5
  407. package/src/daemon/handlers/conversation-history.ts +1 -1
  408. package/src/daemon/handlers/conversations.ts +62 -17
  409. package/src/daemon/handlers/shared.ts +12 -0
  410. package/src/daemon/handlers/skills.ts +11 -1
  411. package/src/daemon/identity-helpers.ts +38 -10
  412. package/src/daemon/install-assistant-command.ts +322 -0
  413. package/src/daemon/lifecycle.ts +14 -3
  414. package/src/daemon/message-types/conversations.ts +6 -7
  415. package/src/daemon/message-types/sync.ts +1 -0
  416. package/src/daemon/persist-media-references.ts +23 -14
  417. package/src/daemon/process-message.ts +6 -6
  418. package/src/daemon/tool-setup-types.ts +19 -122
  419. package/src/daemon/tool-side-effects.ts +17 -94
  420. package/src/daemon/trust-context-types.ts +39 -0
  421. package/src/daemon/trust-context.ts +14 -6
  422. package/src/daemon/turn-tail-chain.ts +73 -0
  423. package/src/daemon/web-search-history.ts +47 -33
  424. package/src/heartbeat/heartbeat-service.ts +6 -6
  425. package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
  426. package/src/inbound/oauth-callback-url.ts +61 -0
  427. package/src/ipc/__tests__/socket-path.test.ts +21 -0
  428. package/src/ipc/assistant-server.ts +17 -10
  429. package/src/ipc/cli-client.ts +10 -1
  430. package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
  431. package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
  432. package/src/ipc/socket-cleanup.ts +9 -7
  433. package/src/ipc/socket-path.ts +4 -96
  434. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
  435. package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
  436. package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
  437. package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
  438. package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
  439. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
  440. package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
  441. package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
  442. package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
  443. package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
  444. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
  445. package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
  446. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
  447. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
  448. package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
  449. package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
  450. package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
  451. package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
  452. package/src/live-voice/__tests__/protocol.test.ts +52 -0
  453. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
  454. package/src/live-voice/live-voice-archive.ts +13 -13
  455. package/src/live-voice/live-voice-connection.ts +7 -1
  456. package/src/live-voice/live-voice-metrics.ts +57 -32
  457. package/src/live-voice/live-voice-photo.ts +168 -0
  458. package/src/live-voice/live-voice-session.ts +1137 -351
  459. package/src/live-voice/live-voice-tts.ts +18 -2
  460. package/src/live-voice/progress-narration.ts +277 -0
  461. package/src/live-voice/progress-phrases.ts +105 -2
  462. package/src/live-voice/protocol.ts +110 -2
  463. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
  464. package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
  465. package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
  466. package/src/mcp/client.ts +14 -7
  467. package/src/mcp/mcp-auth-orchestrator.ts +13 -22
  468. package/src/mcp/mcp-oauth-provider.ts +195 -240
  469. package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
  470. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
  471. package/src/messaging/providers/callback-routing.ts +1 -0
  472. package/src/messaging/providers/discord/api.ts +312 -0
  473. package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
  474. package/src/messaging/providers/discord/render.test.ts +204 -0
  475. package/src/messaging/providers/discord/render.ts +195 -0
  476. package/src/messaging/providers/discord/send.test.ts +179 -0
  477. package/src/messaging/providers/discord/send.ts +212 -0
  478. package/src/messaging/providers/discord/transport.ts +81 -0
  479. package/src/messaging/providers/index.ts +3 -1
  480. package/src/messaging/providers/retry-policy.test.ts +247 -0
  481. package/src/messaging/providers/retry-policy.ts +206 -0
  482. package/src/messaging/providers/slack/mention-source.test.ts +285 -0
  483. package/src/messaging/providers/slack/mention-source.ts +287 -0
  484. package/src/messaging/providers/telegram-bot/api.ts +47 -130
  485. package/src/messaging/providers/whatsapp/api.ts +47 -136
  486. package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
  487. package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
  488. package/src/monitoring/db-integrity-sample.ts +6 -1
  489. package/src/monitoring/plugin-auto-update.ts +463 -0
  490. package/src/monitoring/resource-sampler.ts +21 -5
  491. package/src/monitoring/worker.ts +13 -0
  492. package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
  493. package/src/notifications/__tests__/broadcaster.test.ts +37 -1
  494. package/src/notifications/__tests__/copy-composer.test.ts +165 -0
  495. package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
  496. package/src/notifications/__tests__/notification-utils.test.ts +249 -0
  497. package/src/notifications/adapters/platform.ts +31 -2
  498. package/src/notifications/assistant-reply-producer.ts +79 -32
  499. package/src/notifications/broadcaster.ts +30 -10
  500. package/src/notifications/copy-composer.ts +69 -0
  501. package/src/notifications/emit-signal.ts +82 -15
  502. package/src/notifications/events-store.ts +10 -2
  503. package/src/notifications/home-feed-side-effect.ts +6 -3
  504. package/src/notifications/notification-utils.ts +173 -5
  505. package/src/notifications/signal.ts +13 -0
  506. package/src/oauth/AGENTS.md +4 -2
  507. package/src/oauth/byo-connection.ts +6 -1
  508. package/src/oauth/credential-token-resolver.ts +3 -10
  509. package/src/oauth/manual-token-connection.ts +29 -64
  510. package/src/oauth/manual-token-providers.ts +62 -0
  511. package/src/oauth/provider-serializer.ts +13 -0
  512. package/src/oauth/seed-providers.ts +62 -11
  513. package/src/onboarding/onboarding-events-store.ts +91 -1
  514. package/src/permissions/question-prompter.test.ts +4 -0
  515. package/src/permissions/question-prompter.ts +21 -4
  516. package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
  517. package/src/persistence/attachments-store.ts +15 -15
  518. package/src/persistence/conversation-attention-store.ts +30 -10
  519. package/src/persistence/conversation-bootstrap.ts +17 -0
  520. package/src/persistence/conversation-crud.ts +363 -97
  521. package/src/persistence/conversation-group-migration.ts +1 -1
  522. package/src/persistence/conversation-key-store.ts +38 -10
  523. package/src/persistence/conversation-lineage.ts +226 -0
  524. package/src/persistence/conversation-plugin-facade.ts +28 -3
  525. package/src/persistence/conversation-queries.ts +207 -33
  526. package/src/persistence/conversation-types.ts +107 -1
  527. package/src/persistence/jobs-store.ts +32 -8
  528. package/src/persistence/llm-usage-store.ts +22 -0
  529. package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
  530. package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
  531. package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
  532. package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
  533. package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
  534. package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
  535. package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
  536. package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
  537. package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
  538. package/src/persistence/schema/conversations.ts +32 -0
  539. package/src/persistence/schema/infrastructure.ts +3 -0
  540. package/src/persistence/steps.ts +30 -0
  541. package/src/persistence/subagent-store.ts +148 -6
  542. package/src/platform/managed-speech.test.ts +20 -0
  543. package/src/platform/managed-speech.ts +9 -0
  544. package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
  545. package/src/plugin-api/index.ts +28 -5
  546. package/src/plugin-api/webhook-url.ts +151 -0
  547. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
  548. package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
  549. package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
  550. package/src/plugins/collect-source-versions.ts +4 -27
  551. package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
  552. package/src/plugins/defaults/exploration-drift/package.json +1 -1
  553. package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
  554. package/src/plugins/defaults/image-recovery/recover.ts +67 -57
  555. package/src/plugins/defaults/index.ts +1 -1
  556. package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
  557. package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
  558. package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
  559. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
  560. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
  561. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
  562. package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
  563. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
  564. package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
  565. package/src/plugins/defaults/memory/job-handlers.ts +87 -5
  566. package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
  567. package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
  568. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
  569. package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
  570. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
  571. package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
  572. package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
  573. package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
  574. package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
  575. package/src/plugins/defaults/memory/v2/router.ts +8 -0
  576. package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
  577. package/src/plugins/external-plugin-loader.ts +64 -8
  578. package/src/plugins/installed-plugin-dirs.ts +101 -0
  579. package/src/plugins/mcp-servers.ts +343 -0
  580. package/src/plugins/mtime-cache.ts +34 -22
  581. package/src/plugins/types.ts +69 -8
  582. package/src/prompts/persona-resolver.ts +6 -2
  583. package/src/prompts/system-prompt.ts +72 -17
  584. package/src/providers/__tests__/inference.test.ts +40 -0
  585. package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
  586. package/src/providers/__tests__/retry-callsite.test.ts +72 -2
  587. package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
  588. package/src/providers/anthropic/client.ts +112 -95
  589. package/src/providers/connection-resolution.ts +35 -1
  590. package/src/providers/gemini/client.ts +5 -5
  591. package/src/providers/inference/adapter-factory.ts +16 -10
  592. package/src/providers/inference/auth.ts +16 -0
  593. package/src/providers/inference/connection-availability.ts +164 -12
  594. package/src/providers/inference/connections.ts +25 -1
  595. package/src/providers/media-resolve.ts +21 -11
  596. package/src/providers/model-catalog.ts +15 -0
  597. package/src/providers/openai/chat-completions-provider.ts +4 -4
  598. package/src/providers/openai/responses-provider.ts +12 -25
  599. package/src/providers/registry.ts +9 -4
  600. package/src/providers/retry.ts +25 -0
  601. package/src/providers/server-tool-pairing.ts +134 -0
  602. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
  603. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
  604. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
  605. package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
  606. package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
  607. package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
  608. package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
  609. package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
  610. package/src/providers/speech-to-text/deepgram.ts +1 -0
  611. package/src/providers/speech-to-text/provider-catalog.ts +137 -8
  612. package/src/providers/speech-to-text/resolve.ts +105 -12
  613. package/src/providers/speech-to-text/vellum-managed.ts +7 -1
  614. package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
  615. package/src/providers/types.ts +26 -6
  616. package/src/providers/vellum-model-routing.ts +6 -7
  617. package/src/runtime/AGENTS.md +2 -0
  618. package/src/runtime/__tests__/agent-wake.test.ts +16 -16
  619. package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
  620. package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
  621. package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
  622. package/src/runtime/agent-wake.ts +63 -24
  623. package/src/runtime/assistant-event-hub.ts +110 -12
  624. package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
  625. package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
  626. package/src/runtime/auth/same-actor.ts +56 -8
  627. package/src/runtime/auth/scopes.ts +12 -0
  628. package/src/runtime/auth/types.ts +2 -0
  629. package/src/runtime/background-job-runner.ts +5 -27
  630. package/src/runtime/channel-readiness-service.ts +59 -3
  631. package/src/runtime/channel-readiness-types.ts +18 -0
  632. package/src/runtime/desktop-presence.ts +69 -0
  633. package/src/runtime/http-server.ts +12 -0
  634. package/src/runtime/local-actor-identity.ts +29 -14
  635. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
  636. package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
  637. package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
  638. package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
  639. package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
  640. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
  641. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
  642. package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
  643. package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
  644. package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
  645. package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
  646. package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
  647. package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
  648. package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
  649. package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
  650. package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
  651. package/src/runtime/routes/archive-utils.ts +16 -12
  652. package/src/runtime/routes/attachment-routes.ts +13 -6
  653. package/src/runtime/routes/canned-reply-release.ts +54 -0
  654. package/src/runtime/routes/channel-availability-routes.ts +32 -14
  655. package/src/runtime/routes/channel-verification-routes.ts +2 -27
  656. package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
  657. package/src/runtime/routes/client-routes.ts +114 -2
  658. package/src/runtime/routes/conversation-cli-routes.ts +10 -12
  659. package/src/runtime/routes/conversation-list-routes.ts +59 -10
  660. package/src/runtime/routes/conversation-management-routes.ts +47 -18
  661. package/src/runtime/routes/conversation-query-routes.ts +32 -53
  662. package/src/runtime/routes/conversation-routes.ts +360 -293
  663. package/src/runtime/routes/credential-prompt-routes.ts +20 -2
  664. package/src/runtime/routes/debug-routes.ts +4 -1
  665. package/src/runtime/routes/default-provider-routes.ts +7 -11
  666. package/src/runtime/routes/documents-routes.ts +44 -1
  667. package/src/runtime/routes/events-routes.ts +29 -1
  668. package/src/runtime/routes/host-app-control-routes.ts +2 -0
  669. package/src/runtime/routes/host-bash-routes.ts +2 -0
  670. package/src/runtime/routes/host-browser-routes.ts +2 -0
  671. package/src/runtime/routes/host-cu-routes.ts +2 -0
  672. package/src/runtime/routes/host-file-routes.ts +2 -0
  673. package/src/runtime/routes/host-transfer-routes.ts +4 -0
  674. package/src/runtime/routes/identity-routes.ts +6 -6
  675. package/src/runtime/routes/inbound-message-handler.ts +5 -5
  676. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
  677. package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
  678. package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
  679. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
  680. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
  681. package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
  682. package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
  683. package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
  684. package/src/runtime/routes/inference-profiles-routes.ts +152 -73
  685. package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
  686. package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
  687. package/src/runtime/routes/log-export-routes.ts +1 -1
  688. package/src/runtime/routes/mcp-auth-routes.ts +141 -57
  689. package/src/runtime/routes/notification-routes.ts +0 -14
  690. package/src/runtime/routes/oauth-providers.ts +1 -0
  691. package/src/runtime/routes/plugins-routes.ts +127 -4
  692. package/src/runtime/routes/profiler-routes.ts +4 -2
  693. package/src/runtime/routes/question-routes.ts +28 -10
  694. package/src/runtime/routes/retrospective-routes.ts +9 -3
  695. package/src/runtime/routes/schedule-routes.ts +210 -12
  696. package/src/runtime/routes/secret-routes.ts +40 -23
  697. package/src/runtime/routes/settings-routes.ts +41 -13
  698. package/src/runtime/routes/skills-routes.ts +79 -0
  699. package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
  700. package/src/runtime/routes/stt-routes.ts +31 -25
  701. package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
  702. package/src/runtime/routes/wake-conversation-routes.ts +23 -0
  703. package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
  704. package/src/runtime/services/conversation-serializer.ts +24 -9
  705. package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
  706. package/src/runtime/sync/resource-sync-events.ts +79 -1
  707. package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
  708. package/src/runtime/sync/worker-daemon-notify.ts +44 -6
  709. package/src/runtime/verification-outbound-actions.ts +250 -222
  710. package/src/runtime/verification-templates.ts +41 -30
  711. package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
  712. package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
  713. package/src/schedule/inference-profile.ts +64 -3
  714. package/src/schedule/plugin-schedule-declarations.ts +518 -0
  715. package/src/schedule/plugin-schedule-reconciler.ts +605 -0
  716. package/src/schedule/plugin-schedules-gate.ts +23 -0
  717. package/src/schedule/schedule-store.ts +520 -26
  718. package/src/schedule/scheduler.ts +38 -0
  719. package/src/security/credential-backend.ts +51 -4
  720. package/src/security/encrypted-store.ts +64 -42
  721. package/src/security/oauth2.ts +6 -22
  722. package/src/security/secure-keys.ts +55 -7
  723. package/src/skills/catalog-install.ts +20 -7
  724. package/src/skills/inline-command-runner.ts +2 -7
  725. package/src/skills/skill-history.test.ts +269 -0
  726. package/src/skills/skill-history.ts +250 -0
  727. package/src/skills/skillssh-registry.ts +1 -1
  728. package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
  729. package/src/stt/__tests__/language-metadata.test.ts +85 -0
  730. package/src/stt/__tests__/speech-energy.test.ts +79 -0
  731. package/src/stt/__tests__/types.test.ts +94 -0
  732. package/src/stt/daemon-batch-transcriber.ts +18 -6
  733. package/src/stt/language-metadata.ts +65 -0
  734. package/src/stt/speech-energy.ts +115 -12
  735. package/src/stt/stt-stream-session.ts +8 -4
  736. package/src/stt/types.ts +119 -0
  737. package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
  738. package/src/subagent/consult-prompt.ts +3 -1
  739. package/src/subagent/consult-transcript.ts +1 -1
  740. package/src/subagent/manager.ts +304 -44
  741. package/src/subagent/progress-events.ts +35 -0
  742. package/src/subagent/role-resolution.ts +88 -0
  743. package/src/subagent/types.ts +303 -67
  744. package/src/subagent/validate-allowlists.ts +3 -4
  745. package/src/telegram/__tests__/webhook-health.test.ts +73 -1
  746. package/src/telegram/webhook-health.ts +101 -3
  747. package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
  748. package/src/telemetry/activation-funnel.ts +6 -1
  749. package/src/telemetry/live-voice-funnel.ts +75 -0
  750. package/src/telemetry/telemetry-event-sources.ts +6 -0
  751. package/src/telemetry/telemetry-wire-source.json +1 -1
  752. package/src/telemetry/telemetry-wire.generated.ts +2 -0
  753. package/src/telemetry/types.ts +23 -4
  754. package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
  755. package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
  756. package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
  757. package/src/tools/ask-question/ask-question-tool.ts +56 -2
  758. package/src/tools/browser/browser-execution.ts +1 -1
  759. package/src/tools/calls/call-start.ts +8 -5
  760. package/src/tools/conversation-groups/move_to_group.ts +10 -12
  761. package/src/tools/document/document-tool.ts +11 -4
  762. package/src/tools/execution-timeout.ts +9 -2
  763. package/src/tools/executor.ts +7 -6
  764. package/src/tools/filesystem/edit.ts +1 -1
  765. package/src/tools/filesystem/list.ts +1 -1
  766. package/src/tools/filesystem/read.ts +7 -13
  767. package/src/tools/filesystem/search.ts +23 -21
  768. package/src/tools/filesystem/write.ts +4 -1
  769. package/src/tools/host-filesystem/edit.ts +7 -2
  770. package/src/tools/host-filesystem/read.ts +21 -8
  771. package/src/tools/host-filesystem/write.ts +10 -2
  772. package/src/tools/host-terminal/host-shell.ts +2 -2
  773. package/src/tools/mcp/mcp-tool-factory.ts +25 -1
  774. package/src/tools/permission-checker.ts +8 -3
  775. package/src/tools/schedule/delete.ts +13 -1
  776. package/src/tools/schedule/list.ts +14 -3
  777. package/src/tools/schedule/update.ts +56 -33
  778. package/src/tools/shared/filesystem/audio-read.ts +6 -4
  779. package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
  780. package/src/tools/shared/filesystem/image-read.ts +12 -12
  781. package/src/tools/shared/filesystem/path-policy.ts +70 -5
  782. package/src/tools/shared/filesystem/size-guard.ts +6 -6
  783. package/src/tools/shared/input-misuse.ts +72 -0
  784. package/src/tools/skills/scaffold-managed.ts +82 -0
  785. package/src/tools/skills/skill-tool-factory.ts +14 -1
  786. package/src/tools/subagent/consult-deadline.ts +12 -7
  787. package/src/tools/subagent/message.ts +3 -1
  788. package/src/tools/subagent/read.ts +66 -4
  789. package/src/tools/subagent/spawn.ts +500 -78
  790. package/src/tools/tool-approval-handler.ts +45 -4
  791. package/src/tools/types.ts +22 -3
  792. package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
  793. package/src/tools/ui-surface/visual-validation.ts +2 -0
  794. package/src/tts/__tests__/provider-adapters.test.ts +147 -0
  795. package/src/tts/__tests__/speakable-segments.test.ts +470 -0
  796. package/src/tts/language-voices.ts +23 -0
  797. package/src/tts/providers/deepgram-provider.ts +3 -1
  798. package/src/tts/providers/elevenlabs-provider.ts +73 -1
  799. package/src/tts/providers/xai-provider.ts +28 -2
  800. package/src/tts/speakable-segments.ts +293 -23
  801. package/src/tts/synthesis-stream.ts +7 -0
  802. package/src/tts/types.ts +7 -0
  803. package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
  804. package/src/usage/subagent-attribution.ts +108 -0
  805. package/src/util/__tests__/language-subtag.test.ts +54 -0
  806. package/src/util/__tests__/sqlite-retry.test.ts +112 -0
  807. package/src/util/abort-reasons.ts +16 -0
  808. package/src/util/ansi.ts +44 -0
  809. package/src/util/disk-usage.ts +17 -13
  810. package/src/util/image-conversion.ts +20 -13
  811. package/src/util/language-subtag.ts +43 -0
  812. package/src/util/schedule-source-key.ts +23 -0
  813. package/src/util/unicode.ts +1 -1
  814. package/src/util/worker-process.ts +7 -1
  815. package/src/workspace/git-service.ts +33 -9
  816. package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
  817. package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
  818. package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
  819. package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
  820. package/src/workspace/migrations/registry.ts +6 -0
  821. package/src/workspace/top-level-renderer.ts +15 -2
  822. package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
  823. package/src/daemon/install-symlink.ts +0 -207
  824. package/src/live-voice/__tests__/front-decision.test.ts +0 -525
  825. package/src/live-voice/front-decision.ts +0 -429
  826. package/src/notifications/deferred-emit.ts +0 -147
@@ -17,7 +17,6 @@ import type {
17
17
  import {
18
18
  getConversationTurnTeardown,
19
19
  resolveProcessingWaitMs,
20
- VOICE_NO_SETUP_FLOWS_RULE,
21
20
  waitForPriorTurnTeardown,
22
21
  } from "../calls/voice-session-bridge.js";
23
22
  import {
@@ -25,38 +24,58 @@ import {
25
24
  classifyFrontDoorLeading,
26
25
  ESCALATE_VERDICT_TOKEN,
27
26
  ESCALATION_CONTINUATION_CONTENT,
28
- FALLBACK_ESCALATION_BRIDGE,
27
+ FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
28
+ fallbackEscalationBridgeFor,
29
29
  isEscalationBridgeComplete,
30
30
  MIN_SPOKEN_BRIDGE_CHARS,
31
31
  type VoiceRoutingLeg,
32
32
  } from "../calls/voice-triage-escalate.js";
33
33
  import { getConfig } from "../config/loader.js";
34
34
  import {
35
+ type LiveVoiceFluxConfig,
36
+ LiveVoiceFluxConfigSchema,
35
37
  type LiveVoiceFrontModelConfig,
36
38
  LiveVoiceFrontModelConfigSchema,
37
39
  } from "../config/schemas/live-voice.js";
38
40
  import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
39
41
  import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
40
42
  import type { TrustContext } from "../daemon/trust-context-types.js";
43
+ import {
44
+ recordLiveVoiceSessionEnded,
45
+ recordLiveVoiceSessionStarted,
46
+ } from "../onboarding/onboarding-events-store.js";
41
47
  import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
42
48
  import { ensureConversationExists } from "../persistence/conversation-crud.js";
43
49
  import {
44
50
  listProviderIds,
51
+ pinnedListeningLanguage,
45
52
  supportsBoundary,
53
+ supportsProviderTurnDetection,
46
54
  } from "../providers/speech-to-text/provider-catalog.js";
47
55
  import type { ResolveStreamingTranscriberOptions } from "../providers/speech-to-text/resolve.js";
48
56
  import { broadcastMessage } from "../runtime/assistant-event-hub.js";
49
57
  import { publishConversationListAndMetadataChanged } from "../runtime/sync/resource-sync-events.js";
50
- import { detectPcm16SpeechActivity } from "../stt/speech-energy.js";
58
+ import {
59
+ dominantLanguageTag,
60
+ voteDominantLanguage,
61
+ } from "../stt/language-metadata.js";
62
+ import {
63
+ DEFAULT_SPEECH_ENERGY_THRESHOLD,
64
+ pcm16MaxNormalizedCorrelation,
65
+ pcm16MeanAmplitude,
66
+ } from "../stt/speech-energy.js";
51
67
  import type {
52
68
  StreamingTranscriber,
69
+ SttProviderId,
53
70
  SttStreamServerErrorEvent,
54
71
  SttStreamServerEvent,
55
72
  } from "../stt/types.js";
56
73
  import { getSubagentManager } from "../subagent/index.js";
74
+ import { liveVoiceEndScreen } from "../telemetry/live-voice-funnel.js";
57
75
  import { getToolOwner } from "../tools/registry.js";
58
76
  import { extractSpeakableSegments } from "../tts/speakable-segments.js";
59
77
  import { createAbortReason } from "../util/abort-reasons.js";
78
+ import { hasLocalizedEntry } from "../util/language-subtag.js";
60
79
  import { getLogger } from "../util/logger.js";
61
80
  import {
62
81
  activityLabelForTool,
@@ -64,11 +83,6 @@ import {
64
83
  dismissesUiSurface,
65
84
  revealsUiSurface,
66
85
  } from "./activity-label.js";
67
- import {
68
- createVoiceFrontDecider,
69
- type VoiceFrontDecider,
70
- type VoiceProgressTextInput,
71
- } from "./front-decision.js";
72
86
  import { LiveActivityReporter } from "./live-activity-reporter.js";
73
87
  import type {
74
88
  LiveVoiceAudioArchiveResult,
@@ -80,10 +94,11 @@ import {
80
94
  type LiveVoiceMetricsClock,
81
95
  LiveVoiceMetricsCollector,
82
96
  type LiveVoiceMetricsEvent,
83
- type LiveVoiceSpokenAckKind,
84
97
  type LiveVoiceTurnSeedMarks,
85
98
  type VoiceEndpointAction,
99
+ type VoiceEndpointSource,
86
100
  } from "./live-voice-metrics.js";
101
+ import { persistLiveVoicePhoto } from "./live-voice-photo.js";
87
102
  import {
88
103
  type LiveVoiceSession as LiveVoiceSessionContract,
89
104
  type LiveVoiceSessionCloseReason,
@@ -95,8 +110,19 @@ import type {
95
110
  LiveVoiceTtsOptions,
96
111
  LiveVoiceTtsResult,
97
112
  } from "./live-voice-tts.js";
98
- import { pickProgressPhrase } from "./progress-phrases.js";
99
113
  import {
114
+ createVoiceProgressNarrator,
115
+ type VoiceProgressNarrator,
116
+ type VoiceProgressTextInput,
117
+ } from "./progress-narration.js";
118
+ import {
119
+ APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
120
+ approvalPendingPhraseFor,
121
+ pickProgressPhrase,
122
+ PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
123
+ } from "./progress-phrases.js";
124
+ import {
125
+ type LiveVoiceClientAttachImageFrame,
100
126
  type LiveVoiceClientFrame,
101
127
  type LiveVoiceClientUpdateConfigFrame,
102
128
  LiveVoiceProtocolErrorCode,
@@ -112,6 +138,13 @@ type LiveVoiceSessionState =
112
138
  | "failed"
113
139
  | "closed";
114
140
 
141
+ type VadEnergyClassification = "speech" | "silence" | "echo";
142
+
143
+ interface VadClassifiedChunk {
144
+ readonly chunk: Buffer;
145
+ readonly classification: VadEnergyClassification;
146
+ }
147
+
115
148
  // Cap on audio buffered while a server-VAD utterance waits for its
116
149
  // transcriber (PCM16 mono seconds; oldest chunks are dropped past the cap).
117
150
  const SERVER_VAD_PENDING_AUDIO_MAX_SECONDS = 10;
@@ -130,7 +163,34 @@ const FINALIZE_GRACE_MS = 1_000;
130
163
  // cancellation cannot kill a reply mid-sentence. Mirrors the
131
164
  // liveVoice.vad.bargeInMinSpeechMs schema default; 0 disables the guard for
132
165
  // instant barge-in.
166
+ //
167
+ // Local energy detection owns barge-in in every mode, including when a
168
+ // turn-detecting provider owns the turn boundary: an interrupt during TTS has
169
+ // to feel instant, and no provider roundtrip beats a local gate on the audio
170
+ // already in hand. The echo-adaptive part of this guard also has to stay
171
+ // upstream of the provider, which hears only the microphone: it cannot tell
172
+ // our own TTS bleeding through imperfect echo cancellation from the caller,
173
+ // and would report a user turn nobody took.
133
174
  const DEFAULT_BARGE_IN_MIN_SPEECH_MS = 250;
175
+ // The playback echo gate learns microphone energy while assistant audio is
176
+ // expected at the speaker. Input must rise above the learned level by this
177
+ // margin to count as user speech.
178
+ const DEFAULT_ECHO_BARGE_IN_MARGIN = 1.5;
179
+ const DEFAULT_ECHO_EMA_HALF_LIFE_MS = 400;
180
+ // Before learning a microphone power baseline, compare a short input window
181
+ // with the PCM sent to the speaker. This keeps a user's first interruption
182
+ // from becoming its own echo threshold.
183
+ const ECHO_CORRELATION_PROBE_MS = 100;
184
+ const ECHO_CORRELATION_MIN_MS = 50;
185
+ const ECHO_CORRELATION_THRESHOLD = 0.65;
186
+ const ECHO_REFERENCE_MAX_MS = 10_000;
187
+ // Echo should reach the microphone near playback onset. If no signal arrives
188
+ // within this much input audio, the gate returns to the fixed base threshold.
189
+ // The same interval expires a learned reference after a real silent gap.
190
+ const ECHO_ONSET_ELIGIBILITY_MS = 300;
191
+ // Client buffering makes audible playback trail the server's send-time
192
+ // estimate. Keep the echo window open briefly past that estimate.
193
+ const DEFAULT_ECHO_DRAIN_SLACK_MS = 300;
134
194
  // Mirrors MediaTurnDetector's DEFAULT_SILENCE_THRESHOLD_MS: the session
135
195
  // tracks the effective trailing-silence threshold (the detector keeps its own
136
196
  // copy private) so the endpoint decider can report the pause length.
@@ -153,6 +213,12 @@ const BARGE_IN_GAP_TOLERANCE_MS = 200;
153
213
  // (1 / (1 + ratio) ≈ 20%): once the run is mostly silence it resets, so genuine
154
214
  // choppy speech still lands but periodic noise cannot accumulate into one.
155
215
  const BARGE_IN_MAX_TOLERATED_SILENCE_RATIO = 4;
216
+ // Slack added to the configured end-of-turn timeout before the session stops
217
+ // waiting for an event that is not coming and falls the utterance back onto
218
+ // the silence-boundary path. A turn-detecting provider force-ends its own turn
219
+ // at that timeout, so anything past it plus a frame's network and parse hop
220
+ // means the stream is gone, not slow.
221
+ const PROVIDER_TURN_END_FALLBACK_MARGIN_MS = 1_000;
156
222
  // At most this many TTS segment jobs are open (provider stream started,
157
223
  // frames not yet fully emitted) per turn: the emitting job plus one
158
224
  // prefetching job. The prefetch buffers its chunks in memory until promoted;
@@ -261,25 +327,41 @@ export interface LiveVoiceSessionOptions {
261
327
  * defaults to `DEFAULT_BARGE_IN_MIN_SPEECH_MS`.
262
328
  */
263
329
  bargeInMinSpeechMs?: number;
330
+ /**
331
+ * Multiplier over the learned playback echo level that input must exceed
332
+ * to count as speech while assistant audio is playing. Values at or below
333
+ * 1 disable adaptation for internal fixed-gate callers; workspace config
334
+ * requires a value greater than 1.
335
+ */
336
+ echoBargeInMargin?: number;
337
+ /** Half-life in milliseconds for the learned playback echo level. */
338
+ echoEmaHalfLifeMs?: number;
339
+ /** Extra time after the playback estimate during which echo is expected. */
340
+ echoDrainSlackMs?: number;
264
341
  /**
265
342
  * Overrides the bounded wait for the shared transcriber's finalize
266
343
  * flush in persistent mode (test hook). Defaults to `FINALIZE_GRACE_MS`.
267
344
  */
268
345
  finalizeGraceMs?: number;
269
346
  /**
270
- * Front-model presence tuning (semantic endpointing + spoken acks). The
347
+ * Voice front-door endpointing and progress narration tuning. The
271
348
  * production factory seeds this from `liveVoice.frontModel` config when
272
349
  * unset; absent fields fall back to the schema defaults (the constructor
273
350
  * schema-parses the partial into a complete config).
274
351
  */
275
352
  frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
276
353
  /**
277
- * Front-model phrasing service: spoken acks and progress narration. The
278
- * production factory constructs it from `liveVoice.frontModel` config when
279
- * unset; `null` disables front-model LLM calls, so no ack is ever spoken
280
- * and narration falls back to its static phrases.
354
+ * Deepgram Flux turn-detection tuning. The production factory seeds this
355
+ * from `liveVoice.flux` config when unset; absent fields fall back to the
356
+ * schema defaults, which leave `turnEnd.enabled` false.
357
+ */
358
+ fluxConfig?: Partial<LiveVoiceFluxConfig>;
359
+ /**
360
+ * Progress narration service. The production factory constructs it from
361
+ * `liveVoice.frontModel.progress` config when unset. `null` disables
362
+ * generated narration, so progress updates use their static fallback.
281
363
  */
282
- frontDecider?: VoiceFrontDecider | null;
364
+ progressNarrator?: VoiceProgressNarrator | null;
283
365
  /**
284
366
  * Spawns the background continuation for a barged-in turn. The factory wires
285
367
  * the real SubagentManager-backed implementation; tests inject a stub.
@@ -357,11 +439,44 @@ interface UtteranceCycle {
357
439
  // Consecutive semantic-endpointing "hold" extensions this utterance has
358
440
  // consumed, bounded by `endpointMaxExtensions`.
359
441
  endpointExtensionCount: number;
442
+ // The provider's end-of-turn never arrived for this cycle, so it fell back
443
+ // to the silence-boundary path and stays there: a late event must not
444
+ // commit a boundary the fallback already owns.
445
+ providerTurnEndTimedOut: boolean;
446
+ // `vadSpeechGeneration` as of the local silence boundary that handed this
447
+ // cycle to the provider, or null while no boundary has fired yet (a turn
448
+ // model routinely beats the trailing-silence countdown, and that fast commit
449
+ // is the point).
450
+ // The staleness signal of last resort: it stands in for the provider's own
451
+ // turn numbering when the provider sends none (see isStaleProviderTurnEnd).
452
+ turnBoundaryGeneration: number | null;
453
+ // Index of the newest provider turn opened in this cycle, from the turn
454
+ // model's own numbering, or null when the provider does not number its
455
+ // turns. An end-of-turn closing an older index describes a turn the
456
+ // provider has already superseded (see isStaleProviderTurnEnd).
457
+ openProviderTurnIndex: number | null;
360
458
  // The transcript the most recent hold verdict judged (unified front-door).
361
459
  // A final segment arriving during the extension window that extends this
362
460
  // text replays the boundary immediately — the hold was judged on stale
363
461
  // text, so waiting out the extension only adds silence.
364
462
  heldSpeculativeContent: string | null;
463
+ // Count per detected-language base subtag (see voteDominantLanguage)
464
+ // across this cycle's final transcript events. Resolves the turn's spoken
465
+ // language (see turnLanguageFor); empty when the provider tags nothing.
466
+ languageTally: Map<string, number>;
467
+ // Detected languages of the most recent partial that carried any, already
468
+ // normalized, dominance order. Speculative turns dispatch from partials
469
+ // before the first tagged final lands, so turnLanguageFor falls back to
470
+ // this when the final tally is still empty. Never cleared: the tally
471
+ // outranks it once finals arrive, and a revising partial without tags
472
+ // must not wipe an earlier partial's detection.
473
+ latestPartialLanguages: readonly string[] | null;
474
+ // The provider that actually transcribed this cycle, recorded when its
475
+ // transcriber is assigned and kept after teardown nulls `transcriber`.
476
+ // The resolver can silently dial managed vellum when a BYOK provider has
477
+ // no credential, so the language-pin gate in turnLanguageFor must follow
478
+ // this, not the configured provider.
479
+ dialedSttProvider: SttProviderId | null;
365
480
  turnId: string | null;
366
481
  userMessageId: string | null;
367
482
  userAudioChunks: Buffer[];
@@ -392,6 +507,11 @@ type UtteranceStartResult =
392
507
  // client in job-list order.
393
508
  interface TtsSegmentJob {
394
509
  readonly text: string;
510
+ // Per-segment language-hint override, preferred over the turn's language.
511
+ // Set on fixed phrases whose localized table lacks the turn's language:
512
+ // the English fallback text carries "en" so an enforcing provider never
513
+ // renders English words as ar/ko/ta. Undefined means the turn language.
514
+ readonly language: string | undefined;
395
515
  // The provider stream was started (the job holds an open-job slot).
396
516
  started: boolean;
397
517
  // Emission finished; the slot is free for the next queued segment.
@@ -476,6 +596,13 @@ interface ActiveAssistantTurn {
476
596
  token: symbol;
477
597
  turnId: string;
478
598
  utterance: UtteranceCycle;
599
+ // The caller's spoken language for this turn as a lowercase base subtag
600
+ // (see turnLanguageFor): the dominant STT-detected language, else a
601
+ // monolingual services.stt.language pin. Undefined when unknown, which
602
+ // disables every language-aware path (prompt note, TTS hint, localized
603
+ // fallbacks). Re-resolved when a speculative turn commits, since finals
604
+ // can land between dispatch and verdict.
605
+ language: string | undefined;
479
606
  abortController: AbortController;
480
607
  handle: VoiceTurnHandle | null;
481
608
  // When the turn launched, for narration's turnElapsedMs.
@@ -578,27 +705,11 @@ interface ActiveAssistantTurn {
578
705
  // context. A bump since then means a stop/interrupt/supersede deliberately
579
706
  // dropped the continuation, so a rollback must not resurrect it.
580
707
  pendingContextStopGeneration: number;
581
- // The agent run started a definitive tool use this turn — tool use implies
582
- // a guaranteed-slow turn, so acknowledgment logic can key off this.
583
- toolUseStarted: boolean;
584
- // The model's first streamed delta reached this session — the spoken-ack
585
- // timer is moot once real output is flowing.
586
- firstDeltaSeen: boolean;
587
708
  // Counts assistant text deltas seen this turn. A narration generation
588
709
  // captures it at launch and discards its result if it moved: text the model
589
710
  // produced mid-generation makes the narration stale, and proves the model
590
711
  // is alive — which is exactly what narration exists to paper over.
591
712
  deltaEpoch: number;
592
- // A spoken ack was enqueued this turn. Every ack trigger shares this
593
- // one-per-turn budget so a slow turn never stacks fillers.
594
- ackSpoken: boolean;
595
- // An ack generation is awaiting the decider. While true, the ack has not
596
- // yet stamped `lastFloorHolderAtMs`, so narration must treat it as a
597
- // floor-holder-in-waiting and stand down —
598
- // otherwise a progress phrase could land back-to-back with the ack.
599
- ackGenerationPending: boolean;
600
- // Pending slow-first-delta ack timer; null once cleared or fired.
601
- ackTimer: ReturnType<typeof setTimeout> | null;
602
713
  // Triage-and-escalate (Voice Mode): the front-door leg gave the escalate
603
714
  // verdict and the strong "escalated" leg has taken over this same turn.
604
715
  // Guards the
@@ -674,14 +785,8 @@ function createControlMarkerHoldback(
674
785
  // barge-in, the interruption merge note is appended to it (see
675
786
  // buildInterruptionMergeNote) so the model reconciles the interrupted request
676
787
  // with the new utterance.
677
- // Spoken once when a turn starts waiting on the user's decision. Fixed rather
678
- // than generated: this is a statement about the system's state, not about the
679
- // work, and it has to be true every time. Kept in the shape of the progress
680
- // phrases it displaces (short, neutral, no claim about tools).
681
- const APPROVAL_PENDING_PHRASE = "I need your okay for that one. Take a look.";
682
-
683
788
  const LIVE_VOICE_CONTROL_PROMPT_BASE =
684
- "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. ";
789
+ "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. ";
685
790
 
686
791
  // Appended for the legs that can actually put something on screen: the main
687
792
  // leg and the escalated leg. The front-door (fast) leg never receives it, for
@@ -699,6 +804,20 @@ const LIVE_VOICE_CONTROL_PROMPT_BASE =
699
804
  const LIVE_VOICE_SCREEN_REVEAL_TEACHING =
700
805
  "The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. Never emit bracketed markers of any kind. ";
701
806
 
807
+ // The setup-flow case, spelled out because it is the one the model gets wrong
808
+ // on its own: connecting an account reads as something a call cannot do, so it
809
+ // declines and offers to do it later in text, which is the exact thing the base
810
+ // prompt above forbids.
811
+ //
812
+ // It can do it. The connect card is a ui surface like any other, so showing it
813
+ // minimizes the room on the same latch (`revealsUiSurface`), and the user is at
814
+ // the screen the browser window opens on. Appended alongside the screen-reveal
815
+ // teaching, to the same legs, for the same reason: the front-door leg is
816
+ // toolless, so a setup flow is not its to run, and its capability digest
817
+ // already tells it to escalate anything needing a tool rather than refuse.
818
+ const LIVE_VOICE_SETUP_FLOW_TEACHING =
819
+ "This includes connecting accounts. If a task needs an account connected or a sign-in completed, put the connection up on screen and let the user do it now. Do not decline it, and do not defer it to text chat. Say what you are connecting and that it is in front of them. ";
820
+
702
821
  // System-level guidance appended to a barge-in turn's control prompt so the
703
822
  // model treats the new utterance as a continuation of the request it was cut
704
823
  // off answering, rather than a fresh follow-up. Reaches the model only; it is
@@ -755,21 +874,24 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
755
874
  return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
756
875
  }
757
876
 
758
- // Assemble a leg's model-facing control prompt: the base live-voice rules,
759
- // the screen-reveal teaching (withheld from the front-door leg, see
760
- // LIVE_VOICE_SCREEN_REVEAL_TEACHING), the shared no-setup-flows rule, plus
761
- // any pending barge-in merge context, completed-continuation context, and/or
762
- // the announcement instruction. A turn can carry several (a barge-in follow-up
763
- // that also has a continuation result waiting); the notes are model-only and
764
- // never render as user bubbles.
877
+ // Assemble a leg's model-facing control prompt: the base live-voice rules, the
878
+ // screen-reveal and setup-flow teaching (both withheld from the front-door leg,
879
+ // see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
880
+ // context, completed-continuation context, and/or the announcement instruction.
881
+ // A turn can carry several (a barge-in follow-up that also has a continuation
882
+ // result waiting); the notes are model-only and never render as user bubbles.
765
883
  function buildVoiceControlPrompt(
766
884
  turn: ActiveAssistantTurn,
767
885
  leg: { frontDoor?: boolean },
768
886
  ): string {
769
887
  let prompt =
770
888
  LIVE_VOICE_CONTROL_PROMPT_BASE +
771
- (leg.frontDoor === true ? "" : LIVE_VOICE_SCREEN_REVEAL_TEACHING) +
772
- VOICE_NO_SETUP_FLOWS_RULE;
889
+ (leg.frontDoor === true
890
+ ? ""
891
+ : LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING);
892
+ if (turn.language !== undefined) {
893
+ prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
894
+ }
773
895
  if (turn.interruptedRequest) {
774
896
  prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
775
897
  }
@@ -817,7 +939,13 @@ function createUtteranceCycle(): UtteranceCycle {
817
939
  finalTranscriptSegments: [],
818
940
  latestPartialText: null,
819
941
  endpointExtensionCount: 0,
942
+ providerTurnEndTimedOut: false,
943
+ turnBoundaryGeneration: null,
944
+ openProviderTurnIndex: null,
820
945
  heldSpeculativeContent: null,
946
+ languageTally: new Map(),
947
+ latestPartialLanguages: null,
948
+ dialedSttProvider: null,
821
949
  turnId: null,
822
950
  userMessageId: null,
823
951
  userAudioChunks: [],
@@ -972,11 +1100,41 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
972
1100
  private outboundFrames: Promise<void> = Promise.resolve();
973
1101
  private activeAssistantTurn: ActiveAssistantTurn | null = null;
974
1102
  private sessionEndMetricsEmitted = false;
1103
+ /**
1104
+ * Protocol error code of the failure that killed the session, latched by
1105
+ * {@link sendFrame} when an error frame goes out on an already-`failed`
1106
+ * session. Both fatal paths (`failStartup` before `ready`, a failed
1107
+ * utterance arm after it) set `state = "failed"` before sending, and no
1108
+ * other error frame does, so this catches exactly the session-ending
1109
+ * failures and ignores the recoverable mid-session ones. `null` on a
1110
+ * session that never failed.
1111
+ */
1112
+ private failureCode: LiveVoiceProtocolErrorCode | null = null;
975
1113
  // Non-null iff the start frame requested turnDetection "server_vad".
976
1114
  private readonly turnDetector: MediaTurnDetector | null;
977
- // Energy gate for server-VAD speech classification; undefined defers to
978
- // DEFAULT_SPEECH_ENERGY_THRESHOLD.
1115
+ // Base energy gate for server-VAD speech classification. During estimated
1116
+ // playback, classifyVadEnergy raises this above the learned echo level.
979
1117
  private readonly speechEnergyThreshold: number | undefined;
1118
+ private readonly echoBargeInMargin: number;
1119
+ private readonly echoEmaHalfLifeMs: number;
1120
+ private readonly echoDrainSlackMs: number;
1121
+ // Learned microphone energy attributable to assistant playback.
1122
+ private echoEnergyEma = 0;
1123
+ // Signal-bearing microphone audio held until it can be compared with the
1124
+ // assistant PCM. A nonmatch is replayed through VAD in original order.
1125
+ private echoProbeChunks: Buffer[] = [];
1126
+ // Recent raw assistant PCM from the current playback burst.
1127
+ private echoReferenceAudio = Buffer.alloc(0);
1128
+ private echoWindowTotalAudioMs = 0;
1129
+ // Consecutive sub-base input expires a reference that can no longer
1130
+ // describe audible playback.
1131
+ private echoSubBaseRunMs = 0;
1132
+ // Once onset eligibility lapses, later user speech cannot seed a new echo
1133
+ // reference in the same playback window.
1134
+ private echoOnsetLapsed = false;
1135
+ // A live speech run that predates playback belongs to the user and bypasses
1136
+ // echo warm-up until that run genuinely resets.
1137
+ private echoWindowGuardCarryover = false;
980
1138
  // Mutable so a mid-session `update_config` frame can retune "interrupt
981
1139
  // sensitivity" live (see applyConfigUpdate).
982
1140
  private bargeInMinSpeechMs: number;
@@ -1080,13 +1238,38 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1080
1238
  private readonly finalizeGraceMs: number;
1081
1239
  // Rotates through the progress fallback phrases across the session's turns.
1082
1240
  private progressPhraseCounter = 0;
1083
- // Front-model service: phrases spoken acks and progress narration; null
1084
- // disables both (the turn then holds the floor with silence).
1085
- private readonly frontDecider: VoiceFrontDecider | null;
1241
+ // Progress-only phrasing service. It never makes routing decisions and
1242
+ // never emits an answer.
1243
+ private readonly progressNarrator: VoiceProgressNarrator | null;
1086
1244
  // Complete front-model tunables: the constructor schema-parses the partial
1087
1245
  // option once, so every field carries its `liveVoice.frontModel` schema
1088
1246
  // default when unset.
1089
1247
  private readonly frontModelConfig: LiveVoiceFrontModelConfig;
1248
+ // Complete Flux tunables (the constructor schema-parses the partial option
1249
+ // once, so every field carries its `liveVoice.flux` schema default).
1250
+ private readonly fluxConfig: LiveVoiceFluxConfig;
1251
+ /**
1252
+ * Per-session latch: the provider's committed end-of-turn owns the turn
1253
+ * boundary instead of the silence boundary's front-door hold verdict. Set
1254
+ * when the config flag is on AND the resolved streaming provider declares
1255
+ * `turnDetection: "provider"` in the STT catalog AND the session runs server
1256
+ * VAD. Push-to-talk is excluded deliberately: there the client's release IS
1257
+ * the boundary, and answering while the caller still holds the button is not
1258
+ * turn detection, it is a bug. False leaves every other code path exactly as
1259
+ * it is, with no provider turn detection in the picture.
1260
+ */
1261
+ private providerTurnEndActive = false;
1262
+ // Wall-clock of the newest above-gate audio chunk, tracked in every
1263
+ // server-VAD session. It is the local VAD's speech-stop mark: the one anchor
1264
+ // the reported end-of-turn latency is measured from whichever decider
1265
+ // commits the turn, and the anchor for the provider fallback deadline. Null
1266
+ // in push-to-talk, where no local VAD runs.
1267
+ private localSpeechStopAtMs: number | null = null;
1268
+ // Fail-open deadline for a provider end-of-turn that never arrives. On
1269
+ // expiry the utterance falls back to the silence-boundary path, so a dead
1270
+ // provider stream degrades to the silence-path behavior instead of a hung
1271
+ // turn.
1272
+ private providerTurnEndTimer: ReturnType<typeof setTimeout> | null = null;
1090
1273
  // Effective trailing-silence threshold, mirroring the detector's private
1091
1274
  // copy (constructor seed + update_config), reported to the endpoint decider.
1092
1275
  private silenceThresholdMs: number;
@@ -1149,11 +1332,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1149
1332
  context.startFrame.bargeInMinSpeechMs ??
1150
1333
  options.bargeInMinSpeechMs ??
1151
1334
  DEFAULT_BARGE_IN_MIN_SPEECH_MS;
1335
+ this.echoBargeInMargin =
1336
+ options.echoBargeInMargin ?? DEFAULT_ECHO_BARGE_IN_MARGIN;
1337
+ this.echoEmaHalfLifeMs =
1338
+ options.echoEmaHalfLifeMs ?? DEFAULT_ECHO_EMA_HALF_LIFE_MS;
1339
+ this.echoDrainSlackMs =
1340
+ options.echoDrainSlackMs ?? DEFAULT_ECHO_DRAIN_SLACK_MS;
1152
1341
  this.finalizeGraceMs = options.finalizeGraceMs ?? FINALIZE_GRACE_MS;
1153
- this.frontDecider = options.frontDecider ?? null;
1342
+ this.progressNarrator = options.progressNarrator ?? null;
1154
1343
  this.frontModelConfig = LiveVoiceFrontModelConfigSchema.parse(
1155
1344
  options.frontModelConfig ?? {},
1156
1345
  );
1346
+ this.fluxConfig = LiveVoiceFluxConfigSchema.parse(options.fluxConfig ?? {});
1157
1347
  const turnDetectorConfig: TurnDetectorConfig = {
1158
1348
  ...(options.turnDetectorConfig ?? {}),
1159
1349
  ...(context.startFrame.silenceThresholdMs !== undefined
@@ -1184,6 +1374,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1184
1374
  return;
1185
1375
  }
1186
1376
 
1377
+ // Before the preflight, not after: a session rejected for missing
1378
+ // credentials is precisely the one the failure rate needs to count, and
1379
+ // recording the start only once `ready` goes out would hide every such
1380
+ // session from both the numerator and the denominator.
1381
+ recordLiveVoiceSessionStarted(this.context.sessionId);
1382
+
1187
1383
  if (this.resolveCredentialReadiness) {
1188
1384
  const readiness = await this.resolveCredentialReadiness();
1189
1385
  if (readiness.status === "not-ready") {
@@ -1237,9 +1433,44 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1237
1433
  case "update_config":
1238
1434
  this.applyConfigUpdate(frame);
1239
1435
  return;
1436
+ case "attach_image":
1437
+ this.persistPhoto(frame);
1438
+ return;
1240
1439
  }
1241
1440
  }
1242
1441
 
1442
+ /**
1443
+ * Persist a photo taken mid-call into the conversation, running no turn.
1444
+ *
1445
+ * Fire-and-forget on purpose: the persist waits out any in-flight turn, and
1446
+ * the socket must keep pumping audio meanwhile. The client already showed a
1447
+ * thumbnail from the local frame, so nothing on screen is waiting on this.
1448
+ *
1449
+ * The photo becomes its own user message rather than riding the next spoken
1450
+ * turn, which is what makes shutter-then-speak and speak-then-shutter
1451
+ * behave the same: either way the model's history has the image by the time
1452
+ * it answers. See `live-voice-photo.ts` for the full reasoning.
1453
+ */
1454
+ private persistPhoto(frame: LiveVoiceClientAttachImageFrame): void {
1455
+ void persistLiveVoicePhoto(this.conversationId, frame.attachmentId).then(
1456
+ (result) => {
1457
+ if (!result.ok && !this.isClosed) {
1458
+ void this.sendFrame({
1459
+ type: "error",
1460
+ code: LiveVoiceProtocolErrorCode.InvalidFrame,
1461
+ message: "Could not attach that photo to the conversation.",
1462
+ // Names the photo as the casualty so the client can retract the
1463
+ // thumbnail it already showed, rather than filing this with the
1464
+ // transient transcriber and TTS blips that share `recoverable`.
1465
+ frameType: "attach_image",
1466
+ // The session is fine; only this photo failed.
1467
+ recoverable: true,
1468
+ });
1469
+ }
1470
+ },
1471
+ );
1472
+ }
1473
+
1243
1474
  /**
1244
1475
  * Apply a mid-session `update_config` frame: retune the live turn detector's
1245
1476
  * pause ("pause before reply") and/or the barge-in guard ("interrupt
@@ -1261,11 +1492,24 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1261
1492
  await this.handleAudio(Buffer.from(chunk));
1262
1493
  }
1263
1494
 
1264
- async close(_reason: LiveVoiceSessionCloseReason): Promise<void> {
1495
+ async close(reason: LiveVoiceSessionCloseReason): Promise<void> {
1265
1496
  if (this.isClosed) {
1266
1497
  return;
1267
1498
  }
1268
1499
 
1500
+ // Recorded first, and independently of `shouldEmitSessionEndMetrics`
1501
+ // below: that flag governs the client-facing `metrics` frame, and a
1502
+ // failed session (which suppresses the frame) is the one whose end
1503
+ // telemetry matters most. Session duration downstream is this row's
1504
+ // `recorded_at` minus the started row's, so it must be written before the
1505
+ // teardown below, which awaits a pending continuation and can run long.
1506
+ const failed = this.state === "failed";
1507
+ recordLiveVoiceSessionEnded({
1508
+ sessionId: this.context.sessionId,
1509
+ screen: liveVoiceEndScreen(reason, failed ? this.failureCode : null),
1510
+ outcome: failed ? "failed" : "completed",
1511
+ });
1512
+
1269
1513
  const shouldEmitSessionEndMetrics = this.state !== "failed";
1270
1514
  this.state = "closed";
1271
1515
  // Retire the island before the teardown below starts awaiting things. A
@@ -1275,6 +1519,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1275
1519
  this.liveActivityReporter.end();
1276
1520
  this.turnDetector?.dispose();
1277
1521
  this.clearEndpointExtensionTimer();
1522
+ this.clearProviderTurnEndTimer();
1278
1523
  // There is no longer anyone on the call to speak to, so a queued
1279
1524
  // announcement cannot be spoken — but the answer behind it is finished
1280
1525
  // work the user asked for, so it takes the same conversation route a
@@ -1316,6 +1561,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1316
1561
  // Persistent re-arm: the shared stream is already open, so the cycle
1317
1562
  // goes straight to streaming with no resolve/start round-trip.
1318
1563
  utterance.transcriber = shared;
1564
+ utterance.dialedSttProvider = shared.providerId;
1319
1565
  return await this.activateUtterance(utterance, replayTurnEnd);
1320
1566
  }
1321
1567
  // The shared stream is pinned to the old language, so retire it and
@@ -1325,11 +1571,28 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1325
1571
  }
1326
1572
 
1327
1573
  try {
1328
- // One language snapshot serves both the dial and the re-arm
1329
- // comparison: passing it to the resolver keeps the stream's actual
1330
- // language and the recorded sharedTranscriberLanguage identical even
1331
- // when config changes while the resolver awaits credentials.
1332
- const sttLanguage = getConfig().services.stt.language;
1574
+ // One config snapshot serves the dial, the re-arm comparison and the
1575
+ // latch seed below: passing the language to the resolver keeps the
1576
+ // stream's actual language and the recorded sharedTranscriberLanguage
1577
+ // identical even when config changes while the resolver awaits
1578
+ // credentials.
1579
+ const stt = getConfig().services.stt;
1580
+ const sttLanguage = stt.language;
1581
+ // Seed the latch from the CONFIGURED provider before the dial, not
1582
+ // only from the resolved one after it. `start()` sends `ready` without
1583
+ // waiting on this resolve, so the caller can speak and close an entire
1584
+ // silence boundary while the provider handshake is still in flight. A
1585
+ // latch still on its default `false` at that boundary hands the first
1586
+ // turn of the session to the silence path and then ignores the
1587
+ // provider end-of-turn that follows, which is invisible from the
1588
+ // outside: the session looks like a turn-detecting session while its
1589
+ // opening turn is not one. The resolved provider reconciles this guess
1590
+ // below.
1591
+ this.setProviderTurnEndActive(
1592
+ this.fluxConfig.turnEnd.enabled &&
1593
+ supportsProviderTurnDetection(stt.provider as SttProviderId) &&
1594
+ this.turnDetector !== null,
1595
+ );
1333
1596
  const transcriber = await this.resolveTranscriber({
1334
1597
  sampleRate: this.context.startFrame.audio.sampleRate,
1335
1598
  ...(sttLanguage ? { language: sttLanguage } : {}),
@@ -1341,6 +1604,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1341
1604
  }
1342
1605
 
1343
1606
  if (!transcriber) {
1607
+ // No stream answered, so no end-of-turn ever will: the guess above
1608
+ // must not outlive the dial that disproved it.
1609
+ this.setProviderTurnEndActive(false);
1344
1610
  return {
1345
1611
  status: "unavailable",
1346
1612
  message: unavailableTranscriberMessage(),
@@ -1348,6 +1614,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1348
1614
  }
1349
1615
 
1350
1616
  utterance.transcriber = transcriber;
1617
+ utterance.dialedSttProvider = transcriber.providerId;
1618
+ // Reconcile the pre-dial guess with the provider that actually
1619
+ // answered. The resolver reads the provider from config, which cannot
1620
+ // change under a live session, so this normally confirms the guess; it
1621
+ // clears it when the dial fell back to another provider or resolved
1622
+ // one the config did not name.
1623
+ this.setProviderTurnEndActive(
1624
+ this.fluxConfig.turnEnd.enabled &&
1625
+ supportsProviderTurnDetection(transcriber.providerId) &&
1626
+ this.turnDetector !== null,
1627
+ );
1351
1628
  if (
1352
1629
  this.turnDetector &&
1353
1630
  typeof transcriber.finalizeUtterance === "function"
@@ -1381,6 +1658,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1381
1658
  if (this.isUtteranceStale(utterance)) {
1382
1659
  return { status: "stale" };
1383
1660
  }
1661
+ // The dial threw, so the pre-dial guess is disproved the same way the
1662
+ // unavailable case disproves it: nothing will send an end-of-turn.
1663
+ this.setProviderTurnEndActive(false);
1384
1664
  return {
1385
1665
  status: "error",
1386
1666
  message: `Live voice transcription could not be started: ${errorMessage(
@@ -1508,6 +1788,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1508
1788
 
1509
1789
  this.state = "failed";
1510
1790
  this.clearEndpointExtensionTimer();
1791
+ this.clearProviderTurnEndTimer();
1511
1792
  await this.sendFrame({
1512
1793
  type: "error",
1513
1794
  code:
@@ -1568,12 +1849,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1568
1849
  return;
1569
1850
  }
1570
1851
 
1571
- const hasSpeech = detectPcm16SpeechActivity(
1572
- chunk,
1573
- this.speechEnergyThreshold,
1574
- );
1852
+ for (const classified of this.classifyVadEnergy(chunk)) {
1853
+ await this.handleClassifiedVadAudio(detector, classified);
1854
+ }
1855
+ }
1856
+
1857
+ private async handleClassifiedVadAudio(
1858
+ detector: MediaTurnDetector,
1859
+ classified: VadClassifiedChunk,
1860
+ ): Promise<void> {
1861
+ const { chunk, classification: energyClassification } = classified;
1862
+ const hasSpeech = energyClassification === "speech";
1575
1863
  detector.onMediaChunk(hasSpeech);
1576
- this.trackBargeInGuard(hasSpeech, chunk);
1864
+ this.trackBargeInGuard(energyClassification, chunk);
1865
+ if (hasSpeech) {
1866
+ this.localSpeechStopAtMs = Date.now();
1867
+ }
1868
+
1869
+ // Playback echo is neither user audio nor useful pre-roll. Dropping it
1870
+ // prevents the assistant's reply from reaching transcription as a ghost
1871
+ // follow-up turn.
1872
+ if (energyClassification === "echo") {
1873
+ return;
1874
+ }
1577
1875
 
1578
1876
  // Idle mic: hold silent chunks in the bounded pre-roll instead of
1579
1877
  // collecting or streaming them; flushed on speech onset so the
@@ -1651,6 +1949,193 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1651
1949
  await this.routeVadAudio(utterance, chunk);
1652
1950
  }
1653
1951
 
1952
+ /**
1953
+ * Classify microphone energy while keeping assistant playback echo out of
1954
+ * barge-in, turn detection, pre-roll, and transcription.
1955
+ *
1956
+ * A short onset probe must correlate with PCM sent to the speaker before its
1957
+ * microphone power can seed the adaptive threshold. Nonmatching probe audio
1958
+ * is replayed through VAD in original order, so a user who talks at playback
1959
+ * onset is neither learned as echo nor lost. Once seeded, the EMA follows
1960
+ * confirmed echo while speech above the learned margin remains frozen out.
1961
+ */
1962
+ private classifyVadEnergy(chunk: Buffer): VadClassifiedChunk[] {
1963
+ const baseThreshold =
1964
+ this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD;
1965
+ const meanAmplitude = pcm16MeanAmplitude(chunk);
1966
+ if (
1967
+ this.echoBargeInMargin <= 1 ||
1968
+ !this.isAssistantPlaybackEchoPossible()
1969
+ ) {
1970
+ this.resetEchoReference();
1971
+ return [
1972
+ this.classifyAtFixedThreshold(chunk, baseThreshold, meanAmplitude),
1973
+ ];
1974
+ }
1975
+
1976
+ if (this.echoWindowTotalAudioMs === 0) {
1977
+ this.echoWindowGuardCarryover =
1978
+ this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
1979
+ } else if (this.pendingBargeIn === null) {
1980
+ this.echoWindowGuardCarryover = false;
1981
+ }
1982
+
1983
+ const chunkMs = pcm16DurationMs(
1984
+ chunk.byteLength,
1985
+ this.context.startFrame.audio.sampleRate,
1986
+ );
1987
+ const onsetWasEligible =
1988
+ !this.echoOnsetLapsed &&
1989
+ this.echoWindowTotalAudioMs < ECHO_ONSET_ELIGIBILITY_MS;
1990
+ this.echoWindowTotalAudioMs += chunkMs;
1991
+
1992
+ if (this.echoProbeChunks.length > 0) {
1993
+ this.echoProbeChunks.push(Buffer.from(chunk));
1994
+ return this.resolveEchoProbe(baseThreshold);
1995
+ }
1996
+
1997
+ if (meanAmplitude <= baseThreshold) {
1998
+ this.echoSubBaseRunMs += chunkMs;
1999
+ if (this.echoSubBaseRunMs >= ECHO_ONSET_ELIGIBILITY_MS) {
2000
+ this.echoEnergyEma = 0;
2001
+ this.echoOnsetLapsed = true;
2002
+ }
2003
+ return [{ chunk, classification: "silence" }];
2004
+ }
2005
+
2006
+ this.echoSubBaseRunMs = 0;
2007
+ if (
2008
+ this.echoEnergyEma === 0 &&
2009
+ onsetWasEligible &&
2010
+ !this.echoWindowGuardCarryover
2011
+ ) {
2012
+ this.echoProbeChunks.push(Buffer.from(chunk));
2013
+ return this.resolveEchoProbe(baseThreshold);
2014
+ }
2015
+
2016
+ if (this.echoEnergyEma === 0) {
2017
+ this.echoOnsetLapsed = true;
2018
+ return [{ chunk, classification: "speech" }];
2019
+ }
2020
+
2021
+ const speechThreshold = Math.max(
2022
+ baseThreshold,
2023
+ this.echoBargeInMargin * this.echoEnergyEma,
2024
+ );
2025
+ if (meanAmplitude > speechThreshold) {
2026
+ const guardHasSpeech =
2027
+ this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
2028
+ if (!guardHasSpeech && this.echoMatchesAssistant(chunk)) {
2029
+ this.updateEchoEnergy(meanAmplitude, chunkMs);
2030
+ return [{ chunk, classification: "echo" }];
2031
+ }
2032
+ return [{ chunk, classification: "speech" }];
2033
+ }
2034
+
2035
+ this.updateEchoEnergy(meanAmplitude, chunkMs);
2036
+ return [{ chunk, classification: "echo" }];
2037
+ }
2038
+
2039
+ private resolveEchoProbe(baseThreshold: number): VadClassifiedChunk[] {
2040
+ const probe = Buffer.concat(this.echoProbeChunks);
2041
+ const probeAudioMs = pcm16DurationMs(
2042
+ probe.byteLength,
2043
+ this.context.startFrame.audio.sampleRate,
2044
+ );
2045
+ if (
2046
+ probeAudioMs >= ECHO_CORRELATION_MIN_MS &&
2047
+ this.echoMatchesAssistant(probe)
2048
+ ) {
2049
+ this.echoEnergyEma = Math.max(baseThreshold, pcm16MeanAmplitude(probe));
2050
+ const chunks = this.echoProbeChunks.splice(0);
2051
+ return chunks.map((chunk) => ({ chunk, classification: "echo" }));
2052
+ }
2053
+ if (probeAudioMs < ECHO_CORRELATION_PROBE_MS) {
2054
+ return [];
2055
+ }
2056
+
2057
+ this.echoOnsetLapsed = true;
2058
+ const chunks = this.echoProbeChunks.splice(0);
2059
+ return chunks.map((chunk) =>
2060
+ this.classifyAtFixedThreshold(chunk, baseThreshold),
2061
+ );
2062
+ }
2063
+
2064
+ private echoMatchesAssistant(chunk: Buffer): boolean {
2065
+ const sampleRate = this.context.startFrame.audio.sampleRate;
2066
+ const minimumBytes = Math.ceil(
2067
+ (sampleRate * ECHO_CORRELATION_MIN_MS * 2) / 1_000,
2068
+ );
2069
+ if (
2070
+ chunk.byteLength < minimumBytes ||
2071
+ this.echoReferenceAudio.byteLength < minimumBytes
2072
+ ) {
2073
+ return false;
2074
+ }
2075
+ const probeByteLength = Math.min(
2076
+ chunk.byteLength,
2077
+ Math.ceil((sampleRate * ECHO_CORRELATION_PROBE_MS * 2) / 1_000),
2078
+ );
2079
+ return (
2080
+ pcm16MaxNormalizedCorrelation(
2081
+ chunk.subarray(0, probeByteLength),
2082
+ this.echoReferenceAudio,
2083
+ ) >= ECHO_CORRELATION_THRESHOLD
2084
+ );
2085
+ }
2086
+
2087
+ private updateEchoEnergy(meanAmplitude: number, chunkMs: number): void {
2088
+ const alpha = 1 - 0.5 ** (chunkMs / this.echoEmaHalfLifeMs);
2089
+ this.echoEnergyEma =
2090
+ alpha * meanAmplitude + (1 - alpha) * this.echoEnergyEma;
2091
+ }
2092
+
2093
+ private classifyAtFixedThreshold(
2094
+ chunk: Buffer,
2095
+ baseThreshold: number,
2096
+ meanAmplitude = pcm16MeanAmplitude(chunk),
2097
+ ): VadClassifiedChunk {
2098
+ return {
2099
+ chunk,
2100
+ classification: meanAmplitude > baseThreshold ? "speech" : "silence",
2101
+ };
2102
+ }
2103
+
2104
+ private isAssistantPlaybackEchoPossible(): boolean {
2105
+ return (
2106
+ Date.now() < this.assistantPlaybackTailUntilMs + this.echoDrainSlackMs
2107
+ );
2108
+ }
2109
+
2110
+ private resetEchoReference(): void {
2111
+ this.echoEnergyEma = 0;
2112
+ this.echoProbeChunks = [];
2113
+ this.echoReferenceAudio = Buffer.alloc(0);
2114
+ this.echoWindowTotalAudioMs = 0;
2115
+ this.echoSubBaseRunMs = 0;
2116
+ this.echoOnsetLapsed = false;
2117
+ this.echoWindowGuardCarryover = false;
2118
+ }
2119
+
2120
+ private appendEchoReference(chunk: LiveVoiceTtsAudioChunk): void {
2121
+ if (
2122
+ chunk.contentType.split(";", 1)[0]?.trim().toLowerCase() !==
2123
+ "audio/pcm" ||
2124
+ chunk.sampleRate !== this.context.startFrame.audio.sampleRate
2125
+ ) {
2126
+ return;
2127
+ }
2128
+ const audio = Buffer.from(chunk.dataBase64, "base64");
2129
+ const maxBytes = Math.ceil(
2130
+ (chunk.sampleRate * ECHO_REFERENCE_MAX_MS * 2) / 1_000,
2131
+ );
2132
+ const combined = Buffer.concat([this.echoReferenceAudio, audio]);
2133
+ this.echoReferenceAudio =
2134
+ combined.byteLength > maxBytes
2135
+ ? combined.subarray(combined.byteLength - maxBytes)
2136
+ : combined;
2137
+ }
2138
+
1654
2139
  private async routeVadAudio(
1655
2140
  utterance: UtteranceCycle,
1656
2141
  chunk: Buffer,
@@ -1780,6 +2265,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1780
2265
  // utterance keeps accumulating and the detector fires a fresh turn-end.
1781
2266
  this.vadSpeechGeneration += 1;
1782
2267
  this.clearEndpointExtensionTimer();
2268
+ // ...and so is a fallback deadline armed for the boundary the caller just
2269
+ // spoke through. The next silence boundary arms a fresh one.
2270
+ this.clearProviderTurnEndTimer();
1783
2271
 
1784
2272
  // Speech resumed while a speculative leg was awaiting its verdict: the
1785
2273
  // pause was mid-thought after all. Discard silently (no frames were ever
@@ -1798,7 +2286,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1798
2286
  // The client can still be draining audible playback after tts_done
1799
2287
  // (the turn is already cleared server-side) — that tail deserves the
1800
2288
  // same guard, or a noise blip clips the reply's last words.
1801
- const drainingPlayback = Date.now() < this.assistantPlaybackTailUntilMs;
2289
+ const drainingPlayback = this.isAssistantPlaybackEchoPossible();
1802
2290
 
1803
2291
  if ((bargeableTurn || drainingPlayback) && this.bargeInMinSpeechMs > 0) {
1804
2292
  // Onset audio keeps flowing into the cycle/pre-roll while the guard
@@ -1820,13 +2308,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1820
2308
  }
1821
2309
  }
1822
2310
 
1823
- // Advances the sustained-speech barge-in guard by one server-VAD chunk:
1824
- // above-gate speech accumulates toward bargeInMinSpeechMs while brief
1825
- // sub-threshold gaps are tolerated (a single continuous silence longer than
1826
- // BARGE_IN_GAP_TOLERANCE_MS, or cumulative tolerated silence past the
1827
- // duty-cycle ceiling, zeroes the run), and once met the deferred
1828
- // speech_started + barge-in fire.
1829
- private trackBargeInGuard(hasSpeech: boolean, chunk: Buffer): void {
2311
+ // Advance the sustained-speech barge-in guard by one server-VAD chunk.
2312
+ // Speech accumulates toward bargeInMinSpeechMs, short true-silence gaps are
2313
+ // tolerated, and classified playback echo resets the run immediately.
2314
+ // Longer or mostly silent runs reset through the existing gap limits.
2315
+ private trackBargeInGuard(
2316
+ classification: VadEnergyClassification,
2317
+ chunk: Buffer,
2318
+ ): void {
1830
2319
  const guard = this.pendingBargeIn;
1831
2320
  if (!guard) {
1832
2321
  return;
@@ -1835,7 +2324,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1835
2324
  chunk.byteLength,
1836
2325
  this.context.startFrame.audio.sampleRate,
1837
2326
  );
1838
- if (!hasSpeech) {
2327
+ if (classification === "echo") {
2328
+ this.resetBargeInGuardRun();
2329
+ return;
2330
+ }
2331
+ if (classification === "silence") {
1839
2332
  guard.silenceMs += chunkMs;
1840
2333
  guard.toleratedSilenceMs += chunkMs;
1841
2334
  // Strictly greater on the per-gap check: a gap of exactly
@@ -1849,9 +2342,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1849
2342
  guard.toleratedSilenceMs >
1850
2343
  this.bargeInMinSpeechMs * BARGE_IN_MAX_TOLERATED_SILENCE_RATIO
1851
2344
  ) {
1852
- guard.speechMs = 0;
1853
- guard.silenceMs = 0;
1854
- guard.toleratedSilenceMs = 0;
2345
+ this.resetBargeInGuardRun();
1855
2346
  }
1856
2347
  return;
1857
2348
  }
@@ -1869,6 +2360,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1869
2360
  }
1870
2361
  }
1871
2362
 
2363
+ private resetBargeInGuardRun(): void {
2364
+ const guard = this.pendingBargeIn;
2365
+ if (!guard) {
2366
+ return;
2367
+ }
2368
+ guard.speechMs = 0;
2369
+ guard.silenceMs = 0;
2370
+ guard.toleratedSilenceMs = 0;
2371
+ if (this.echoWindowGuardCarryover) {
2372
+ this.echoWindowGuardCarryover = false;
2373
+ this.echoEnergyEma = 0;
2374
+ this.echoProbeChunks = [];
2375
+ }
2376
+ }
2377
+
1872
2378
  private bargeIn(turn: ActiveAssistantTurn): void {
1873
2379
  // Abort synchronously so no tts_audio frame can follow turn_cancelled,
1874
2380
  // and settle the cancelled turn's metrics so the next utterance's marks
@@ -1918,7 +2424,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1918
2424
  // Tagged reason: provider catch-sites classify untagged caller aborts as
1919
2425
  // retryable transport failures (ERROR log + futile retry against the
1920
2426
  // aborted signal). This signal reaches the brain leg and any in-flight
1921
- // ack generation.
2427
+ // progress narration.
1922
2428
  turn.abortController.abort(
1923
2429
  createAbortReason("voice_session_aborted", "live-voice-barge-in"),
1924
2430
  );
@@ -2456,8 +2962,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2456
2962
  // Spoken, because opening the room is only a cue for someone looking at
2457
2963
  // the screen, and the case this exists for is a phone the user has put
2458
2964
  // down. One line, not narration: the turn is not working, it is waiting,
2459
- // and it says which.
2460
- this.enqueueFillerPhrase(turn, APPROVAL_PENDING_PHRASE);
2965
+ // and it says which, in the turn's spoken language, like every other
2966
+ // filler phrase.
2967
+ this.enqueueFillerPhrase(
2968
+ turn,
2969
+ approvalPendingPhraseFor(turn.language),
2970
+ this.fixedPhraseLanguage(turn, APPROVAL_PENDING_PHRASE_BY_LANGUAGE),
2971
+ );
2461
2972
  }
2462
2973
 
2463
2974
  /** Clear the wait once a decision lands, so the turn narrates normally again. */
@@ -2666,8 +3177,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2666
3177
  this.pendingBargeIn = null;
2667
3178
  if (reason === "max-duration") {
2668
3179
  // A max-duration boundary always releases: drop any pending hold
2669
- // replay so it cannot re-fire a boundary this one already owns.
3180
+ // replay (or provider fallback deadline) so it cannot re-fire a
3181
+ // boundary this one already owns.
2670
3182
  this.clearEndpointExtensionTimer();
3183
+ this.clearProviderTurnEndTimer();
2671
3184
  }
2672
3185
  const utterance = this.currentUtterance;
2673
3186
  if (!utterance || utterance.released || utterance.completed) {
@@ -2689,6 +3202,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2689
3202
  // release (ptt_release forced the boundary) is the user saying "answer
2690
3203
  // now" — never second-guess it.
2691
3204
  if (reason === "silence" && !manualRelease) {
3205
+ // The provider owns this boundary: its end-of-turn commits the
3206
+ // utterance, so the silence timer only arms the fail-open deadline. An
3207
+ // utterance whose deadline already elapsed falls through to the hold
3208
+ // path below, which is the whole point of the fallback.
3209
+ if (this.providerTurnEndActive && !utterance.providerTurnEndTimedOut) {
3210
+ // Stamp the speech run this boundary closed. This is the staleness
3211
+ // signal of last resort, used only for a provider that does not
3212
+ // number its turns: where the provider's own turn index is on the
3213
+ // wire it answers the question directly (see
3214
+ // isStaleProviderTurnEnd).
3215
+ utterance.turnBoundaryGeneration = this.vadSpeechGeneration;
3216
+ this.armProviderTurnEndFallbackTimer(utterance);
3217
+ return;
3218
+ }
2692
3219
  if (await this.launchSpeculativeAssistantTurn(utterance)) {
2693
3220
  return;
2694
3221
  }
@@ -2719,6 +3246,217 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2719
3246
  }, this.frontModelConfig.endpointExtensionMs);
2720
3247
  }
2721
3248
 
3249
+ // Arms the fail-open deadline for a provider end-of-turn. A turn-detecting
3250
+ // provider force-ends its own turn after `eotTimeoutMs` of silence, so once
3251
+ // that budget plus a margin has passed since the caller stopped speaking, no
3252
+ // end-of-turn is coming and the utterance replays this boundary on the
3253
+ // silence path.
3254
+ // `waitMsOverride` collapses that wait when the caller already knows no
3255
+ // end-of-turn is coming (see setProviderTurnEndActive).
3256
+ private armProviderTurnEndFallbackTimer(
3257
+ utterance: UtteranceCycle,
3258
+ waitMsOverride?: number,
3259
+ ): void {
3260
+ this.clearProviderTurnEndTimer();
3261
+ const budgetMs =
3262
+ this.fluxConfig.eotTimeoutMs + PROVIDER_TURN_END_FALLBACK_MARGIN_MS;
3263
+ const waitMs =
3264
+ waitMsOverride ?? Math.max(0, budgetMs - this.msSinceLocalSpeechStop());
3265
+ this.providerTurnEndTimer = setTimeout(() => {
3266
+ this.providerTurnEndTimer = null;
3267
+ if (
3268
+ this.currentUtterance !== utterance ||
3269
+ utterance.released ||
3270
+ utterance.completed ||
3271
+ utterance.assistantTurnStarted ||
3272
+ // Speech resumed (the detector owns the next boundary). Onset also
3273
+ // clears this timer, so this is a belt to that suspender.
3274
+ this.turnDetector?.isActive
3275
+ ) {
3276
+ return;
3277
+ }
3278
+ utterance.providerTurnEndTimedOut = true;
3279
+ log.warn(
3280
+ { budgetMs, waitMs },
3281
+ "No provider end-of-turn is coming; falling back to the silence boundary for this utterance",
3282
+ );
3283
+ this.handleVadUtteranceEnd("silence");
3284
+ }, waitMs);
3285
+ }
3286
+
3287
+ /**
3288
+ * Flips the provider end-of-turn latch, unwinding an optimistic arm.
3289
+ *
3290
+ * `beginUtterance` arms the latch from the configured provider before the
3291
+ * dial resolves, so a silence boundary can already have deferred to the
3292
+ * provider by the time the resolved one says otherwise. That deferred
3293
+ * boundary is parked on the fail-open deadline, a whole end-of-turn budget
3294
+ * away, waiting for an event that will now never arrive. Collapse the wait to
3295
+ * zero rather than burn the budget: the deadline body re-checks the cycle
3296
+ * and replays the silence boundary, which with the latch down takes the
3297
+ * ordinary hold path. Replaying through the deadline instead of calling
3298
+ * `handleVadUtteranceEnd` directly keeps the release off the arming
3299
+ * caller's stack, which is still mid-dial.
3300
+ *
3301
+ * A cleared latch with no deadline armed needs no unwind: either no
3302
+ * boundary ever deferred, or the caller resumed speaking and
3303
+ * `handleVadSpeechStart` already dropped the deadline, leaving the detector
3304
+ * owning the next boundary. The cycle's `turnBoundaryGeneration` stamp is
3305
+ * left as it is: `isStaleProviderTurnEnd` is only ever consulted from
3306
+ * `handleProviderTurnEnd`, which returns immediately once the latch is down.
3307
+ */
3308
+ private setProviderTurnEndActive(active: boolean): void {
3309
+ if (active === this.providerTurnEndActive) {
3310
+ return;
3311
+ }
3312
+ this.providerTurnEndActive = active;
3313
+ if (active || this.providerTurnEndTimer === null) {
3314
+ return;
3315
+ }
3316
+ const utterance = this.currentUtterance;
3317
+ this.clearProviderTurnEndTimer();
3318
+ if (utterance && !utterance.released && !utterance.completed) {
3319
+ this.armProviderTurnEndFallbackTimer(utterance, 0);
3320
+ }
3321
+ }
3322
+
3323
+ /**
3324
+ * Record the provider turn a cycle is currently inside. Providers without
3325
+ * turn numbering send no index, which leaves the cycle on the local speech
3326
+ * generation as its only staleness signal (see isStaleProviderTurnEnd).
3327
+ */
3328
+ private recordProviderTurnStart(
3329
+ utterance: UtteranceCycle,
3330
+ turnIndex: number | undefined,
3331
+ ): void {
3332
+ if (turnIndex === undefined) {
3333
+ return;
3334
+ }
3335
+ utterance.openProviderTurnIndex = turnIndex;
3336
+ }
3337
+
3338
+ /**
3339
+ * A provider end-of-turn that lost its race with the caller's next breath:
3340
+ * it describes speech the caller has already spoken past. Acting on one
3341
+ * would send utterance_end and stop the transcriber while local VAD is
3342
+ * still routing the resumed speech into this same cycle, cutting off the
3343
+ * next words or folding them into the wrong turn.
3344
+ *
3345
+ * The provider's own turn numbering answers this directly and is preferred
3346
+ * wherever it is available: the cycle records the newest turn the provider
3347
+ * has opened, so an end-of-turn for an older index closes a turn the
3348
+ * provider itself has already superseded. An end-of-turn for the turn still
3349
+ * in progress is never stale, however many times the caller drew breath
3350
+ * inside it: the mid-thought pause is exactly the case a provider's turn
3351
+ * model exists to judge, and its verdict covers the resumed speech too.
3352
+ *
3353
+ * With no turn numbering the local speech generation is the only signal
3354
+ * left. The silence boundary stamps the cycle with the generation it
3355
+ * closed and `handleVadSpeechStart` bumps that generation when the caller
3356
+ * resumes, so a turn-end arriving on a newer generation is treated as
3357
+ * stale. That is conservative: it also drops the fast end-of-turn this
3358
+ * feature exists for, which the turn index would have committed. Before
3359
+ * any boundary has fired the stamp is null and nothing is stale.
3360
+ */
3361
+ private isStaleProviderTurnEnd(
3362
+ utterance: UtteranceCycle,
3363
+ turnIndex: number | undefined,
3364
+ ): boolean {
3365
+ if (turnIndex !== undefined && utterance.openProviderTurnIndex !== null) {
3366
+ return turnIndex < utterance.openProviderTurnIndex;
3367
+ }
3368
+ return (
3369
+ utterance.turnBoundaryGeneration !== null &&
3370
+ utterance.turnBoundaryGeneration !== this.vadSpeechGeneration
3371
+ );
3372
+ }
3373
+
3374
+ private clearProviderTurnEndTimer(): void {
3375
+ if (this.providerTurnEndTimer !== null) {
3376
+ clearTimeout(this.providerTurnEndTimer);
3377
+ this.providerTurnEndTimer = null;
3378
+ }
3379
+ }
3380
+
3381
+ // Time since the local VAD last heard above-gate audio: the speech-stop
3382
+ // mark both the fallback deadline and the reported end-of-turn latency are
3383
+ // measured from. Zero when no speech has been heard at all.
3384
+ private msSinceLocalSpeechStop(): number {
3385
+ if (this.localSpeechStopAtMs === null) {
3386
+ return 0;
3387
+ }
3388
+ return Math.max(0, Date.now() - this.localSpeechStopAtMs);
3389
+ }
3390
+
3391
+ /**
3392
+ * The provider committed an end of turn: the caller has finished, so the utterance
3393
+ * commits now instead of waiting out the trailing-silence timer and asking
3394
+ * the front door whether the pause was mid-thought. This runs the ordinary
3395
+ * post-release path (utterance_end, then release), so the front-door leg it
3396
+ * dispatches is non-speculative: `speculativeHoldAllowed` is false, its
3397
+ * decision rule is built with `includeHold: false`, and the model is told
3398
+ * the caller has finished. Escalation and progress narration are untouched;
3399
+ * only the hold verdict is bypassed.
3400
+ */
3401
+ private async handleProviderTurnEnd(
3402
+ utterance: UtteranceCycle,
3403
+ turnIndex: number | undefined,
3404
+ ): Promise<void> {
3405
+ if (
3406
+ !this.providerTurnEndActive ||
3407
+ this.currentUtterance !== utterance ||
3408
+ utterance.released ||
3409
+ utterance.completed ||
3410
+ utterance.assistantTurnStarted ||
3411
+ // The fallback already took this utterance back to the silence path.
3412
+ utterance.providerTurnEndTimedOut
3413
+ ) {
3414
+ return;
3415
+ }
3416
+ if (this.isStaleProviderTurnEnd(utterance, turnIndex)) {
3417
+ // At `info`, not `debug`: a drop is rare and is the only signal of the
3418
+ // one known latency-outlier mode, so an operator measuring the two
3419
+ // paths has to be able to see it at the default log level.
3420
+ log.info(
3421
+ {
3422
+ turnIndex,
3423
+ openTurnIndex: utterance.openProviderTurnIndex,
3424
+ boundaryGeneration: utterance.turnBoundaryGeneration,
3425
+ speechGeneration: this.vadSpeechGeneration,
3426
+ },
3427
+ "Dropping a stale provider end-of-turn: the caller resumed speaking past the boundary it closed",
3428
+ );
3429
+ // The cycle stays open and local VAD keeps routing the resumed speech
3430
+ // into it. The detector's next silence boundary re-stamps the
3431
+ // generation and re-arms the fail-open deadline, so the turn still
3432
+ // commits: on the end-of-turn for the resumed speech, or on that
3433
+ // deadline. Re-arm here only if the detector has somehow already gone
3434
+ // idle, so the deadline handleVadSpeechStart cleared can never strand
3435
+ // the cycle with nothing left to close it.
3436
+ if (this.turnDetector?.isActive !== true) {
3437
+ utterance.turnBoundaryGeneration = this.vadSpeechGeneration;
3438
+ this.armProviderTurnEndFallbackTimer(utterance);
3439
+ }
3440
+ return;
3441
+ }
3442
+ this.clearProviderTurnEndTimer();
3443
+ this.markEndpointDecision(
3444
+ utterance,
3445
+ "release",
3446
+ this.msSinceLocalSpeechStop(),
3447
+ "provider",
3448
+ );
3449
+ await this.sendFrame({ type: "utterance_end", reason: "silence" });
3450
+ await this.releaseUtterance();
3451
+ // Leave the local detector idle, which is where every other commit path
3452
+ // leaves it. A provider can close a turn while the trailing-silence
3453
+ // countdown is still running, and barge-in fires from the detector's speech
3454
+ // ONSET: a detector left mid-turn reports no onset, so the caller could
3455
+ // not interrupt the reply they just triggered. The forced boundary
3456
+ // reaches an already-released utterance and returns.
3457
+ this.turnDetector?.forceEnd();
3458
+ }
3459
+
2722
3460
  /**
2723
3461
  * Unified front-door: dispatch the assistant turn speculatively at the
2724
3462
  * silence boundary, judging the transcript accumulated so far — the same
@@ -2780,6 +3518,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2780
3518
  // are skipped; the thinking frame and timers still apply.
2781
3519
  const alreadyReleased = utterance.released;
2782
3520
  turn.speculativePending = false;
3521
+ // Finals can land between the speculative dispatch and this verdict.
3522
+ // Fill the language only when dispatch had none: the model request was
3523
+ // already issued with the dispatch language, so overwriting here would
3524
+ // hint TTS (and any voice override) in a different language than the
3525
+ // text it speaks. The tally still carries the corrected detection into
3526
+ // the next turn.
3527
+ turn.language ??= this.turnLanguageFor(utterance);
2783
3528
  if (turn.verdictDeadlineTimer !== null) {
2784
3529
  clearTimeout(turn.verdictDeadlineTimer);
2785
3530
  turn.verdictDeadlineTimer = null;
@@ -2791,13 +3536,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2791
3536
  if (!alreadyReleased) {
2792
3537
  void this.releaseUtterance();
2793
3538
  }
2794
- if (this.streamTtsAudio) {
2795
- this.armAckTimer(turn);
2796
- }
2797
3539
  if (
2798
3540
  this.frontModelConfig.progress.enabled &&
2799
3541
  this.streamTtsAudio &&
2800
- this.frontDecider
3542
+ this.progressNarrator
2801
3543
  ) {
2802
3544
  this.armProgressIdleTimer(turn);
2803
3545
  }
@@ -2864,8 +3606,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2864
3606
  ): void {
2865
3607
  turn.speculativePending = false;
2866
3608
  // The dispatch is being unwound, so the barge-in merge note, the finished
2867
- // continuation's answer, and its queued announcement all go back to the
2868
- // session — the turn that would have delivered them is gone.
3609
+ // continuation's answer, its queued announcement, and any photos it claimed
3610
+ // all go back to the session. The turn that would have delivered them is
3611
+ // gone, and the utterance they belong to is about to be sent by another.
2869
3612
  this.restorePendingTurnContext(turn);
2870
3613
  // Latched before the handle check: when the discard beats the bridge
2871
3614
  // handle's resolution (startVoiceTurn still persisting), the handle's
@@ -2930,6 +3673,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2930
3673
  !utterance.completed
2931
3674
  ) {
2932
3675
  this.clearEndpointExtensionTimer();
3676
+ this.clearProviderTurnEndTimer();
2933
3677
  await this.sendFrame({ type: "utterance_end", reason: "silence" });
2934
3678
  }
2935
3679
  await this.releaseUtterance();
@@ -2944,6 +3688,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2944
3688
  if (utterance.phase === "transcriber_closed") {
2945
3689
  utterance.released = true;
2946
3690
  this.markUtteranceReleased(utterance);
3691
+ this.markEndpointCommit(utterance);
2947
3692
  await this.startAssistantTurnIfReady();
2948
3693
  await this.drainOutboundFrames();
2949
3694
  return;
@@ -2955,6 +3700,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2955
3700
 
2956
3701
  utterance.released = true;
2957
3702
  this.markUtteranceReleased(utterance);
3703
+ this.markEndpointCommit(utterance);
2958
3704
 
2959
3705
  if (utterance.phase === "pending") {
2960
3706
  // The transcriber is still starting; beginUtterance completes the release.
@@ -3117,16 +3863,39 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3117
3863
  switch (event.type) {
3118
3864
  case "partial":
3119
3865
  utterance.latestPartialText = event.text;
3866
+ this.capturePartialLanguages(utterance, event.languages);
3120
3867
  this.markFirstPartial(utterance);
3121
3868
  await this.sendFrame({ type: "stt_partial", text: event.text });
3122
3869
  return;
3123
3870
  case "final":
3124
- await this.recordFinalTranscript(utterance, event.text);
3871
+ await this.recordFinalTranscript(
3872
+ utterance,
3873
+ event.text,
3874
+ event.languages,
3875
+ );
3125
3876
  return;
3126
3877
  case "finalized":
3127
3878
  // Per-cycle transcribers are torn down with stop(); the finalize
3128
3879
  // completion signal has no cycle to advance here.
3129
3880
  return;
3881
+ case "turn-start":
3882
+ // Barge-in is deliberately untouched: local VAD still owns it,
3883
+ // because a provider roundtrip cannot beat a local energy gate on an
3884
+ // interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
3885
+ // index is recorded so a later end-of-turn can be told apart from one
3886
+ // this turn superseded (see isStaleProviderTurnEnd).
3887
+ this.recordProviderTurnStart(utterance, event.turnIndex);
3888
+ return;
3889
+ case "eager-turn-end":
3890
+ case "turn-resumed":
3891
+ // Speculative end-of-turn stays off (the config leaves
3892
+ // `eagerEotThreshold` unset, so Deepgram never emits these). Wiring
3893
+ // them onto the speculative dispatch machinery is a deferred
3894
+ // follow-up.
3895
+ return;
3896
+ case "turn-end":
3897
+ await this.handleProviderTurnEnd(utterance, event.turnIndex);
3898
+ return;
3130
3899
  case "error":
3131
3900
  await this.sendTranscriberErrorFrame(event);
3132
3901
  return;
@@ -3148,6 +3917,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3148
3917
  }
3149
3918
  await this.startAssistantTurnIfReady();
3150
3919
  return;
3920
+ default: {
3921
+ const _exhaustive: never = event;
3922
+ return;
3923
+ }
3151
3924
  }
3152
3925
  }
3153
3926
 
@@ -3190,6 +3963,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3190
3963
  return;
3191
3964
  }
3192
3965
  target.latestPartialText = event.text;
3966
+ this.capturePartialLanguages(target, event.languages);
3193
3967
  this.markFirstPartial(target);
3194
3968
  await this.sendFrame({ type: "stt_partial", text: event.text });
3195
3969
  return;
@@ -3205,7 +3979,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3205
3979
  // newer cycle.
3206
3980
  const owner = this.finalizeQueue[0];
3207
3981
  if (owner && !owner.assistantTurnStarted && !owner.completed) {
3208
- await this.recordFinalTranscript(owner, event.text);
3982
+ await this.recordFinalTranscript(
3983
+ owner,
3984
+ event.text,
3985
+ event.languages,
3986
+ );
3209
3987
  } else {
3210
3988
  log.warn(
3211
3989
  "Dropping a late finalize flush segment: its assistant turn already dispatched",
@@ -3222,7 +4000,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3222
4000
  );
3223
4001
  return;
3224
4002
  }
3225
- await this.recordFinalTranscript(target, event.text);
4003
+ await this.recordFinalTranscript(target, event.text, event.languages);
3226
4004
  return;
3227
4005
  }
3228
4006
  case "finalized": {
@@ -3246,6 +4024,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3246
4024
  await this.startAssistantTurnIfReady();
3247
4025
  return;
3248
4026
  }
4027
+ case "turn-start": {
4028
+ // Barge-in is deliberately untouched: local VAD still owns it,
4029
+ // because a provider roundtrip cannot beat a local energy gate on an
4030
+ // interrupt during playback (see DEFAULT_BARGE_IN_MIN_SPEECH_MS). The
4031
+ // index is recorded so a later end-of-turn can be told apart from one
4032
+ // this turn superseded (see isStaleProviderTurnEnd).
4033
+ const target = this.pendingTranscriptCycle();
4034
+ if (target) {
4035
+ this.recordProviderTurnStart(target, event.turnIndex);
4036
+ }
4037
+ return;
4038
+ }
4039
+ case "eager-turn-end":
4040
+ case "turn-resumed":
4041
+ // Speculative end-of-turn stays off (the config leaves
4042
+ // `eagerEotThreshold` unset, so Deepgram never emits these). Wiring
4043
+ // them onto the speculative dispatch machinery is a deferred
4044
+ // follow-up.
4045
+ return;
4046
+ case "turn-end": {
4047
+ // Same routing as transcript events: the cycle awaiting its
4048
+ // transcript owns the turn the provider just closed.
4049
+ const target = this.pendingTranscriptCycle();
4050
+ if (target) {
4051
+ await this.handleProviderTurnEnd(target, event.turnIndex);
4052
+ }
4053
+ return;
4054
+ }
3249
4055
  case "error":
3250
4056
  await this.sendTranscriberErrorFrame(event);
3251
4057
  return;
@@ -3285,16 +4091,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3285
4091
  await this.startAssistantTurnIfReady();
3286
4092
  return;
3287
4093
  }
4094
+ default: {
4095
+ const _exhaustive: never = event;
4096
+ return;
4097
+ }
3288
4098
  }
3289
4099
  }
3290
4100
 
3291
4101
  private async recordFinalTranscript(
3292
4102
  utterance: UtteranceCycle,
3293
4103
  text: string,
4104
+ languages?: readonly string[],
3294
4105
  ): Promise<void> {
3295
4106
  const transcript = text.trim();
3296
4107
  if (transcript.length > 0) {
3297
4108
  utterance.finalTranscriptSegments.push(transcript);
4109
+ // Tally only finals that committed transcript: empty silence frames
4110
+ // can still carry container-level language tags describing no emitted
4111
+ // words, and counting those would let silence outvote real speech
4112
+ // (same choice as the adapter's boundary-final aggregation).
4113
+ voteDominantLanguage(utterance.languageTally, languages);
3298
4114
  }
3299
4115
  // The final commits (and supersedes) whatever partial was trailing it.
3300
4116
  utterance.latestPartialText = null;
@@ -3333,6 +4149,59 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3333
4149
  await this.startAssistantTurnIfReady();
3334
4150
  }
3335
4151
 
4152
+ // Record a partial event's detected languages so speculative dispatch
4153
+ // has a detection before the first tagged final. The event contract
4154
+ // (stt/types.ts) guarantees the tags arrive as normalized base subtags
4155
+ // in dominance order, so they are stored as-is. Partials revise each
4156
+ // other, so this overwrites rather than tallies, and a tag-less partial
4157
+ // keeps the previous value.
4158
+ private capturePartialLanguages(
4159
+ utterance: UtteranceCycle,
4160
+ languages: readonly string[] | undefined,
4161
+ ): void {
4162
+ if (!languages || languages.length === 0) {
4163
+ return;
4164
+ }
4165
+ utterance.latestPartialLanguages = languages;
4166
+ }
4167
+
4168
+ /**
4169
+ * The caller's spoken language for a turn on this utterance, as a
4170
+ * lowercase base subtag: the dominant tallied STT-detected language
4171
+ * (most final-event counts, ties by first appearance), else the latest
4172
+ * tagged partial's dominant language (speculative turns dispatch from
4173
+ * partials), else a monolingual `services.stt.language` pin (a pinned
4174
+ * language IS the spoken language), else undefined ("multi" with no tags,
4175
+ * non-tagging providers, silence).
4176
+ */
4177
+ private turnLanguageFor(utterance: UtteranceCycle): string | undefined {
4178
+ const dominant = dominantLanguageTag(utterance.languageTally);
4179
+ if (dominant !== undefined) {
4180
+ return dominant;
4181
+ }
4182
+ // No tagged final yet (speculative turns dispatch from partials): the
4183
+ // latest tagged partial is the best detection available and outranks a
4184
+ // static pin for the same reason the tally does.
4185
+ const partialDominant = utterance.latestPartialLanguages?.[0];
4186
+ if (partialDominant !== undefined) {
4187
+ return partialDominant;
4188
+ }
4189
+ // A persisted pin only counts when the provider that actually
4190
+ // transcribed honors manual language selection (the shared
4191
+ // pinnedListeningLanguage gate). The DIALED transcriber's providerId
4192
+ // is authoritative, because the resolver silently falls back to
4193
+ // managed vellum (which honors the pin) when a BYOK provider has no
4194
+ // credential; the configured provider is only the last resort when no
4195
+ // transcriber reference survives.
4196
+ const { language: configured, provider: sttProvider } =
4197
+ getConfig().services.stt;
4198
+ const dialedProvider =
4199
+ utterance.dialedSttProvider ??
4200
+ this.sharedTranscriber?.providerId ??
4201
+ (sttProvider as SttProviderId);
4202
+ return pinnedListeningLanguage(dialedProvider, configured);
4203
+ }
4204
+
3336
4205
  // Providers emit `error` mid-stream and may keep streaming; `closed` /
3337
4206
  // `final` still drive turn lifecycle. Only transient categories are
3338
4207
  // recoverable — auth/rate-limit/invalid-audio will not self-heal, so
@@ -3361,8 +4230,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3361
4230
  this.assistantPlaybackTailUntilMs = 0;
3362
4231
  this.takeVadPreRoll();
3363
4232
  this.vadPendingTurnEnd = null;
3364
- // ...and abandons any semantic-endpointing hold still awaiting replay.
4233
+ // ...and abandons any semantic-endpointing hold still awaiting replay,
4234
+ // along with any provider end-of-turn the session was still waiting on.
3365
4235
  this.clearEndpointExtensionTimer();
4236
+ this.clearProviderTurnEndTimer();
3366
4237
  // A client interrupt is a hard reset: any barge-in merge context waiting for
3367
4238
  // the next turn is now stale (the interrupted utterance may be discarded
3368
4239
  // without ever reaching finalizePendingUtterance).
@@ -3588,6 +4459,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3588
4459
  token,
3589
4460
  turnId,
3590
4461
  utterance,
4462
+ language: this.turnLanguageFor(utterance),
3591
4463
  abortController,
3592
4464
  handle: null,
3593
4465
  launchedAtMs: Date.now(),
@@ -3628,12 +4500,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3628
4500
  consumedAnnouncement: pending?.announcement ?? null,
3629
4501
  pendingContextStopGeneration: this.detachStopGeneration,
3630
4502
  continuationDelivery: opts?.continuationDelivery ?? null,
3631
- toolUseStarted: false,
3632
- firstDeltaSeen: false,
3633
4503
  deltaEpoch: 0,
3634
- ackSpoken: false,
3635
- ackGenerationPending: false,
3636
- ackTimer: null,
3637
4504
  escalationHandedOff: false,
3638
4505
  ttsBuffer: "",
3639
4506
  ttsSegmentEnqueued: false,
@@ -3658,8 +4525,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3658
4525
  // Verdict-deadline fail-open: the deferred-everything window is only
3659
4526
  // safe while the verdict is fast. If the leg produces no verdict
3660
4527
  // within the endpoint budget (provider TTFT tail), commit anyway so
3661
- // the thinking frame shows and the ack timer arms — bounded dead air
3662
- // instead of unbounded structural silence. A verdict that arrives
4528
+ // the thinking frame shows and progress narration can begin, bounding
4529
+ // the structural silence. A verdict that arrives
3663
4530
  // after the commit still works: escalate hands off normally, and a
3664
4531
  // late hold token is stripped like any stray token (the utterance is
3665
4532
  // already released, so holding is no longer possible).
@@ -3682,22 +4549,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3682
4549
  }, this.frontModelConfig.endpointDecisionTimeoutMs);
3683
4550
  }
3684
4551
 
3685
- // Slow-first-delta spoken ack: if the model produces no delta within the
3686
- // budget, a short front-model-phrased sentence holds the floor. Only
3687
- // meaningful on TTS sessions — a text-only session has no audio gap to
3688
- // bridge.
3689
- if (this.streamTtsAudio && !opts?.speculative) {
3690
- this.armAckTimer(activeTurn);
3691
- }
3692
-
3693
4552
  // Progress narration speaks into the turn's audible dead air on a
3694
- // cadence, wherever in the turn it occurs; without TTS or a decider there
4553
+ // cadence, wherever in the turn it occurs; without TTS or a narrator there
3695
4554
  // is nothing to speak (the idle trigger's static fallback still needs a
3696
4555
  // generation attempt to fall back from).
3697
4556
  if (
3698
4557
  this.frontModelConfig.progress.enabled &&
3699
4558
  this.streamTtsAudio &&
3700
- this.frontDecider &&
4559
+ this.progressNarrator &&
3701
4560
  !opts?.speculative
3702
4561
  ) {
3703
4562
  this.armProgressIdleTimer(activeTurn);
@@ -3770,7 +4629,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3770
4629
  return;
3771
4630
  }
3772
4631
  this.markFirstAssistantDelta(utterance, turnId);
3773
- this.markFirstDeltaForAck(activeTurn);
4632
+ this.markAssistantDelta(activeTurn);
3774
4633
  // Send-time abort gate: a delta queued behind a backed-up outbound
3775
4634
  // frame must not be written once barge-in aborts the turn, or the
3776
4635
  // cancelled reply's text leaks ahead of turn_cancelled. Key off this
@@ -3807,8 +4666,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3807
4666
  voiceSessionId: this.context.sessionId,
3808
4667
  userMessageChannel: "vellum",
3809
4668
  assistantMessageChannel: "vellum",
4669
+ // Fixed, and NOT the originating client: this pair resolves the turn's
4670
+ // channel capabilities, where `macos` is what grants a live-voice turn
4671
+ // desktop UI and dynamic surfaces. Reporting the true client here would
4672
+ // strip `supportsDynamicUi` from every iOS session: a behavior change
4673
+ // wearing an attribution fix's clothes. The originating client travels
4674
+ // as telemetry instead, on `voiceTelemetry` below.
3810
4675
  userMessageInterface: "macos",
3811
4676
  assistantMessageInterface: "macos",
4677
+ voiceTelemetry: {
4678
+ sessionId: this.context.sessionId,
4679
+ ...(this.context.startFrame.client
4680
+ ? { client: this.context.startFrame.client }
4681
+ : {}),
4682
+ },
3812
4683
  voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
3813
4684
  ...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
3814
4685
  }),
@@ -3996,7 +4867,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3996
4867
  if (current?.token !== token) {
3997
4868
  return;
3998
4869
  }
3999
- current.toolUseStarted = true;
4000
4870
  // Foreground wins the workspace: the continuation runs with full
4001
4871
  // subagent abilities (it can write files, run commands), so the
4002
4872
  // moment a live turn starts a consequential tool the two could
@@ -4045,21 +4915,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4045
4915
  current.progress.stateEpoch += 1;
4046
4916
  this.refreshActivity(current);
4047
4917
  log.debug({ turnId, toolName }, "Live voice turn started tool use");
4048
- // Definitive tool use means the turn is guaranteed slow: speak
4049
- // the floor-holding ack now instead of waiting out the
4050
- // first-delta timer (same one-ack-per-turn `ackSpoken` budget).
4051
- // Post-handoff the escalation bridge phrase holds the floor and
4052
- // front-door deltas never set firstDeltaSeen, so the escalated
4053
- // leg's tool starts must not stack another filler on the bridge.
4054
- if (
4055
- this.streamTtsAudio &&
4056
- !current.firstDeltaSeen &&
4057
- !current.ackSpoken &&
4058
- !current.escalationHandedOff
4059
- ) {
4060
- this.clearAckTimer(current);
4061
- this.speakAck(current, "tool_use", toolName);
4062
- }
4063
4918
  this.maybeNarrateProgress(current, "ops");
4064
4919
  },
4065
4920
  tool_result: (event) => {
@@ -4213,9 +5068,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4213
5068
  return;
4214
5069
  }
4215
5070
  activeTurn.escalationHandedOff = true;
4216
- // The escalation bridge below holds the floor, so a pending
4217
- // slow-first-delta ack or progress narration would only stack a second
4218
- // filler on top of it.
5071
+ // The escalation bridge below holds the floor, so pending progress
5072
+ // narration would only stack a second filler on top of it.
4219
5073
  this.clearFillerTimers(activeTurn);
4220
5074
  // Abort the front-door leg so a model that keeps generating past the
4221
5075
  // bridge cap adds no latency before the escalated leg starts.
@@ -4228,21 +5082,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4228
5082
  // deleted row for a bridge the model never produced).
4229
5083
  const usesFallbackBridge = cappedBridge.length < MIN_SPOKEN_BRIDGE_CHARS;
4230
5084
  const spokenBridge = usesFallbackBridge
4231
- ? FALLBACK_ESCALATION_BRIDGE
5085
+ ? fallbackEscalationBridgeFor(activeTurn.language)
4232
5086
  : cappedBridge;
4233
5087
  if (!usesFallbackBridge) {
4234
5088
  this.markFirstAssistantDelta(activeTurn.utterance, activeTurn.turnId);
4235
- this.markFirstDeltaForAck(activeTurn);
5089
+ this.markAssistantDelta(activeTurn);
4236
5090
  void this.sendFrame(
4237
5091
  { type: "assistant_text_delta", text: spokenBridge },
4238
5092
  () => !activeTurn.abortController.signal.aborted && !this.isClosed,
4239
5093
  );
5094
+ this.bufferAssistantTextForTts(activeTurn.token, `${spokenBridge} `);
5095
+ // Force-flush now: on the TTS path an unpunctuated bridge would
5096
+ // otherwise sit buffered until a sentence boundary and leave the
5097
+ // caller in silence during the escalated model's call.
5098
+ this.flushTtsBuffer(activeTurn.token, true);
5099
+ } else {
5100
+ // The canned bridge is a fixed localized-table phrase, enqueued
5101
+ // directly (it is already one complete sentence) so the segment can
5102
+ // carry the "en" override when the table lacks the turn's language.
5103
+ const speakable = sanitizeForTts(spokenBridge).trim();
5104
+ if (speakable.length > 0) {
5105
+ this.enqueueTtsSegment(activeTurn.token, speakable, {
5106
+ language: this.fixedPhraseLanguage(
5107
+ activeTurn,
5108
+ FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
5109
+ ),
5110
+ });
5111
+ }
4240
5112
  }
4241
- this.bufferAssistantTextForTts(activeTurn.token, `${spokenBridge} `);
4242
- // Force-flush now: on the TTS path an unpunctuated bridge would otherwise
4243
- // sit buffered until a sentence boundary and leave the caller in silence
4244
- // during the escalated model's call.
4245
- this.flushTtsBuffer(activeTurn.token, true);
4246
5113
 
4247
5114
  // No overrideProfile: the escalated leg runs on the call-site default —
4248
5115
  // the exact profile an un-routed voice turn would use (see
@@ -4265,7 +5132,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4265
5132
  if (
4266
5133
  this.frontModelConfig.progress.enabled &&
4267
5134
  this.streamTtsAudio &&
4268
- this.frontDecider
5135
+ this.progressNarrator
4269
5136
  ) {
4270
5137
  this.armProgressIdleTimer(activeTurn);
4271
5138
  }
@@ -4337,42 +5204,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4337
5204
  );
4338
5205
  }
4339
5206
 
4340
- // Arms the slow-first-delta ack timer for a just-launched turn: on expiry,
4341
- // if no assistant delta has arrived and the turn is still live, one short
4342
- // phrase is spoken to hold the floor.
4343
- private armAckTimer(activeTurn: ActiveAssistantTurn): void {
4344
- const { token } = activeTurn;
4345
- activeTurn.ackTimer = setTimeout(() => {
4346
- activeTurn.ackTimer = null;
4347
- if (
4348
- !this.isActiveAssistantTurn(token) ||
4349
- activeTurn.firstDeltaSeen ||
4350
- activeTurn.ackSpoken
4351
- ) {
4352
- return;
4353
- }
4354
- this.speakAck(activeTurn, "first_delta");
4355
- }, this.frontModelConfig.ackFirstDeltaTimeoutMs);
4356
- }
4357
-
4358
- private clearAckTimer(turn: ActiveAssistantTurn): void {
4359
- if (turn.ackTimer !== null) {
4360
- clearTimeout(turn.ackTimer);
4361
- turn.ackTimer = null;
4362
- }
4363
- }
4364
-
4365
- // The model produced real output for this turn: the pending slow-first-delta
4366
- // ack is moot, and any in-flight narration generation is stale (it captures
4367
- // deltaEpoch to notice). The narration idle timer stays armed — the dead air
4368
- // narration exists to fill is almost always MID-turn (between tool phases,
4369
- // during long silent LLM calls after the model's opening words), so it must
4370
- // keep watching after the model speaks and re-trigger once the audio drains
4371
- // and silence stretches again.
4372
- private markFirstDeltaForAck(turn: ActiveAssistantTurn): void {
4373
- turn.firstDeltaSeen = true;
5207
+ // A real assistant delta makes any in-flight progress narration stale. The
5208
+ // idle timer stays armed so it can detect a later stretch of mid-turn
5209
+ // silence.
5210
+ private markAssistantDelta(turn: ActiveAssistantTurn): void {
4374
5211
  turn.deltaEpoch += 1;
4375
- this.clearAckTimer(turn);
4376
5212
  }
4377
5213
 
4378
5214
  // Arms (or re-arms) the dead-air narration timer. The countdown measures
@@ -4448,14 +5284,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4448
5284
  }
4449
5285
  }
4450
5286
 
4451
- // Clears the turn's filler timers (slow-first-delta ack, dead-air
4452
- // narration, verdict deadline) for events that end the current filler
4453
- // lifecycle: escalation hand-off (which then re-arms narration for the
4454
- // escalated leg), barge-in, cancellation, tts-completion, finalize. Real
4455
- // output moots only the ack (markFirstDeltaForAck) — the narration timer
4456
- // keeps watching for mid-turn dead air.
5287
+ // Clears progress narration and verdict timers for events that end the
5288
+ // current filler lifecycle.
4457
5289
  private clearFillerTimers(turn: ActiveAssistantTurn): void {
4458
- this.clearAckTimer(turn);
4459
5290
  this.clearProgressIdleTimer(turn);
4460
5291
  if (turn.verdictDeadlineTimer !== null) {
4461
5292
  clearTimeout(turn.verdictDeadlineTimer);
@@ -4463,26 +5294,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4463
5294
  }
4464
5295
  }
4465
5296
 
4466
- // A spoken ack may only speak into a live turn's pre-first-delta dead air.
4467
- // Once the turn is stale, the brain's first delta has arrived, an
4468
- // escalation hand-off has enqueued its bridge phrase (which holds the floor
4469
- // itself), or the turn has completed outright — a tool-only/no-text turn
4470
- // completes with firstDeltaSeen still false — an ack is moot and must not
4471
- // speak over real output, stack on the escalation bridge, or voice a
4472
- // finished turn.
4473
- private turnCanSpeakFiller(turn: ActiveAssistantTurn): boolean {
4474
- return (
4475
- this.isActiveAssistantTurn(turn.token) &&
4476
- !turn.firstDeltaSeen &&
4477
- !turn.escalationHandedOff &&
4478
- !turn.assistantCompleted
4479
- );
4480
- }
4481
-
4482
- // Narration, unlike the ack, is not confined to the pre-first-delta window:
4483
- // the dead air it exists to fill is almost always mid-turn, after the
4484
- // model's opening words. It may speak whenever the live turn is audibly
4485
- // silent right now — nothing streaming, queued, or still playing — and has
5297
+ // Narration may speak whenever the live turn is audibly silent and has
4486
5298
  // not completed. Escalated turns qualify too: the bridge phrase holds the
4487
5299
  // floor only while its audio plays (covered by the audible-idle check),
4488
5300
  // and the strong leg's tool loops are the longest dead air in the system.
@@ -4519,31 +5331,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4519
5331
  return false;
4520
5332
  }
4521
5333
 
4522
- // Gatekeeper for spoken progress narration: it speaks only while the turn
4523
- // is audibly silent, spaced `minGapMs` from any spoken floor-holder (ack or
4524
- // narration), one generation at a time, and — per trigger — only once the
4525
- // ops trigger has `opsThreshold` ops accumulated or the idle trigger has
4526
- // something new to report. No per-turn count cap: the cadence guards bound
4527
- // the rate, and going quiet deep into a long turn is the failure mode
4528
- // narration exists to prevent. Every failing guard short-circuits silently;
4529
- // a skipped ops trigger keeps its accumulated count, so the next tool event
4530
- // or idle tick retries.
5334
+ // Gatekeeper for progress narration. It speaks only while the turn is
5335
+ // audibly silent, with one generation at a time and the configured spacing.
4531
5336
  private maybeNarrateProgress(
4532
5337
  turn: ActiveAssistantTurn,
4533
5338
  trigger: "ops" | "idle" | "op_complete",
4534
5339
  ): void {
4535
5340
  const cfg = this.frontModelConfig.progress;
4536
5341
  const { progress } = turn;
4537
- const frontDecider = this.frontDecider;
5342
+ const progressNarrator = this.progressNarrator;
4538
5343
  if (
4539
5344
  !cfg.enabled ||
4540
5345
  !this.streamTtsAudio ||
4541
- !frontDecider ||
5346
+ !progressNarrator ||
4542
5347
  !this.turnCanNarrateProgress(turn) ||
4543
- // A pending ack generation is a floor-holder-in-waiting: starting a
4544
- // narration generation now would only be discarded by the post-await
4545
- // re-check once the ack enqueues — a guaranteed wasted provider call.
4546
- turn.ackGenerationPending ||
4547
5348
  progress.narrationInFlight ||
4548
5349
  (progress.lastFloorHolderAtMs !== null &&
4549
5350
  Date.now() - progress.lastFloorHolderAtMs < cfg.minGapMs) ||
@@ -4552,25 +5353,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4552
5353
  ) {
4553
5354
  return;
4554
5355
  }
4555
- void this.speakProgressUpdate(turn, frontDecider, trigger);
5356
+ void this.speakProgressUpdate(turn, progressNarrator, trigger);
4556
5357
  }
4557
5358
 
4558
- // Generate and speak one progress narration. Like the ack, it is audio-only
4559
- // — no assistant_text_delta frame — and enqueues as a standalone sentence
4560
- // on the turn's ordered TTS queue. The decider internally bounds the call
4561
- // by `progress.generationTimeoutMs` and resolves null on every failure
4562
- // mode; on null the idle trigger falls back to a static phrase (silence is
4563
- // actively harmful there) while the tool-activity triggers stay silent — a
4564
- // generic filler is not worth it when narration was merely opportunistic.
5359
+ // Generate and speak one audio-only progress narration. On a null result,
5360
+ // idle narration falls back to a static phrase while activity-triggered
5361
+ // narration stays silent.
4565
5362
  private async speakProgressUpdate(
4566
5363
  turn: ActiveAssistantTurn,
4567
- frontDecider: VoiceFrontDecider,
5364
+ progressNarrator: VoiceProgressNarrator,
4568
5365
  trigger: "ops" | "idle" | "op_complete",
4569
5366
  ): Promise<void> {
4570
5367
  const { progress } = turn;
4571
5368
  progress.narrationInFlight = true;
4572
- // Any delta that lands while the decider call is in flight makes the
4573
- // generated text stale — and proves the model is speaking again.
5369
+ // Any delta that lands while generation is in flight makes the text stale.
4574
5370
  const deltaEpochAtLaunch = turn.deltaEpoch;
4575
5371
  // The activity this update describes. Tool events that land mid-generation
4576
5372
  // are news the generated text cannot carry, so they must leave the idle
@@ -4588,7 +5384,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4588
5384
  (op): op is TurnProgressOp & { completedAtMs: number } =>
4589
5385
  op.completedAtMs !== undefined,
4590
5386
  )
4591
- // `ops` is in start order; the decider contract wants completion
5387
+ // `ops` is in start order; the narrator wants completion
4592
5388
  // order, which differs when parallel tools finish out of order.
4593
5389
  .sort((a, b) => a.completedAtMs - b.completedAtMs)
4594
5390
  .map((op) => ({
@@ -4606,27 +5402,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4606
5402
  : null,
4607
5403
  turnElapsedMs: now - turn.launchedAtMs,
4608
5404
  updateIndex: progress.updatesSpoken + 1,
5405
+ ...(turn.language !== undefined ? { languageHint: turn.language } : {}),
4609
5406
  };
4610
- const generated = await frontDecider
5407
+ const generated = await progressNarrator
4611
5408
  .generateProgressText(input, turn.abortController.signal)
4612
- // The decider contract never rejects; belt-and-braces for a stub.
5409
+ // The narrator contract never rejects. Keep test stubs fail-soft too.
4613
5410
  .catch(() => null);
4614
- // Liveness re-check after the await: any turnCanNarrateProgress
4615
- // condition may have flipped while generating, and a delta that landed
4616
- // mid-generation (deltaEpoch moved) makes the text stale even when the
4617
- // resulting audio has already drained by now. The ack-generation and
4618
- // floor re-checks close the race with a generated ack that
4619
- // STARTED after this generation did — the entry guard rejects a
4620
- // narration while an ack generation is already pending, but cannot see
4621
- // one that begins mid-generation: while that ack is pending it has not
4622
- // yet stamped `lastFloorHolderAtMs`, and once it enqueues the minGapMs
4623
- // spacing must be re-applied from that stamp — either way, enqueueing
4624
- // now would stack back-to-back filler. A bail here is silent and keeps
4625
- // the update budget, exactly like the stale-turn bail.
5411
+ // Re-check liveness and staleness after the provider call.
4626
5412
  if (
4627
5413
  !this.turnCanNarrateProgress(turn) ||
4628
5414
  turn.deltaEpoch !== deltaEpochAtLaunch ||
4629
- turn.ackGenerationPending ||
4630
5415
  (progress.lastFloorHolderAtMs !== null &&
4631
5416
  Date.now() - progress.lastFloorHolderAtMs <
4632
5417
  this.frontModelConfig.progress.minGapMs)
@@ -4634,20 +5419,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4634
5419
  return;
4635
5420
  }
4636
5421
  let raw = generated;
5422
+ // Generated text is in the turn's language; only the static
5423
+ // fallback comes from a localized table and may need the "en" override.
5424
+ let fillerLanguage: string | undefined;
4637
5425
  if (raw === null) {
4638
5426
  if (trigger !== "idle") {
4639
5427
  return;
4640
5428
  }
4641
- raw = pickProgressPhrase(this.progressPhraseCounter++);
5429
+ raw = pickProgressPhrase(this.progressPhraseCounter++, turn.language);
5430
+ fillerLanguage = this.fixedPhraseLanguage(
5431
+ turn,
5432
+ PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
5433
+ );
4642
5434
  }
4643
- if (!this.enqueueFillerPhrase(turn, raw)) {
5435
+ if (!this.enqueueFillerPhrase(turn, raw, fillerLanguage)) {
4644
5436
  return;
4645
5437
  }
4646
5438
  progress.opsSinceNarration = 0;
4647
5439
  progress.narratedEpoch = stateEpochAtLaunch;
4648
5440
  progress.updatesSpoken += 1;
4649
- // Like the ack mark, recorded only when narration audio actually
4650
- // enqueued (decider text or static fallback alike).
5441
+ // Record only when narration audio actually enqueued.
4651
5442
  this.markProgressSpoken(turn);
4652
5443
  // Restart the dead-air countdown from this narration.
4653
5444
  this.armProgressIdleTimer(turn);
@@ -4656,101 +5447,40 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4656
5447
  }
4657
5448
  }
4658
5449
 
4659
- // Speak the turn's one floor-holding ack (every ack trigger shares the
4660
- // `ackSpoken` budget). The ack is audio-only — no assistant_text_delta frame
4661
- // — so captions and the persisted assistant message carry only the model's
4662
- // own output. It enqueues as a complete standalone sentence on the turn's
4663
- // ordered TTS queue, so the model's audio always follows it cleanly; it
4664
- // never consumes the eager first-clause flush reserved for the model's real
4665
- // first segment.
4666
- private speakAck(
4667
- activeTurn: ActiveAssistantTurn,
4668
- kind: LiveVoiceSpokenAckKind,
4669
- toolName?: string,
4670
- ): void {
4671
- if (!this.frontDecider) {
4672
- // No decider, no ack: a canned line is worse than the silence, which
4673
- // progress narration already covers once the turn runs long.
4674
- return;
4675
- }
4676
- activeTurn.ackSpoken = true;
4677
- // Optimistic ackSpoken above keeps the one-per-turn budget honest for
4678
- // any other ack trigger while the generation is in flight; the async
4679
- // path undoes it if the ack turns out moot.
4680
- void this.speakGeneratedAck(activeTurn, this.frontDecider, kind, toolName);
4681
- }
4682
-
4683
- // Ask the front model for one short contextual sentence. The decider
4684
- // internally bounds the call by `ackGenerationTimeoutMs` and resolves null
4685
- // on every failure mode, so the enqueue never waits beyond that budget; a
4686
- // null generation speaks nothing.
4687
- private async speakGeneratedAck(
4688
- activeTurn: ActiveAssistantTurn,
4689
- frontDecider: VoiceFrontDecider,
4690
- kind: LiveVoiceSpokenAckKind,
4691
- toolName?: string,
4692
- ): Promise<void> {
4693
- activeTurn.ackGenerationPending = true;
4694
- try {
4695
- const transcript = activeTurn.utterance.finalTranscriptSegments
4696
- .join(" ")
4697
- .trim();
4698
- const generated = await frontDecider
4699
- .generateAckText(
4700
- {
4701
- transcriptSoFar: transcript,
4702
- toolName,
4703
- },
4704
- activeTurn.abortController.signal,
4705
- )
4706
- // The decider contract never rejects; belt-and-braces for a stub.
4707
- .catch(() => null);
4708
- // Liveness re-check after the await: any turnCanSpeakFiller condition
4709
- // may have flipped while generating, mooting the ack. A null generation
4710
- // (no provider, timeout, unusable output) speaks nothing. Either way,
4711
- // undo the optimistic ackSpoken so the flag reflects reality and a
4712
- // later trigger this turn can still hold the floor.
4713
- if (generated === null || !this.turnCanSpeakFiller(activeTurn)) {
4714
- activeTurn.ackSpoken = false;
4715
- return;
4716
- }
4717
- this.enqueueAckPhrase(activeTurn, generated, kind);
4718
- } finally {
4719
- activeTurn.ackGenerationPending = false;
4720
- }
4721
- }
4722
-
4723
- // Records the ack-spoken metric only when a phrase actually enqueues; a
4724
- // phrase that sanitizes away releases the turn's ack budget again.
4725
- private enqueueAckPhrase(
4726
- activeTurn: ActiveAssistantTurn,
5450
+ // Sanitize and enqueue one progress sentence on the ordered TTS queue.
5451
+ private enqueueFillerPhrase(
5452
+ turn: ActiveAssistantTurn,
4727
5453
  raw: string,
4728
- kind: LiveVoiceSpokenAckKind,
4729
- ): void {
4730
- if (this.enqueueFillerPhrase(activeTurn, raw)) {
4731
- this.markAckSpoken(activeTurn, kind);
4732
- return;
4733
- }
4734
- activeTurn.ackSpoken = false;
4735
- }
4736
-
4737
- // Sanitize and enqueue one filler sentence (spoken ack or progress
4738
- // narration) on the turn's ordered TTS queue — the shared tail of every
4739
- // filler path. Returns whether a phrase actually enqueued; per-kind metric
4740
- // marks and bookkeeping are the caller's.
4741
- private enqueueFillerPhrase(turn: ActiveAssistantTurn, raw: string): boolean {
5454
+ language?: string,
5455
+ ): boolean {
4742
5456
  const phrase = sanitizeForTts(raw).trim();
4743
5457
  if (phrase.length === 0) {
4744
5458
  return false;
4745
5459
  }
4746
5460
  this.enqueueTtsSegment(turn.token, phrase, {
4747
5461
  countsAsFirstSegment: false,
5462
+ ...(language !== undefined ? { language } : {}),
4748
5463
  });
4749
5464
  // A spoken filler holds the floor, so narration's minGapMs spaces from it.
4750
5465
  turn.progress.lastFloorHolderAtMs = Date.now();
4751
5466
  return true;
4752
5467
  }
4753
5468
 
5469
+ // The TTS hint override for a fixed phrase picked from a localized table:
5470
+ // "en" when the turn has a language the table does not cover (the picker
5471
+ // fell back to English text, which must not be synthesized under an
5472
+ // ar/ko/ta hint), undefined otherwise (the segment rides the turn's
5473
+ // language, or no hint at all when the language is unknown).
5474
+ private fixedPhraseLanguage(
5475
+ turn: ActiveAssistantTurn,
5476
+ table: Readonly<Record<string, unknown>>,
5477
+ ): string | undefined {
5478
+ return turn.language !== undefined &&
5479
+ !hasLocalizedEntry(table, turn.language)
5480
+ ? "en"
5481
+ : undefined;
5482
+ }
5483
+
4754
5484
  private bufferAssistantTextForTts(token: symbol, text: string): void {
4755
5485
  if (!this.streamTtsAudio || text.length === 0) {
4756
5486
  return;
@@ -4871,20 +5601,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4871
5601
  private enqueueTtsSegment(
4872
5602
  token: symbol,
4873
5603
  segment: string,
4874
- options: { countsAsFirstSegment?: boolean } = {},
5604
+ options: { countsAsFirstSegment?: boolean; language?: string } = {},
4875
5605
  ): void {
4876
5606
  const activeTurn = this.activeAssistantTurn;
4877
5607
  if (activeTurn?.token !== token || !this.streamTtsAudio) {
4878
5608
  return;
4879
5609
  }
4880
5610
 
4881
- // A spoken ack (countsAsFirstSegment: false) leaves the eager
4882
- // first-clause flush available for the model's real first segment.
5611
+ // Progress narration leaves the eager first-clause flush available for
5612
+ // the model's real first segment.
4883
5613
  if (options.countsAsFirstSegment ?? true) {
4884
5614
  activeTurn.ttsSegmentEnqueued = true;
4885
5615
  }
4886
5616
  const job: TtsSegmentJob = {
4887
5617
  text: segment,
5618
+ language: options.language,
4888
5619
  started: false,
4889
5620
  settled: false,
4890
5621
  emitting: false,
@@ -4924,10 +5655,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4924
5655
  return;
4925
5656
  }
4926
5657
  job.started = true;
5658
+ // The segment's own language override (fixed English fallback text)
5659
+ // wins over the turn's language.
5660
+ const language = job.language ?? activeTurn.language;
4927
5661
  let synthesis: Promise<void>;
4928
5662
  try {
4929
5663
  synthesis = streamTtsAudio({
4930
5664
  text: job.text,
5665
+ ...(language !== undefined ? { language } : {}),
4931
5666
  signal: activeTurn.abortController.signal,
4932
5667
  outputFormat: "pcm",
4933
5668
  sampleRate: this.context.startFrame.audio.sampleRate,
@@ -5067,6 +5802,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5067
5802
  chunk.sampleRate,
5068
5803
  );
5069
5804
  const now = Date.now();
5805
+ if (!this.isAssistantPlaybackEchoPossible()) {
5806
+ this.resetEchoReference();
5807
+ }
5808
+ this.appendEchoReference(chunk);
5070
5809
  this.assistantPlaybackTailUntilMs =
5071
5810
  Math.max(now, this.assistantPlaybackTailUntilMs) + chunkMs;
5072
5811
  const turnAfterSend = this.activeAssistantTurn;
@@ -5173,23 +5912,42 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5173
5912
  utterance: UtteranceCycle,
5174
5913
  action: VoiceEndpointAction,
5175
5914
  latencyMs: number,
5915
+ // Absent means the front door decided, which is what the collector
5916
+ // defaults to; the provider path names itself so one metrics frame can
5917
+ // compare the two.
5918
+ source?: VoiceEndpointSource,
5176
5919
  ): void {
5177
5920
  const turnId = this.ensureTurnId(utterance);
5178
5921
  if (!this.startMetricsTurnIfNeeded(utterance, turnId)) {
5179
5922
  return;
5180
5923
  }
5181
- this.metrics.markEndpointDecision(turnId, { action, latencyMs });
5924
+ this.metrics.markEndpointDecision(turnId, {
5925
+ action,
5926
+ latencyMs,
5927
+ ...(source ? { source } : {}),
5928
+ });
5182
5929
  }
5183
5930
 
5184
- private markAckSpoken(
5185
- activeTurn: ActiveAssistantTurn,
5186
- kind: LiveVoiceSpokenAckKind,
5187
- ): void {
5188
- const { utterance } = activeTurn;
5189
- if (!this.startMetricsTurnIfNeeded(utterance, activeTurn.turnId)) {
5931
+ /**
5932
+ * The turn's end-of-turn latency, stamped at the moment it commits. Called
5933
+ * from `releaseUtterance`, which every committed turn passes through
5934
+ * whichever decider released it, so the front-door and provider samples span
5935
+ * the same thing from the same anchor over the same population. That is
5936
+ * what `endpointDecisionMaxLatencyMs` cannot offer, and why this exists
5937
+ * alongside it.
5938
+ *
5939
+ * Skipped with no speech-stop mark, which is push-to-talk: there the
5940
+ * client's release is the boundary and no local VAD runs.
5941
+ */
5942
+ private markEndpointCommit(utterance: UtteranceCycle): void {
5943
+ if (this.localSpeechStopAtMs === null) {
5944
+ return;
5945
+ }
5946
+ const turnId = this.ensureTurnId(utterance);
5947
+ if (!this.startMetricsTurnIfNeeded(utterance, turnId)) {
5190
5948
  return;
5191
5949
  }
5192
- this.metrics.markAckSpoken(activeTurn.turnId, kind);
5950
+ this.metrics.markEndpointCommit(turnId, this.msSinceLocalSpeechStop());
5193
5951
  }
5194
5952
 
5195
5953
  private markProgressSpoken(activeTurn: ActiveAssistantTurn): void {
@@ -5506,6 +6264,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5506
6264
  // method, and a per-call-site hook would drift from it on the first frame
5507
6265
  // anyone added. Fire-and-forget by contract — see the reporter.
5508
6266
  this.liveActivityReporter.report(frame);
6267
+ // Latched here for the same reason: both fatal paths mark the session
6268
+ // `failed` and then send their error frame through this method, so the
6269
+ // code that ended the session is readable at close without either path
6270
+ // having to remember to stash it.
6271
+ if (frame.type === "error" && this.state === "failed") {
6272
+ this.failureCode ??= frame.code;
6273
+ }
5509
6274
  let sent = false;
5510
6275
  this.outboundFrames = this.outboundFrames
5511
6276
  .catch(() => {})
@@ -5538,14 +6303,16 @@ export function createLiveVoiceSession(
5538
6303
  options: LiveVoiceSessionOptions = {},
5539
6304
  ): LiveVoiceSession {
5540
6305
  // Workspace-tunable server-VAD thresholds. The `liveVoice.vad` schema
5541
- // defaults match the code defaults (800 energy / 800 ms silence / 30 s max
5542
- // turn / 60 ms barge-in guard), so an unset config leaves behavior
5543
- // unchanged. Optional-chained
5544
- // because hand-built test configs may predate the liveVoice namespace;
5545
- // absent config falls through to the in-code defaults.
6306
+ // defaults are 800 energy / 1200 ms silence / 30 s max turn / 250 ms
6307
+ // barge-in guard. Three of the four match their in-code fallback; the
6308
+ // trailing-silence threshold does not, so a session built with a
6309
+ // `liveVoice` config waits 1200 ms and one built without waits the in-code
6310
+ // DEFAULT_SILENCE_THRESHOLD_MS of 800. Optional-chained because hand-built
6311
+ // test configs may predate the liveVoice namespace; absent config falls
6312
+ // through to the in-code defaults.
5546
6313
  const liveVoiceConfig = getConfig().liveVoice;
5547
6314
  const vadConfig = liveVoiceConfig?.vad;
5548
- // Parsed once here into a complete config, shared by the decider and the
6315
+ // Parsed once here into a complete config, shared by endpointing and the
5549
6316
  // session (the constructor's own parse is then a no-op re-validation).
5550
6317
  const frontModelConfig = LiveVoiceFrontModelConfigSchema.parse(
5551
6318
  options.frontModelConfig ?? liveVoiceConfig?.frontModel ?? {},
@@ -5564,14 +6331,20 @@ export function createLiveVoiceSession(
5564
6331
  options.speechEnergyThreshold ?? vadConfig?.speechEnergyThreshold,
5565
6332
  bargeInMinSpeechMs:
5566
6333
  options.bargeInMinSpeechMs ?? vadConfig?.bargeInMinSpeechMs,
6334
+ echoBargeInMargin:
6335
+ options.echoBargeInMargin ?? vadConfig?.echoBargeInMargin,
6336
+ echoEmaHalfLifeMs:
6337
+ options.echoEmaHalfLifeMs ?? vadConfig?.echoEmaHalfLifeMs,
6338
+ echoDrainSlackMs: options.echoDrainSlackMs ?? vadConfig?.echoDrainSlackMs,
5567
6339
  frontModelConfig,
5568
- // Eager construction is safe even when the `liveVoice.frontModel` config
5569
- // namespace is absent — schema defaults fill the tunables. An explicit
5570
- // option (incl. `null`) always wins — the test seam.
5571
- frontDecider:
5572
- options.frontDecider !== undefined
5573
- ? options.frontDecider
5574
- : createVoiceFrontDecider({ config: frontModelConfig }),
6340
+ // Absent config leaves the schema defaults, which keep provider turn
6341
+ // detection off.
6342
+ fluxConfig: options.fluxConfig ?? liveVoiceConfig?.flux,
6343
+ // An explicit option, including null, always wins for the test seam.
6344
+ progressNarrator:
6345
+ options.progressNarrator !== undefined
6346
+ ? options.progressNarrator
6347
+ : createVoiceProgressNarrator({ config: frontModelConfig.progress }),
5575
6348
  resolveCredentialReadiness:
5576
6349
  options.resolveCredentialReadiness === undefined
5577
6350
  ? defaultResolveLiveVoiceCredentialReadiness
@@ -5675,6 +6448,12 @@ export async function defaultSpawnBackgroundContinuation(args: {
5675
6448
  objective: args.objective,
5676
6449
  fork: true,
5677
6450
  sendResultToUser: false,
6451
+ // Distinct spawn mode so this unattended continuation is separable in
6452
+ // cost telemetry from a tool-initiated fork. Mechanically both are
6453
+ // forks, but they come from different call sites with different cost
6454
+ // profiles, and lumping them together is what made delegated spend
6455
+ // opaque in the first place.
6456
+ spawnMode: "voice_continuation",
5678
6457
  // Full subagent abilities: the continuation runs like any other
5679
6458
  // background subagent, so it can genuinely finish build-shaped work
5680
6459
  // (JARVIS-1354). Side effects are governed by the standard
@@ -5735,7 +6514,14 @@ async function defaultStartVoiceTurn(
5735
6514
  // inside `startVoiceTurn` trips `FOREIGN KEY constraint failed`. Ensure it
5736
6515
  // exists (idempotent) before persisting. Lives in the production wiring, not
5737
6516
  // the session state machine, so session unit tests stay DB-free.
5738
- const createdConversation = ensureConversationExists(options.conversationId);
6517
+ // Native: this is the local live-voice session, which adopts a conversation
6518
+ // id the app supplied. Its trust context resolves through
6519
+ // `resolveLocalLiveVoiceTrustContext`, and a phone call reaches the assistant
6520
+ // through the telephony path rather than here.
6521
+ const createdConversation = ensureConversationExists(
6522
+ options.conversationId,
6523
+ "vellum",
6524
+ );
5739
6525
  if (createdConversation) {
5740
6526
  // The row was created outside the normal send-message route, which is where
5741
6527
  // sibling clients/sidebars learn about a new conversation. Emit the same