@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162214.308bda0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (859) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +5 -1
  3. package/ARCHITECTURE.md +20 -21
  4. package/Dockerfile +7 -7
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +11 -2
  7. package/docs/architecture/turn-actor.md +15 -8
  8. package/docs/credential-execution-service.md +1 -1
  9. package/docs/desktop-browser-cli.md +35 -0
  10. package/docs/guardian-request-flow.md +16 -0
  11. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  12. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  13. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  14. package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
  15. package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
  16. package/openapi.yaml +564 -111
  17. package/package.json +7 -4
  18. package/scripts/bundled-plugin-packages.ts +154 -0
  19. package/scripts/generate-bundled-plugin-packages.ts +29 -0
  20. package/scripts/postinstall.ts +33 -0
  21. package/scripts/smoke-desktop-browser-cli.ts +280 -0
  22. package/scripts/smoke-desktop-window-theme.ts +186 -0
  23. package/scripts/sync-llm-catalog.ts +3 -0
  24. package/scripts/test.ts +15 -13
  25. package/scripts/voice-ttft-spike.ts +2 -2
  26. package/src/__tests__/agent-loop.test.ts +124 -0
  27. package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
  28. package/src/__tests__/anthropic-provider.test.ts +0 -63
  29. package/src/__tests__/app-compiler.test.ts +6 -0
  30. package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
  31. package/src/__tests__/app-dir-path-guard.test.ts +1 -3
  32. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  33. package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
  34. package/src/__tests__/assistant-event-hub.test.ts +44 -0
  35. package/src/__tests__/attachments-store.test.ts +22 -3
  36. package/src/__tests__/channel-approval.test.ts +9 -14
  37. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  38. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  39. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  40. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
  41. package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
  42. package/src/__tests__/computer-use-tools.test.ts +19 -7
  43. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  44. package/src/__tests__/config-schema.test.ts +29 -30
  45. package/src/__tests__/config-sounds-sync.test.ts +23 -0
  46. package/src/__tests__/config-watcher.test.ts +23 -8
  47. package/src/__tests__/connection-model-compat.test.ts +2 -2
  48. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  49. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  50. package/src/__tests__/conversation-agent-loop.test.ts +257 -6
  51. package/src/__tests__/conversation-attachments.test.ts +106 -0
  52. package/src/__tests__/conversation-error.test.ts +4 -1
  53. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  54. package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
  55. package/src/__tests__/conversation-pairing.test.ts +26 -0
  56. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
  57. package/src/__tests__/conversation-queue.test.ts +48 -207
  58. package/src/__tests__/conversation-retry-discard.test.ts +54 -1
  59. package/src/__tests__/conversation-runtime-assembly.test.ts +33 -3
  60. package/src/__tests__/conversation-skill-tools.test.ts +27 -0
  61. package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
  62. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
  63. package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
  64. package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
  65. package/src/__tests__/conversation-title-service.test.ts +14 -10
  66. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
  67. package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
  68. package/src/__tests__/credential-execution-client.test.ts +70 -53
  69. package/src/__tests__/credential-record-retire.test.ts +171 -1
  70. package/src/__tests__/credential-routes.test.ts +250 -3
  71. package/src/__tests__/credential-security-invariants.test.ts +5 -3
  72. package/src/__tests__/cu-unified-flow.test.ts +14 -4
  73. package/src/__tests__/db-conversation-tool-surface.test.ts +144 -0
  74. package/src/__tests__/document-sync-tags.test.ts +25 -0
  75. package/src/__tests__/drain-kick-guard.test.ts +0 -2
  76. package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
  77. package/src/__tests__/events-tail-route.test.ts +33 -0
  78. package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
  79. package/src/__tests__/external-plugin-loader.test.ts +52 -0
  80. package/src/__tests__/git-porcelain.test.ts +47 -0
  81. package/src/__tests__/headless-browser-mode.test.ts +48 -0
  82. package/src/__tests__/heartbeat-service.test.ts +56 -40
  83. package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
  84. package/src/__tests__/host-bash-proxy.test.ts +26 -45
  85. package/src/__tests__/host-browser-proxy.test.ts +6 -17
  86. package/src/__tests__/host-cu-proxy.test.ts +712 -18
  87. package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
  88. package/src/__tests__/host-file-proxy.test.ts +11 -23
  89. package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
  90. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  91. package/src/__tests__/list-all-apps.test.ts +40 -8
  92. package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
  93. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  94. package/src/__tests__/llm-catalog-parity.test.ts +51 -3
  95. package/src/__tests__/managed-store.test.ts +121 -0
  96. package/src/__tests__/mcp-auth-routes.test.ts +104 -8
  97. package/src/__tests__/mcp-client-auth.test.ts +54 -0
  98. package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
  99. package/src/__tests__/mcp-health-check.test.ts +40 -43
  100. package/src/__tests__/mcp-list-plugin-servers.test.ts +85 -42
  101. package/src/__tests__/mcp-tool-annotations-risk.test.ts +4 -2
  102. package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
  103. package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
  104. package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
  105. package/src/__tests__/messaging-send-tool.test.ts +355 -548
  106. package/src/__tests__/model-intents.test.ts +3 -0
  107. package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
  108. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  109. package/src/__tests__/oauth-commands-routes.test.ts +282 -101
  110. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  111. package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
  112. package/src/__tests__/oauth-provider-serializer.test.ts +23 -0
  113. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  114. package/src/__tests__/openai-provider.test.ts +1 -0
  115. package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
  116. package/src/__tests__/persist-media-references.test.ts +50 -0
  117. package/src/__tests__/platform-callback-registration.test.ts +8 -16
  118. package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +13 -0
  120. package/src/__tests__/qdrant-manager.test.ts +132 -0
  121. package/src/__tests__/resolve-app.test.ts +135 -0
  122. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  123. package/src/__tests__/scaffold-managed-skill-tool.test.ts +317 -6
  124. package/src/__tests__/schedule-routes.test.ts +20 -0
  125. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  126. package/src/__tests__/script-proxy-certs.test.ts +35 -0
  127. package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
  128. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  129. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  130. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  131. package/src/__tests__/secure-keys.test.ts +7 -3
  132. package/src/__tests__/service-account-token.test.ts +247 -0
  133. package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
  134. package/src/__tests__/skill-feature-flags.test.ts +2 -2
  135. package/src/__tests__/skills.test.ts +20 -4
  136. package/src/__tests__/source-fingerprint.test.ts +184 -0
  137. package/src/__tests__/subagent-tool-gate-mode.test.ts +239 -3
  138. package/src/__tests__/subagent-tools.test.ts +3 -2
  139. package/src/__tests__/telephony-tts-guard.test.ts +38 -3
  140. package/src/__tests__/terminal-tools.test.ts +8 -0
  141. package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
  142. package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
  143. package/src/__tests__/ui-channel-variants.test.ts +1 -56
  144. package/src/__tests__/unicode.test.ts +49 -0
  145. package/src/__tests__/user-plugin-loader.test.ts +21 -0
  146. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
  147. package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
  148. package/src/__tests__/workspace-git-service.test.ts +26 -0
  149. package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
  150. package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
  151. package/src/acp/__tests__/acp-credentials.test.ts +13 -0
  152. package/src/acp/__tests__/agent-process.test.ts +258 -4
  153. package/src/acp/__tests__/auth-required.test.ts +96 -0
  154. package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
  155. package/src/acp/__tests__/client-handler.test.ts +90 -2
  156. package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
  157. package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
  158. package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
  159. package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
  160. package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
  161. package/src/acp/__tests__/session-manager.test.ts +895 -4
  162. package/src/acp/acp-claude-oauth.ts +328 -14
  163. package/src/acp/acp-credentials.ts +19 -0
  164. package/src/acp/agent-process.test.ts +4 -1
  165. package/src/acp/agent-process.ts +96 -10
  166. package/src/acp/auth-required.ts +50 -1
  167. package/src/acp/auto-install.test.ts +9 -7
  168. package/src/acp/claude-token-refresh.ts +150 -0
  169. package/src/acp/client-handler.ts +16 -3
  170. package/src/acp/model-config.test.ts +166 -0
  171. package/src/acp/model-config.ts +138 -0
  172. package/src/acp/prepare-agent-env.ts +21 -6
  173. package/src/acp/resolve-agent.test.ts +220 -6
  174. package/src/acp/resolve-agent.ts +88 -23
  175. package/src/acp/session-manager.test.ts +196 -3
  176. package/src/acp/session-manager.ts +441 -18
  177. package/src/acp/session-snapshot.ts +260 -0
  178. package/src/acp/types.ts +50 -5
  179. package/src/agent/loop.ts +20 -0
  180. package/src/api/attachment-provenance.test.ts +68 -0
  181. package/src/api/computer-use-tool.test.ts +55 -0
  182. package/src/api/computer-use-tool.ts +27 -0
  183. package/src/api/events/acp-session-model-update.test.ts +88 -0
  184. package/src/api/events/acp-session-model-update.ts +46 -0
  185. package/src/api/events/app-files-changed.ts +4 -3
  186. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  187. package/src/api/events/desktop-activity-changed.ts +10 -0
  188. package/src/api/events/notification-intent.ts +2 -0
  189. package/src/api/events/user-message-echo.ts +7 -1
  190. package/src/api/index.ts +17 -0
  191. package/src/api/responses/conversation-message.ts +8 -0
  192. package/src/approvals/approval-primitive.ts +5 -2
  193. package/src/approvals/scoped-approval-grants.ts +6 -2
  194. package/src/apps/app-store.ts +7 -6
  195. package/src/apps/resolve-app.ts +81 -0
  196. package/src/apps/source-fingerprint.ts +266 -0
  197. package/src/avatar/identity-avatar.ts +7 -2
  198. package/src/browser/operations.ts +2 -1
  199. package/src/browser/types.ts +7 -0
  200. package/src/browser/virtual-desktop-target.ts +34 -0
  201. package/src/bundler/app-compiler.ts +15 -0
  202. package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
  203. package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
  204. package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
  205. package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
  206. package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
  207. package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
  208. package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
  209. package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
  210. package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +27 -4
  211. package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
  212. package/src/calls/__tests__/voice-control-protocol.test.ts +151 -27
  213. package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
  214. package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
  215. package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
  216. package/src/calls/__tests__/voice-session-bridge.test.ts +140 -3
  217. package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
  218. package/src/calls/barge-in-guard.ts +118 -0
  219. package/src/calls/call-constants.ts +24 -4
  220. package/src/calls/call-controller.ts +499 -199
  221. package/src/calls/call-transport.ts +16 -0
  222. package/src/calls/media-stream-output.ts +32 -0
  223. package/src/calls/media-stream-server.ts +82 -30
  224. package/src/calls/media-stream-stt-session.ts +15 -0
  225. package/src/{live-voice → calls}/progress-narration.ts +17 -4
  226. package/src/calls/resolve-call-tts-provider.ts +33 -2
  227. package/src/calls/tts-call-strategy.ts +9 -5
  228. package/src/calls/types.ts +2 -0
  229. package/src/calls/voice-control-protocol.ts +194 -21
  230. package/src/calls/voice-leg-coordinator.ts +189 -0
  231. package/src/calls/voice-progress-cadence.ts +423 -0
  232. package/src/calls/voice-session-bridge.ts +140 -76
  233. package/src/calls/voice-triage-escalate.ts +178 -59
  234. package/src/cli/commands/__tests__/apps.test.ts +245 -0
  235. package/src/cli/commands/__tests__/bash.test.ts +53 -0
  236. package/src/cli/commands/__tests__/browser.test.ts +84 -3
  237. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  238. package/src/cli/commands/__tests__/plugins.test.ts +85 -0
  239. package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
  240. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  241. package/src/cli/commands/apps.help.ts +67 -13
  242. package/src/cli/commands/apps.ts +183 -0
  243. package/src/cli/commands/bash.help.ts +9 -13
  244. package/src/cli/commands/bash.ts +7 -25
  245. package/src/cli/commands/browser.help.ts +43 -0
  246. package/src/cli/commands/browser.ts +52 -8
  247. package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
  248. package/src/cli/commands/channels/index.help.ts +70 -2
  249. package/src/cli/commands/channels/index.ts +16 -3
  250. package/src/cli/commands/channels/send.ts +128 -0
  251. package/src/cli/commands/credentials.help.ts +9 -5
  252. package/src/cli/commands/credentials.ts +2 -2
  253. package/src/cli/commands/mcp.help.ts +8 -8
  254. package/src/cli/commands/mcp.ts +1 -1
  255. package/src/cli/commands/oauth/index.help.ts +10 -0
  256. package/src/cli/commands/oauth/providers.ts +12 -0
  257. package/src/cli/commands/oauth/request.test.ts +179 -2
  258. package/src/cli/commands/oauth/request.ts +41 -10
  259. package/src/cli/commands/oauth/status.ts +5 -0
  260. package/src/cli/commands/plugins.ts +73 -23
  261. package/src/cli/commands/roadmap.help.ts +5 -1
  262. package/src/cli/commands/roadmap.ts +1 -1
  263. package/src/cli/commands/schedules.ts +2 -0
  264. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  265. package/src/cli/lib/__tests__/install-from-github.test.ts +219 -0
  266. package/src/cli/lib/__tests__/install-from-platform.test.ts +77 -2
  267. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +45 -3
  268. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +251 -0
  269. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +65 -2
  270. package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +14 -0
  271. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +27 -1
  272. package/src/cli/lib/__tests__/plugin-details.test.ts +35 -2
  273. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +20 -0
  274. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +43 -0
  275. package/src/cli/lib/__tests__/search-plugins.test.ts +31 -0
  276. package/src/cli/lib/__tests__/uninstall-plugin.test.ts +122 -5
  277. package/src/cli/lib/bundled-marketplace.json +962 -81
  278. package/src/cli/lib/bundled-plugin-packages.json +786 -0
  279. package/src/cli/lib/bundled-plugin-packages.ts +87 -0
  280. package/src/cli/lib/diff-plugin.ts +1 -1
  281. package/src/cli/lib/inspect-plugin.ts +87 -12
  282. package/src/cli/lib/install-from-github.ts +150 -94
  283. package/src/cli/lib/install-from-platform.ts +14 -9
  284. package/src/cli/lib/list-installed-plugins.ts +21 -35
  285. package/src/cli/lib/plugin-catalog-cache.ts +22 -3
  286. package/src/cli/lib/plugin-catalog-local.ts +13 -3
  287. package/src/cli/lib/plugin-catalog-platform.ts +6 -1
  288. package/src/cli/lib/plugin-catalog-resolve.ts +20 -0
  289. package/src/cli/lib/plugin-details.ts +33 -8
  290. package/src/cli/lib/plugin-marketplace.ts +121 -21
  291. package/src/cli/lib/plugin-pin-history.ts +5 -2
  292. package/src/cli/lib/search-plugins.ts +58 -16
  293. package/src/cli/lib/uninstall-plugin.ts +59 -1
  294. package/src/cli/lib/upgrade-plugin.ts +61 -8
  295. package/src/cli/output.ts +18 -4
  296. package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
  297. package/src/config/__tests__/default-provider.test.ts +4 -0
  298. package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
  299. package/src/config/__tests__/platform-identity.test.ts +255 -0
  300. package/src/config/__tests__/platform-rehydration.test.ts +99 -9
  301. package/src/config/__tests__/plugin-resident-skill-discovery.test.ts +21 -0
  302. package/src/config/acp-agent-write.ts +43 -0
  303. package/src/config/acp-defaults.test.ts +13 -3
  304. package/src/config/acp-defaults.ts +23 -6
  305. package/src/config/acp-schema.test.ts +81 -0
  306. package/src/config/acp-schema.ts +30 -2
  307. package/src/config/bundled-skills/acp/SKILL.md +23 -15
  308. package/src/config/bundled-skills/acp/TOOLS.json +7 -3
  309. package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
  310. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
  311. package/src/config/bundled-skills/computer-use/SKILL.md +59 -2
  312. package/src/config/bundled-skills/computer-use/TOOLS.json +101 -2
  313. package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
  314. package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
  315. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
  316. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
  317. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  318. package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
  319. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
  320. package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
  321. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
  322. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
  323. package/src/config/bundled-skills/schedule/SKILL.md +7 -8
  324. package/src/config/bundled-skills/screen-annotation/SKILL.md +10 -3
  325. package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
  326. package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
  327. package/src/config/bundled-skills/skill-management/TOOLS.json +13 -9
  328. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
  329. package/src/config/default-profile-catalog.ts +1 -1
  330. package/src/config/env.ts +2 -2
  331. package/src/config/feature-flag-registry.json +33 -13
  332. package/src/config/loader.ts +11 -17
  333. package/src/config/memory-v3-gate.test.ts +31 -0
  334. package/src/config/memory-v3-gate.ts +20 -16
  335. package/src/config/platform-identity.ts +201 -0
  336. package/src/config/platform-rehydration.ts +11 -27
  337. package/src/config/profile-text-generation.test.ts +51 -0
  338. package/src/config/profile-text-generation.ts +51 -0
  339. package/src/config/schema.ts +2 -2
  340. package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
  341. package/src/config/schemas/__tests__/voice.test.ts +148 -0
  342. package/src/config/schemas/calls.ts +1 -28
  343. package/src/config/schemas/live-voice.ts +1 -173
  344. package/src/config/schemas/llm.ts +7 -2
  345. package/src/config/schemas/mcp.ts +15 -9
  346. package/src/config/schemas/memory-retrospective.ts +10 -0
  347. package/src/config/schemas/monitoring.ts +9 -0
  348. package/src/config/schemas/services.ts +18 -0
  349. package/src/config/schemas/skills.ts +0 -14
  350. package/src/config/schemas/voice.ts +150 -0
  351. package/src/config/skills.ts +13 -21
  352. package/src/credential-execution/ces-runtime.ts +2 -2
  353. package/src/credential-execution/executable-discovery.ts +36 -134
  354. package/src/credential-execution/process-manager.test.ts +37 -42
  355. package/src/credential-execution/process-manager.ts +24 -19
  356. package/src/credential-health/credential-health-service.ts +11 -17
  357. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +47 -0
  358. package/src/daemon/__tests__/conversation-tool-setup.test.ts +109 -121
  359. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +5 -1
  360. package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
  361. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  362. package/src/daemon/app-change-notify.ts +45 -0
  363. package/src/daemon/assistant-attachments.ts +20 -12
  364. package/src/daemon/config-watcher.ts +13 -194
  365. package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
  366. package/src/daemon/conversation-agent-loop.ts +12 -0
  367. package/src/daemon/conversation-attachments.ts +93 -12
  368. package/src/daemon/conversation-client-surface.ts +26 -0
  369. package/src/daemon/conversation-error.ts +3 -2
  370. package/src/daemon/conversation-media-retry.ts +2 -1
  371. package/src/daemon/conversation-process.ts +35 -64
  372. package/src/daemon/conversation-runtime-assembly.ts +21 -8
  373. package/src/daemon/conversation-skill-tools.ts +13 -31
  374. package/src/daemon/conversation-store.ts +1 -1
  375. package/src/daemon/conversation-surfaces.ts +21 -10
  376. package/src/daemon/conversation-tool-setup.ts +135 -109
  377. package/src/daemon/conversation-turn-finalize.ts +26 -13
  378. package/src/daemon/conversation.ts +17 -3
  379. package/src/daemon/daemon-control.ts +3 -7
  380. package/src/daemon/dictation-profile-store.ts +2 -1
  381. package/src/daemon/handlers/shared.ts +2 -1
  382. package/src/daemon/handlers/skills.ts +6 -3
  383. package/src/daemon/host-bash-proxy.ts +8 -12
  384. package/src/daemon/host-browser-proxy.ts +15 -1
  385. package/src/daemon/host-cu-proxy.ts +275 -45
  386. package/src/daemon/host-file-proxy.ts +8 -10
  387. package/src/daemon/host-proxy-preactivation.ts +4 -1
  388. package/src/daemon/host-transfer-proxy.ts +14 -18
  389. package/src/daemon/lifecycle.ts +4 -9
  390. package/src/daemon/mcp-reload-service.ts +13 -6
  391. package/src/daemon/message-types/shared.ts +1 -0
  392. package/src/daemon/message-types/sync.ts +2 -0
  393. package/src/daemon/orphan-reaper.ts +4 -3
  394. package/src/daemon/persist-media-references.ts +20 -3
  395. package/src/daemon/providers-setup.ts +5 -1
  396. package/src/daemon/reaction-history-render.ts +2 -1
  397. package/src/daemon/shutdown-handlers.ts +1 -9
  398. package/src/daemon/skill-memory-refresh.ts +64 -0
  399. package/src/daemon/tool-setup-types.ts +6 -0
  400. package/src/daemon/tool-side-effects.ts +7 -18
  401. package/src/daemon/wake-conversation-ops.ts +38 -15
  402. package/src/desktop/__tests__/fake-desktop.ts +123 -0
  403. package/src/desktop/desktop-automation-lease.test.ts +351 -0
  404. package/src/desktop/desktop-automation-lease.ts +306 -0
  405. package/src/desktop/desktop-browser-client.test.ts +413 -0
  406. package/src/desktop/desktop-browser-client.ts +430 -0
  407. package/src/desktop/desktop-browser-cursor.ts +40 -0
  408. package/src/desktop/desktop-browser-endpoint.test.ts +78 -0
  409. package/src/desktop/desktop-browser-endpoint.ts +145 -0
  410. package/src/desktop/desktop-browser-operations.ts +130 -0
  411. package/src/desktop/desktop-chrome-policy.test.ts +79 -0
  412. package/src/desktop/desktop-chrome-policy.ts +47 -0
  413. package/src/desktop/desktop-chrome-session.test.ts +48 -0
  414. package/src/desktop/desktop-chrome-session.ts +87 -0
  415. package/src/desktop/desktop-dependencies.test.ts +131 -0
  416. package/src/desktop/desktop-dependencies.ts +329 -0
  417. package/src/desktop/desktop-display.ts +9 -0
  418. package/src/desktop/desktop-panel-config.test.ts +135 -0
  419. package/src/desktop/desktop-panel-config.ts +162 -0
  420. package/src/desktop/desktop-session-manager.test.ts +813 -0
  421. package/src/desktop/desktop-session-manager.ts +970 -0
  422. package/src/desktop/desktop-stream-bridge.test.ts +395 -0
  423. package/src/desktop/desktop-stream-bridge.ts +214 -0
  424. package/src/desktop/desktop-wallpaper.test.ts +191 -0
  425. package/src/desktop/desktop-wallpaper.ts +113 -0
  426. package/src/desktop/desktop-window-drag.ts +177 -0
  427. package/src/desktop/desktop-window-manager-config.test.ts +230 -0
  428. package/src/desktop/desktop-window-manager-config.ts +238 -0
  429. package/src/desktop/desktop-window-theme.ts +170 -0
  430. package/src/desktop/virtual-desktop-feature.ts +18 -0
  431. package/src/desktop/virtual-desktop-platform.test.ts +61 -0
  432. package/src/heartbeat/heartbeat-service.ts +21 -3
  433. package/src/home/__tests__/feed-writer.test.ts +22 -0
  434. package/src/home/feed-writer.ts +11 -2
  435. package/src/hooks/types.ts +33 -0
  436. package/src/i18n/AGENTS.md +22 -0
  437. package/src/i18n/__tests__/i18n.test.ts +112 -0
  438. package/src/i18n/index.ts +16 -0
  439. package/src/i18n/locales.ts +93 -0
  440. package/src/i18n/messages.ts +70 -0
  441. package/src/i18n/title.ts +25 -0
  442. package/src/inbound/platform-callback-registration.ts +4 -5
  443. package/src/ipc/__tests__/browser-ipc.test.ts +230 -2
  444. package/src/ipc/__tests__/cancel-on-disconnect.test.ts +54 -0
  445. package/src/ipc/assistant-server.ts +15 -1
  446. package/src/ipc/cli-client.ts +13 -5
  447. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +143 -0
  448. package/src/live-voice/__tests__/live-voice-events.test.ts +9 -7
  449. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
  450. package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
  451. package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
  452. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +19 -0
  453. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
  454. package/src/live-voice/__tests__/live-voice-vad.test.ts +660 -5
  455. package/src/live-voice/__tests__/protocol.test.ts +95 -0
  456. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
  457. package/src/live-voice/__tests__/session-controls.test.ts +156 -0
  458. package/src/live-voice/live-voice-metrics.ts +9 -0
  459. package/src/live-voice/live-voice-photo.ts +25 -6
  460. package/src/live-voice/live-voice-session.ts +1081 -760
  461. package/src/live-voice/protocol.ts +89 -2
  462. package/src/live-voice/session-controls.ts +197 -0
  463. package/src/mcp/__tests__/credential-target.test.ts +59 -0
  464. package/src/mcp/__tests__/effective-config.test.ts +13 -2
  465. package/src/mcp/__tests__/manager-state.test.ts +128 -0
  466. package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
  467. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +20 -6
  468. package/src/mcp/__tests__/mcp-auth-state-target.test.ts +49 -0
  469. package/src/mcp/__tests__/plugin-mcp-oauth-cleanup.test.ts +107 -0
  470. package/src/mcp/__tests__/plugin-mcp-oauth-provider-isolation.test.ts +89 -0
  471. package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +63 -11
  472. package/src/mcp/__tests__/reload-signal-emission.test.ts +39 -2
  473. package/src/mcp/__tests__/startup.test.ts +30 -7
  474. package/src/mcp/__tests__/tool-caps.test.ts +107 -0
  475. package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
  476. package/src/mcp/client.ts +64 -16
  477. package/src/mcp/credential-target.ts +105 -0
  478. package/src/mcp/effective-config.ts +45 -18
  479. package/src/mcp/manager.ts +201 -81
  480. package/src/mcp/mcp-auth-orchestrator.ts +17 -4
  481. package/src/mcp/mcp-auth-state.ts +38 -6
  482. package/src/mcp/mcp-header-store.ts +29 -45
  483. package/src/mcp/mcp-oauth-provider.ts +101 -33
  484. package/src/mcp/spec-schema.ts +111 -0
  485. package/src/mcp/startup.ts +41 -17
  486. package/src/mcp/tool-caps.ts +176 -0
  487. package/src/mcp/workspace-mcp-config.ts +188 -0
  488. package/src/messaging/provider-message-metadata.ts +3 -3
  489. package/src/messaging/provider.ts +10 -3
  490. package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
  491. package/src/messaging/providers/callback-routing.ts +18 -0
  492. package/src/messaging/providers/channel-transport.ts +56 -0
  493. package/src/messaging/providers/discord/transport.ts +17 -0
  494. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
  495. package/src/messaging/providers/slack/adapter.ts +7 -44
  496. package/src/messaging/providers/slack/transport.ts +16 -0
  497. package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
  498. package/src/messaging/providers/telegram-bot/client.ts +0 -26
  499. package/src/messaging/providers/telegram-bot/transport.ts +20 -0
  500. package/src/messaging/providers/whatsapp/adapter.ts +1 -48
  501. package/src/messaging/providers/whatsapp/transport.ts +18 -0
  502. package/src/monitoring/__tests__/mount-watch.test.ts +236 -0
  503. package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
  504. package/src/monitoring/mount-watch.ts +348 -0
  505. package/src/monitoring/plugin-auto-update.ts +6 -0
  506. package/src/monitoring/worker.ts +9 -0
  507. package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
  508. package/src/notifications/__tests__/copy-composer.test.ts +70 -0
  509. package/src/notifications/__tests__/decision-engine.test.ts +175 -0
  510. package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
  511. package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
  512. package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
  513. package/src/notifications/adapters/macos.ts +3 -0
  514. package/src/notifications/assistant-reply-producer.ts +8 -3
  515. package/src/notifications/conversation-pairing.ts +36 -16
  516. package/src/notifications/copy-composer.ts +11 -3
  517. package/src/notifications/decision-engine.ts +43 -8
  518. package/src/notifications/delivered-post-record.ts +11 -1
  519. package/src/notifications/guardian-feed-projection.ts +125 -5
  520. package/src/notifications/home-feed-side-effect.ts +55 -7
  521. package/src/notifications/schedule-result-producer.ts +64 -22
  522. package/src/oauth/AGENTS.md +4 -0
  523. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  524. package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
  525. package/src/oauth/byo-connection.test.ts +105 -0
  526. package/src/oauth/byo-connection.ts +36 -6
  527. package/src/oauth/connection-resolver.test.ts +100 -0
  528. package/src/oauth/connection-resolver.ts +120 -3
  529. package/src/oauth/identity-verifier.ts +24 -0
  530. package/src/oauth/oauth-store.ts +11 -1
  531. package/src/oauth/provider-serializer.ts +10 -0
  532. package/src/oauth/scope-utils.ts +24 -0
  533. package/src/oauth/seed-providers.ts +367 -27
  534. package/src/onboarding/checkin-event.ts +2 -1
  535. package/src/outbound-proxy/certs.ts +3 -5
  536. package/src/permissions/checker.test.ts +11 -6
  537. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  538. package/src/permissions/confirmation-guardian-request.ts +2 -3
  539. package/src/persistence/attachments-store.ts +84 -44
  540. package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
  541. package/src/persistence/conversation-crud.ts +182 -83
  542. package/src/persistence/conversation-lineage.ts +12 -4
  543. package/src/persistence/conversation-plugin-facade.ts +13 -0
  544. package/src/persistence/conversation-queries.ts +2 -2
  545. package/src/persistence/conversation-title-placeholders.ts +23 -0
  546. package/src/persistence/conversation-title-service.ts +9 -26
  547. package/src/persistence/conversation-tool-surface.ts +86 -0
  548. package/src/persistence/conversation-types.test.ts +32 -0
  549. package/src/persistence/conversation-types.ts +26 -4
  550. package/src/persistence/embeddings/qdrant-manager.ts +163 -18
  551. package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
  552. package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
  553. package/src/persistence/message-cursor.ts +78 -0
  554. package/src/persistence/message-reads.ts +30 -0
  555. package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
  556. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  557. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  558. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  559. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  560. package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
  561. package/src/persistence/schema/conversation-tool-surfaces.ts +22 -0
  562. package/src/persistence/schema/index.ts +1 -0
  563. package/src/persistence/schema/memory-core.ts +5 -0
  564. package/src/persistence/schema/oauth.ts +1 -0
  565. package/src/persistence/steps.ts +15 -3
  566. package/src/platform/client.test.ts +4 -2
  567. package/src/platform/client.ts +2 -10
  568. package/src/plugin-api/constants.ts +2 -0
  569. package/src/plugin-api/conversation-turn.ts +31 -7
  570. package/src/plugin-api/index.ts +10 -1
  571. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  572. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  573. package/src/plugin-api/types.ts +2 -0
  574. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +14 -1
  575. package/src/plugins/__tests__/mcp-servers.test.ts +38 -8
  576. package/src/plugins/defaults/index.ts +2 -0
  577. package/src/plugins/defaults/memory/AGENTS.md +14 -2
  578. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  579. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  580. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  581. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
  582. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
  583. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
  584. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
  585. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +66 -10
  586. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +20 -0
  587. package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
  588. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
  589. package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
  590. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  591. package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
  592. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  593. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  594. package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -3
  595. package/src/plugins/defaults/memory/context-search/format.ts +4 -2
  596. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +7 -3
  597. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +4 -2
  598. package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -8
  599. package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
  600. package/src/plugins/defaults/memory/hooks/init.ts +10 -0
  601. package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
  602. package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
  603. package/src/plugins/defaults/memory/injectors.ts +4 -3
  604. package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
  605. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
  606. package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
  607. package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
  608. package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
  609. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
  610. package/src/plugins/defaults/memory/memory-retrospective-job.ts +67 -184
  611. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  612. package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
  613. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
  614. package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
  615. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  616. package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
  617. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
  618. package/src/plugins/defaults/memory/startup.ts +28 -2
  619. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  620. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  621. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  622. package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
  623. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  624. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  625. package/src/plugins/defaults/memory/substrate/page-index.ts +7 -1
  626. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
  627. package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
  628. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  629. package/src/plugins/defaults/memory/tools.ts +1 -1
  630. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
  631. package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -1
  632. package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
  633. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  634. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  635. package/src/plugins/defaults/memory/v2/reranker.ts +7 -1
  636. package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
  637. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  638. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  639. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
  640. package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
  641. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +51 -0
  642. package/src/plugins/defaults/memory/v3/card.ts +4 -2
  643. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  644. package/src/plugins/defaults/memory/v3/orchestrate.ts +86 -22
  645. package/src/plugins/defaults/memory/v3/pool-select.ts +13 -8
  646. package/src/plugins/defaults/memory/v3/sections.ts +10 -3
  647. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +13 -1
  648. package/src/plugins/defaults/memory/worker.ts +6 -0
  649. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  650. package/src/plugins/external-plugin-loader.ts +49 -86
  651. package/src/plugins/installed-plugin-dirs.ts +6 -4
  652. package/src/plugins/mcp-servers.ts +24 -90
  653. package/src/plugins/mtime-cache.ts +2 -1
  654. package/src/plugins/user-loader.ts +2 -1
  655. package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
  656. package/src/providers/__tests__/retry-callsite.test.ts +41 -0
  657. package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
  658. package/src/providers/anthropic/client.ts +0 -47
  659. package/src/providers/gemini/client.ts +9 -23
  660. package/src/providers/gemini-thought-signature.test.ts +63 -0
  661. package/src/providers/gemini-thought-signature.ts +44 -0
  662. package/src/providers/inference/__tests__/endpoint-probe.test.ts +87 -3
  663. package/src/providers/inference/adapter-factory.ts +6 -0
  664. package/src/providers/inference/auth.ts +2 -3
  665. package/src/providers/inference/connection-availability.ts +27 -8
  666. package/src/providers/inference/endpoint-probe.ts +36 -12
  667. package/src/providers/inference/resolve-auth.ts +22 -4
  668. package/src/providers/inference/service-account-token.ts +248 -0
  669. package/src/providers/jev/client.test.ts +260 -0
  670. package/src/providers/jev/client.ts +518 -0
  671. package/src/providers/model-catalog.ts +92 -3
  672. package/src/providers/model-intents.ts +10 -5
  673. package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
  674. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
  675. package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
  676. package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
  677. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
  678. package/src/providers/openai/chat-completions-provider.ts +212 -79
  679. package/src/providers/openai/codex-models.ts +0 -4
  680. package/src/providers/openai/google-thought-signature.ts +320 -0
  681. package/src/providers/openai/responses-provider.ts +9 -2
  682. package/src/providers/opencode/client.test.ts +240 -4
  683. package/src/providers/opencode/client.ts +129 -3
  684. package/src/providers/openrouter/client.ts +20 -0
  685. package/src/providers/outbound-request-sanitize.ts +96 -0
  686. package/src/providers/retry.ts +30 -19
  687. package/src/providers/unparseable-tool-args.ts +5 -1
  688. package/src/runtime/AGENTS.md +9 -3
  689. package/src/runtime/__tests__/agent-wake.test.ts +86 -1
  690. package/src/runtime/__tests__/channel-send.test.ts +443 -0
  691. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +132 -0
  692. package/src/runtime/__tests__/host-observe.test.ts +40 -1
  693. package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
  694. package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
  695. package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
  696. package/src/runtime/agent-wake.ts +22 -5
  697. package/src/runtime/assistant-event-hub.ts +21 -3
  698. package/src/runtime/auth/same-actor.ts +38 -3
  699. package/src/runtime/channel-send.ts +316 -0
  700. package/src/runtime/guardian-action-service.ts +2 -17
  701. package/src/runtime/guardian-reply-router.ts +1 -8
  702. package/src/runtime/host-observe.ts +11 -1
  703. package/src/runtime/http-server.ts +160 -151
  704. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  705. package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
  706. package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
  707. package/src/runtime/routes/__tests__/acp-routes.test.ts +178 -4
  708. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
  709. package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
  710. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
  711. package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
  712. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
  713. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  714. package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
  715. package/src/runtime/routes/__tests__/plugins-routes.test.ts +127 -3
  716. package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
  717. package/src/runtime/routes/acp-routes.test.ts +15 -0
  718. package/src/runtime/routes/acp-routes.ts +84 -165
  719. package/src/runtime/routes/app-management-routes.ts +90 -2
  720. package/src/runtime/routes/app-routes.ts +1 -1
  721. package/src/runtime/routes/browser-context.ts +39 -0
  722. package/src/runtime/routes/browser-routes.ts +16 -47
  723. package/src/runtime/routes/browser-tabs-routes.ts +72 -29
  724. package/src/runtime/routes/channel-route-shared.ts +1 -9
  725. package/src/runtime/routes/channel-send-routes.ts +85 -0
  726. package/src/runtime/routes/conversation-list-routes.ts +14 -3
  727. package/src/runtime/routes/conversation-query-routes.ts +40 -0
  728. package/src/runtime/routes/conversation-routes.ts +45 -7
  729. package/src/runtime/routes/credential-routes.ts +20 -14
  730. package/src/runtime/routes/desktop-setup-routes.test.ts +89 -0
  731. package/src/runtime/routes/desktop-setup-routes.ts +41 -0
  732. package/src/runtime/routes/diagnostics-routes.ts +61 -4
  733. package/src/runtime/routes/events-routes.ts +4 -0
  734. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  735. package/src/runtime/routes/host-cu-routes.ts +6 -0
  736. package/src/runtime/routes/inbound-message-handler.ts +2 -3
  737. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
  738. package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
  739. package/src/runtime/routes/index.ts +4 -2
  740. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  741. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  742. package/src/runtime/routes/log-export-routes.ts +2 -0
  743. package/src/runtime/routes/mcp-auth-routes.ts +141 -137
  744. package/src/runtime/routes/migration-routes.ts +6 -20
  745. package/src/runtime/routes/oauth-commands-routes.ts +98 -45
  746. package/src/runtime/routes/oauth-providers.ts +11 -0
  747. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  748. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  749. package/src/runtime/routes/platform-routes.ts +18 -15
  750. package/src/runtime/routes/plugins-routes.ts +83 -50
  751. package/src/runtime/routes/schedule-routes.ts +2 -0
  752. package/src/runtime/routes/secret-routes.ts +99 -119
  753. package/src/runtime/routes/workspace-routes.test.ts +55 -26
  754. package/src/runtime/routes/workspace-routes.ts +46 -63
  755. package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
  756. package/src/runtime/routes/workspace-tree-walk.ts +240 -0
  757. package/src/runtime/routes/workspace-utils.ts +17 -0
  758. package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
  759. package/src/runtime/services/conversation-serializer.ts +16 -3
  760. package/src/runtime/sync/resource-sync-events.ts +6 -1
  761. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  762. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  763. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  764. package/src/schedule/run-script.ts +4 -3
  765. package/src/schedule/schedule-store.ts +9 -2
  766. package/src/schedule/scheduler.ts +47 -11
  767. package/src/schedule/tool-surface-readiness.ts +65 -0
  768. package/src/schedule/worker-mcp.ts +85 -0
  769. package/src/schedule/worker.ts +9 -31
  770. package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
  771. package/src/security/ces-http-record-backend.ts +120 -0
  772. package/src/security/ces-rpc-record-backend.ts +7 -0
  773. package/src/security/secure-keys.ts +60 -48
  774. package/src/security/untrusted-content.ts +3 -1
  775. package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
  776. package/src/skills/managed-store.ts +153 -31
  777. package/src/skills/workspace-skill-md-signature.ts +50 -0
  778. package/src/subagent/consult-context.ts +12 -25
  779. package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
  780. package/src/tools/acp/list-agents.test.ts +3 -1
  781. package/src/tools/acp/spawn.test.ts +146 -11
  782. package/src/tools/acp/spawn.ts +21 -3
  783. package/src/tools/acp/status.test.ts +349 -0
  784. package/src/tools/acp/status.ts +112 -9
  785. package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
  786. package/src/tools/browser/browser-execution.ts +45 -14
  787. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +69 -0
  788. package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
  789. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +1 -0
  790. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
  791. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +5 -0
  792. package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
  793. package/src/tools/browser/network-recorder.ts +2 -1
  794. package/src/tools/capability-offer.test.ts +22 -2
  795. package/src/tools/capability-offer.ts +16 -1
  796. package/src/tools/computer-use/definitions.ts +123 -5
  797. package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
  798. package/src/tools/credentials/metadata-store.ts +100 -0
  799. package/src/tools/filesystem/search.ts +2 -1
  800. package/src/tools/host-filesystem/edit.test.ts +2 -1
  801. package/src/tools/host-filesystem/edit.ts +2 -2
  802. package/src/tools/host-filesystem/read.test.ts +2 -1
  803. package/src/tools/host-filesystem/read.ts +2 -2
  804. package/src/tools/host-filesystem/transfer.test.ts +2 -1
  805. package/src/tools/host-filesystem/transfer.ts +2 -2
  806. package/src/tools/host-filesystem/write.test.ts +2 -1
  807. package/src/tools/host-filesystem/write.ts +2 -2
  808. package/src/tools/host-terminal/host-shell.test.ts +16 -0
  809. package/src/tools/host-terminal/host-shell.ts +16 -26
  810. package/src/tools/network/web-search.ts +6 -1
  811. package/src/tools/policy-context.ts +6 -7
  812. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  813. package/src/tools/shared/shell-output.test.ts +10 -0
  814. package/src/tools/shared/shell-output.ts +14 -2
  815. package/src/tools/skills/find-similar-skills.test.ts +214 -2
  816. package/src/tools/skills/find-similar-skills.ts +62 -9
  817. package/src/tools/skills/resolve-execute-invocation.ts +24 -0
  818. package/src/tools/skills/sandbox-runner.ts +13 -2
  819. package/src/tools/skills/scaffold-managed.ts +78 -16
  820. package/src/tools/subagent/spawn.ts +5 -8
  821. package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
  822. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +43 -0
  823. package/src/tools/terminal/safe-env.ts +35 -24
  824. package/src/tools/terminal/sanitized-bash.ts +118 -0
  825. package/src/tools/terminal/shell-launch.test.ts +162 -0
  826. package/src/tools/terminal/shell.test.ts +29 -0
  827. package/src/tools/terminal/shell.ts +13 -7
  828. package/src/tools/ui-surface/channel-variants.ts +10 -59
  829. package/src/tools/watch/watch-retro-report.ts +0 -8
  830. package/src/types/host-capabilities.ts +5 -3
  831. package/src/util/__tests__/language-subtag.test.ts +23 -1
  832. package/src/util/abort-reasons.ts +12 -1
  833. package/src/util/ca-bundle.ts +13 -0
  834. package/src/util/host-process.test.ts +17 -1
  835. package/src/util/host-process.ts +24 -0
  836. package/src/util/language-subtag.ts +16 -0
  837. package/src/util/platform.ts +8 -0
  838. package/src/util/plugin-manifest.ts +203 -0
  839. package/src/util/unicode.ts +44 -1
  840. package/src/watch/watch-retro.ts +6 -8
  841. package/src/watch/watch-timeline.ts +2 -2
  842. package/src/workflows/run-manager.ts +2 -1
  843. package/src/workspace/git-porcelain.ts +66 -0
  844. package/src/workspace/git-service.ts +6 -55
  845. package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
  846. package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
  847. package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
  848. package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
  849. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
  850. package/src/workspace/migrations/registry.ts +6 -0
  851. package/src/__tests__/app-source-watcher.test.ts +0 -218
  852. package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
  853. package/src/__tests__/speaker-identification.test.ts +0 -56
  854. package/src/calls/speaker-identification.ts +0 -233
  855. package/src/daemon/app-source-watcher.ts +0 -218
  856. package/src/messaging/providers/whatsapp/client.ts +0 -29
  857. package/src/runtime/routes/debug-bash-routes.ts +0 -173
  858. package/src/util/fs-watcher-error.ts +0 -36
  859. /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
@@ -1,19 +1,41 @@
1
1
  import { Buffer } from "node:buffer";
2
2
  import { randomUUID } from "node:crypto";
3
3
 
4
+ import {
5
+ type BargeInGuard,
6
+ createBargeInGuard,
7
+ DEFAULT_BARGE_IN_MIN_SPEECH_MS,
8
+ } from "../calls/barge-in-guard.js";
4
9
  import {
5
10
  MediaTurnDetector,
6
11
  type TurnDetectorConfig,
7
12
  } from "../calls/media-turn-detector.js";
13
+ import {
14
+ createVoiceProgressNarrator,
15
+ type VoiceProgressNarrator,
16
+ } from "../calls/progress-narration.js";
17
+ import {
18
+ APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
19
+ approvalPendingPhraseFor,
20
+ } from "../calls/progress-phrases.js";
8
21
  import {
9
22
  SPOKEN_REPLY_LENGTH_RULE,
10
23
  SPOKEN_REPLY_PLAIN_TEXT_RULE,
11
24
  } from "../calls/spoken-reply-rules.js";
12
25
  import { sanitizeForTts } from "../calls/tts-text-sanitizer.js";
13
26
  import {
14
- isIncompleteControlMarkerTail,
15
- stripInternalSpeechMarkers,
27
+ createControlMarkerHoldback,
28
+ TASK_STOP_MARKER,
16
29
  } from "../calls/voice-control-protocol.js";
30
+ import {
31
+ createFrontDoorLegCoordinator,
32
+ type FrontDoorLegCoordinator,
33
+ type SpokenEscalationBridge,
34
+ } from "../calls/voice-leg-coordinator.js";
35
+ import {
36
+ createProgressCadence,
37
+ type ProgressCadence,
38
+ } from "../calls/voice-progress-cadence.js";
17
39
  import type {
18
40
  VoiceTurnHandle,
19
41
  VoiceTurnOptions,
@@ -23,24 +45,16 @@ import {
23
45
  resolveProcessingWaitMs,
24
46
  waitForPriorTurnTeardown,
25
47
  } from "../calls/voice-session-bridge.js";
26
- import {
27
- capEscalationBridge,
28
- classifyFrontDoorLeading,
29
- ESCALATE_VERDICT_TOKEN,
30
- ESCALATION_CONTINUATION_CONTENT,
31
- FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
32
- fallbackEscalationBridgeFor,
33
- isEscalationBridgeComplete,
34
- MIN_SPOKEN_BRIDGE_CHARS,
35
- type VoiceRoutingLeg,
36
- } from "../calls/voice-triage-escalate.js";
48
+ import { type VoiceRoutingLeg } from "../calls/voice-triage-escalate.js";
37
49
  import { getConfig } from "../config/loader.js";
38
50
  import {
39
51
  type LiveVoiceFluxConfig,
40
52
  LiveVoiceFluxConfigSchema,
41
- type LiveVoiceFrontModelConfig,
42
- LiveVoiceFrontModelConfigSchema,
43
53
  } from "../config/schemas/live-voice.js";
54
+ import {
55
+ type VoiceFrontModelConfig,
56
+ VoiceFrontModelConfigSchema,
57
+ } from "../config/schemas/voice.js";
44
58
  import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
45
59
  import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
46
60
  import type { TrustContext } from "../daemon/trust-context-types.js";
@@ -86,14 +100,15 @@ import {
86
100
  liveVoiceSilenceReason,
87
101
  liveVoiceStartScreen,
88
102
  } from "../telemetry/live-voice-funnel.js";
89
- import { getToolOwner } from "../tools/registry.js";
103
+ import { getTool, getToolOwner } from "../tools/registry.js";
104
+ import { resolveSkillExecuteInvocation } from "../tools/skills/resolve-execute-invocation.js";
90
105
  import {
91
106
  createReasoningTagFilter,
92
107
  type ReasoningTagFilter,
93
108
  } from "../tts/reasoning-tag-filter.js";
94
109
  import { extractSpeakableSegments } from "../tts/speakable-segments.js";
95
110
  import { createAbortReason } from "../util/abort-reasons.js";
96
- import { hasLocalizedEntry } from "../util/language-subtag.js";
111
+ import { fixedPhraseLanguage } from "../util/language-subtag.js";
97
112
  import { getLogger } from "../util/logger.js";
98
113
  import {
99
114
  activityLabelForTool,
@@ -136,17 +151,6 @@ import type {
136
151
  LiveVoiceTtsOptions,
137
152
  LiveVoiceTtsResult,
138
153
  } from "./live-voice-tts.js";
139
- import {
140
- createVoiceProgressNarrator,
141
- type VoiceProgressNarrator,
142
- type VoiceProgressTextInput,
143
- } from "./progress-narration.js";
144
- import {
145
- APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
146
- approvalPendingPhraseFor,
147
- pickProgressPhrase,
148
- PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
149
- } from "./progress-phrases.js";
150
154
  import {
151
155
  type LiveVoiceClientAttachFrameFrame,
152
156
  type LiveVoiceClientAttachImageFrame,
@@ -156,7 +160,19 @@ import {
156
160
  type LiveVoiceClientUpdateConfigFrame,
157
161
  LiveVoiceProtocolErrorCode,
158
162
  type LiveVoiceServerFramePayload,
163
+ type LiveVoiceSessionControl,
159
164
  } from "./protocol.js";
165
+ import {
166
+ type ClientSessionControlRequest,
167
+ isLookSessionControl,
168
+ LOOK_FOLLOW_UP_CONTENT,
169
+ LOOK_FRAME_REASON,
170
+ lookFollowUpNote,
171
+ type LookSessionControl,
172
+ progressConfigForCadence,
173
+ requestedSessionControl,
174
+ sessionControlTeaching,
175
+ } from "./session-controls.js";
160
176
 
161
177
  const log = getLogger("live-voice-session");
162
178
 
@@ -200,7 +216,6 @@ const FINALIZE_GRACE_MS = 1_000;
200
216
  // upstream of the provider, which hears only the microphone: it cannot tell
201
217
  // our own TTS bleeding through imperfect echo cancellation from the caller,
202
218
  // and would report a user turn nobody took.
203
- const DEFAULT_BARGE_IN_MIN_SPEECH_MS = 250;
204
219
  // The playback echo gate learns microphone energy while assistant audio is
205
220
  // expected at the speaker. Input must rise above the learned level by this
206
221
  // margin to count as user speech.
@@ -236,24 +251,6 @@ const NOISE_FLOOR_MAX_BASE_MULTIPLE = 4;
236
251
  // tracks the effective trailing-silence threshold (the detector keeps its own
237
252
  // copy private) so the endpoint decider can report the pause length.
238
253
  const DEFAULT_SILENCE_THRESHOLD_MS = 800;
239
- // Longest continuous sub-threshold gap the sustained-speech barge-in run
240
- // tolerates without resetting. A gap this short is a syllable boundary, or the
241
- // choppy energy the browser's half-duplex echo canceller produces while the
242
- // assistant is still playing (it ducks the user's near-end voice, so post-AEC
243
- // user speech arrives as intermittent above-gate chunks) — so the run keeps
244
- // accumulating across it and a barge-in during playback still lands. Only a
245
- // longer continuous silence (a real end of speech, or an isolated cough) resets
246
- // the run.
247
- const BARGE_IN_GAP_TOLERANCE_MS = 200;
248
- // Ceiling on cumulative sub-threshold time across a whole barge-in run, as a
249
- // multiple of bargeInMinSpeechMs. Per-gap tolerance alone lets sparse isolated
250
- // blips (e.g. a 10 ms echo spike every 200 ms) each clear the consecutive-gap
251
- // timer while retaining prior speech, so they would sum to the guard over
252
- // several seconds and fire a barge-in with no sustained user speech. Capping the
253
- // run's total tolerated silence imposes a minimum above-gate duty cycle
254
- // (1 / (1 + ratio) ≈ 20%): once the run is mostly silence it resets, so genuine
255
- // choppy speech still lands but periodic noise cannot accumulate into one.
256
- const BARGE_IN_MAX_TOLERATED_SILENCE_RATIO = 4;
257
254
  // Slack added to the configured end-of-turn timeout before the session stops
258
255
  // waiting for an event that is not coming and falls the utterance back onto
259
256
  // the silence-boundary path. A turn-detecting provider force-ends its own turn
@@ -277,11 +274,26 @@ const CONTINUATION_ANNOUNCE_SILENCE_MS = 1_500;
277
274
  // reply's audio landing between two checks. Beyond the cap the announcement
278
275
  // falls back to the stash rather than chasing a call that keeps talking.
279
276
  const CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS = 3;
277
+ const FOREGROUND_TASK_RESUME_SILENCE_MS = 1_500;
278
+ const FOREGROUND_TASK_MAX_SUSPENDED_MS = 120_000;
279
+ const FOREGROUND_TASK_MAX_INTERVENING_TURNS = 3;
280
280
  // `content` of an announcement turn. The answer never rides here — it goes in
281
281
  // the model-facing control prompt (buildLiveDeliveryNote), so the only thing
282
282
  // persisted on the user side is this marker, and it persists hidden.
283
283
  export const CONTINUATION_DELIVERY_CONTENT =
284
284
  "(background work finished — deliver it now)";
285
+ // How long a look control waits for the fresh frame the client takes for it.
286
+ // Long enough for a camera that has to open and warm up before its first keep;
287
+ // past it the look is dropped rather than answered into a later, unrelated
288
+ // silence.
289
+ const LOOK_FRAME_WAIT_MS = 10_000;
290
+ // How long after the look was asked for its answer may still start. Past it
291
+ // the floor has been taken for so long (a user talking through noise that never
292
+ // became a turn, say) that the look no longer answers what is on screen.
293
+ const LOOK_ANSWER_DEADLINE_MS = 15_000;
294
+ // The shortest wait before checking the floor again, for a blocker with no
295
+ // known end time (the look's own turn clearing, an utterance in capture).
296
+ const LOOK_FOLLOW_UP_REARM_MS = 250;
285
297
 
286
298
  export type LiveVoiceStreamingTranscriberResolver = (
287
299
  options: ResolveStreamingTranscriberOptions,
@@ -407,11 +419,11 @@ export interface LiveVoiceSessionOptions {
407
419
  finalizeGraceMs?: number;
408
420
  /**
409
421
  * Voice front-door endpointing and progress narration tuning. The
410
- * production factory seeds this from `liveVoice.frontModel` config when
422
+ * production factory seeds this from `voice.frontModel` config when
411
423
  * unset; absent fields fall back to the schema defaults (the constructor
412
424
  * schema-parses the partial into a complete config).
413
425
  */
414
- frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
426
+ frontModelConfig?: Partial<VoiceFrontModelConfig>;
415
427
  /**
416
428
  * Deepgram Flux turn-detection tuning. The production factory seeds this
417
429
  * from `liveVoice.flux` config when unset; absent fields fall back to the
@@ -421,7 +433,7 @@ export interface LiveVoiceSessionOptions {
421
433
  fluxConfig?: Partial<LiveVoiceFluxConfig>;
422
434
  /**
423
435
  * Progress narration service. The production factory constructs it from
424
- * `liveVoice.frontModel.progress` config when unset. `null` disables
436
+ * `voice.frontModel.progress` config when unset. `null` disables
425
437
  * generated narration, so progress updates use their static fallback.
426
438
  */
427
439
  progressNarrator?: VoiceProgressNarrator | null;
@@ -457,6 +469,12 @@ export interface LiveVoiceSessionOptions {
457
469
  * `CONTINUATION_ANNOUNCE_SILENCE_MS`.
458
470
  */
459
471
  continuationAnnounceSilenceMs?: number;
472
+ /** Overrides the silence before a suspended foreground task resumes (test hook). */
473
+ foregroundTaskResumeSilenceMs?: number;
474
+ /** Overrides the maximum time a foreground task may remain suspended (test hook). */
475
+ foregroundTaskMaxSuspendedMs?: number;
476
+ /** Overrides the committed user-turn limit while suspended (test hook). */
477
+ foregroundTaskMaxInterveningTurns?: number;
460
478
  }
461
479
 
462
480
  type LiveVoiceUtterancePhase =
@@ -604,70 +622,6 @@ interface TtsSegmentJob {
604
622
  frames: Promise<void>;
605
623
  }
606
624
 
607
- // One tool operation observed on a turn, fed by the bridge's structured tool
608
- // callbacks. `completedAtMs`/`isError`/`resultPreview` land with tool_result.
609
- interface TurnProgressOp {
610
- toolName: string;
611
- toolUseId?: string;
612
- startedAtMs: number;
613
- completedAtMs?: number;
614
- isError?: boolean;
615
- resultPreview?: string;
616
- }
617
-
618
- // Per-turn tool-activity log and narration cadence state for spoken progress
619
- // updates (liveVoice.frontModel.progress).
620
- interface TurnProgressState {
621
- // Tool operations this turn, in start order.
622
- ops: TurnProgressOp[];
623
- // Ops accumulated toward the next ops-triggered narration. Counted once per
624
- // op, on start (not completion), so a burst of slow tools still trips the
625
- // threshold while they run.
626
- opsSinceNarration: number;
627
- // Bumped by every observable change to the turn's tool activity — an op
628
- // starting or finishing. The idle trigger compares it against
629
- // `narratedEpoch` so a tick with nothing new to report stays silent.
630
- stateEpoch: number;
631
- // The `stateEpoch` the last spoken narration described: the activity the
632
- // user has already been told about.
633
- narratedEpoch: number;
634
- // Narrations actually spoken this turn — the metrics count and the
635
- // decider's 1-based updateIndex. Rate, not count, bounds narration:
636
- // idleIntervalMs/minGapMs cap the cadence and the session duration cap
637
- // bounds the turn.
638
- updatesSpoken: number;
639
- // When the last spoken floor-holder (ack or narration) enqueued — gates the
640
- // progress.minGapMs spacing guard. Null until something speaks.
641
- lastFloorHolderAtMs: number | null;
642
- // When the turn's last TTS segment finished emitting (turn launch until
643
- // anything speaks). Together with the session-level playback-tail estimate
644
- // this anchors the dead-air countdown: idle time is measured from when the
645
- // user last heard something, not from when the turn started.
646
- lastAudibleAtMs: number;
647
- // Self-re-arming dead-air narration timer; null once cleared or fired.
648
- idleTimer: ReturnType<typeof setTimeout> | null;
649
- // A narration generation is awaiting the decider; serializes narrations.
650
- narrationInFlight: boolean;
651
- }
652
-
653
- // Newest incomplete op, optionally restricted to a tool name (the
654
- // tool_result fallback match when no toolUseId correlates).
655
- function findLastIncompleteOp(
656
- ops: TurnProgressOp[],
657
- toolName?: string,
658
- ): TurnProgressOp | undefined {
659
- for (let i = ops.length - 1; i >= 0; i -= 1) {
660
- const op = ops[i];
661
- if (
662
- op.completedAtMs === undefined &&
663
- (toolName === undefined || op.toolName === toolName)
664
- ) {
665
- return op;
666
- }
667
- }
668
- return undefined;
669
- }
670
-
671
625
  interface ActiveAssistantTurn {
672
626
  token: symbol;
673
627
  turnId: string;
@@ -683,7 +637,9 @@ interface ActiveAssistantTurn {
683
637
  handle: VoiceTurnHandle | null;
684
638
  // When the turn launched, for narration's turnElapsedMs.
685
639
  launchedAtMs: number;
686
- progress: TurnProgressState;
640
+ // Tool-activity log and spoken progress narration for the turn
641
+ // (voice.frontModel.progress), shared with the phone driver.
642
+ progress: ProgressCadence;
687
643
  assistantCompleted: boolean;
688
644
  ttsDone: boolean;
689
645
  // Latched when the turn puts something on screen (a ui tool ran); consumed
@@ -691,6 +647,10 @@ interface ActiveAssistantTurn {
691
647
  // Never set from anything the model says: the reveal is a consequence of
692
648
  // showing a surface, not a token the model has to remember.
693
649
  minimizeRequested: boolean;
650
+ // The session control the completed reply ended with (see
651
+ // session-controls.ts); consumed at TTS drain like the minimize, where the
652
+ // session_control frame goes out once the acknowledgement has been spoken.
653
+ sessionControlRequested: ClientSessionControlRequest | null;
694
654
  // The activity label the client was last told about, so a run of tools that
695
655
  // map to the same line sends one frame rather than one per call. Empty means
696
656
  // the client believes nothing is running, which is also where a turn ends.
@@ -767,10 +727,21 @@ interface ActiveAssistantTurn {
767
727
  // no user utterance behind it — `content` is CONTINUATION_DELIVERY_CONTENT and
768
728
  // the answer rides the control prompt (buildLiveDeliveryNote).
769
729
  continuationDelivery: ContinuationDelivery | null;
730
+ // Set only on the turn that answers a look: which look it answers. The turn
731
+ // has no user utterance behind it; the instruction rides the control prompt
732
+ // (lookFollowUpNote).
733
+ lookFollowUp: LookSessionControl | null;
770
734
  // The turn's content is an internal instruction rather than user speech (the
771
735
  // greeting that opens a session, say). The row still persists and the model
772
736
  // still sees it; `hiddenSyntheticPrompt` keeps it out of the transcript.
773
737
  hiddenPrompt: boolean;
738
+ // The suspended foreground task this turn may continue. Set only after the user
739
+ // turn commits or on the synthetic resume turn; speculative discards never
740
+ // mutate the session's task state.
741
+ foregroundTaskEpoch: number | null;
742
+ // Snapshot retained while a tool-capable leg owns suspended work. If that
743
+ // leg fails asynchronously, the task returns to the same suspension budget.
744
+ foregroundTaskSuspensionSnapshot: SuspendedForegroundTask | null;
774
745
  // Set when a barge-in handed the interrupted work to a background subagent:
775
746
  // that request's transcript, so the model can tell the user the work is
776
747
  // still running instead of appearing to have dropped it.
@@ -803,7 +774,9 @@ interface ActiveAssistantTurn {
803
774
  // Guards the
804
775
  // front-door leg's trailing completion from finalizing the turn, and makes
805
776
  // the hand-off idempotent.
806
- escalationHandedOff: boolean;
777
+ // The front-door leg's coordinator: whether it handed the turn off to the
778
+ // escalated leg. Null until the front-door leg starts.
779
+ frontDoor: FrontDoorLegCoordinator | null;
807
780
  ttsBuffer: string;
808
781
  // What the caller actually hears this turn, summed over the model's own
809
782
  // segments (acks and progress narration do not count). Logged at tts_done
@@ -829,55 +802,6 @@ interface ActiveAssistantTurn {
829
802
  assistantAudioSampleRate?: number;
830
803
  }
831
804
 
832
- /**
833
- * Control-marker hygiene for one model leg's delta stream, shared by the
834
- * front-door answer stage and the default/escalated leg. The returned flush
835
- * forwards the stripped (stripInternalSpeechMarkers) prefix of `raw` that has
836
- * not been emitted yet and cannot contain a still-streaming control marker:
837
- * the flush stops at the first "[" whose tail is an incomplete marker
838
- * (isIncompleteControlMarkerTail) and holds from there until a later delta
839
- * completes or disproves it; `force` (leg completion) emits the held tail so
840
- * real text that merely resembles a marker prefix is not dropped. The scan
841
- * runs forward from the emitted boundary — not from the last "[" — so
842
- * brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
843
- * in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
844
- * its terminator.
845
- *
846
- * **Markers are stripped, never acted on.** Nothing the model can say
847
- * minimizes the room any more: that is decided by whether a ui tool ran (see
848
- * the `tool_use_start` handler), so the reveal cannot depend on the model
849
- * remembering a token, and a reply whose content happens to contain "[-1]" (an
850
- * array literal, a temperature) cannot move the room either. No prompt teaches
851
- * a marker, so this stripping is defense against a model that emits one
852
- * regardless: an unspoken, unpersisted "[-1]" is the correct handling of a
853
- * token that now means nothing.
854
- */
855
- function createControlMarkerHoldback(
856
- turn: ActiveAssistantTurn,
857
- emit: (chunk: string) => void,
858
- ): (raw: string, opts?: { force?: boolean }) => void {
859
- let emitted = 0;
860
- return (raw, opts) => {
861
- let safeEnd = raw.length;
862
- if (opts?.force !== true) {
863
- for (
864
- let i = raw.indexOf("[", emitted);
865
- i !== -1;
866
- i = raw.indexOf("[", i + 1)
867
- ) {
868
- if (isIncompleteControlMarkerTail(raw.slice(i))) {
869
- safeEnd = i;
870
- break;
871
- }
872
- }
873
- }
874
- if (safeEnd > emitted) {
875
- emit(stripInternalSpeechMarkers(raw.slice(emitted, safeEnd)));
876
- emitted = safeEnd;
877
- }
878
- };
879
- }
880
-
881
805
  // Base control prompt for every live-voice turn. Opens with the spoken-reply
882
806
  // rules shared with the phone path (spoken-reply-rules.ts): this is the only
883
807
  // place the model learns that its text is spoken, since the system prompt has
@@ -885,7 +809,7 @@ function createControlMarkerHoldback(
885
809
  // message. When a turn starts from a barge-in, the interruption merge note is
886
810
  // appended to it (see buildInterruptionMergeNote) so the model reconciles the
887
811
  // interrupted request with the new utterance.
888
- const LIVE_VOICE_CONTROL_PROMPT_BASE = `You are speaking in a local live voice session. ${SPOKEN_REPLY_LENGTH_RULE} ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. `;
812
+ const LIVE_VOICE_CONTROL_PROMPT_BASE = `You are speaking in a local live voice session. ${SPOKEN_REPLY_LENGTH_RULE} ${SPOKEN_REPLY_PLAIN_TEXT_RULE} Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work, or a card with an image or a link instead of reading an address out); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. `;
889
813
 
890
814
  // Appended for the legs that can actually put something on screen: the main
891
815
  // leg and the escalated leg. The front-door (fast) leg never receives it, for
@@ -901,7 +825,7 @@ const LIVE_VOICE_CONTROL_PROMPT_BASE = `You are speaking in a local live voice s
901
825
  // model can get right, which is speaking as though the thing is already in
902
826
  // front of the user, because by the time it stops talking it is.
903
827
  const LIVE_VOICE_SCREEN_REVEAL_TEACHING =
904
- "When the complete answer would run past a few sentences, say the short version out loud and put the detail on screen instead of reading it out. The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. Never emit bracketed markers of any kind. ";
828
+ "When the complete answer would run past a few sentences, say the short version out loud and put the detail on screen instead of reading it out. The call renders as a full-screen overlay covering the app. Whenever you put something on screen, the overlay minimizes by itself as soon as you finish speaking, and the user is looking at what you made. So speak as if you are showing it to them right now (for example, close with something like: take a look), and never say you cannot show it, that this is a voice call, or that they should check it later. ";
905
829
 
906
830
  // The setup-flow case, spelled out because it is the one the model gets wrong
907
831
  // on its own: connecting an account reads as something a call cannot do, so it
@@ -922,7 +846,7 @@ const LIVE_VOICE_SETUP_FLOW_TEACHING =
922
846
  // off answering, rather than a fresh follow-up. Reaches the model only; it is
923
847
  // not a user message and never renders as a transcript bubble.
924
848
  function buildInterruptionMergeNote(interruptedRequest: string): string {
925
- return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together, or stay silent if they only want you to stop.`;
849
+ return `The user interrupted your previous, unfinished reply. Their earlier request was: "${interruptedRequest}". Treat their current message as a continuation of that request and address both together. If they corrected or redirected the unfinished work, continue executing it with that correction instead of merely acknowledging it. If they clearly want the unfinished task abandoned, use the task-stop control.`;
926
850
  }
927
851
 
928
852
  // System-level guidance appended to the NEXT turn's control prompt after a
@@ -955,6 +879,14 @@ function describeInterruptedRequest(request: string): string {
955
879
  : "their earlier request";
956
880
  }
957
881
 
882
+ // A look control waiting on its fresh frame: which look, when it was asked
883
+ // for, and the wait's bound.
884
+ interface PendingLook {
885
+ action: LookSessionControl;
886
+ armedAtMs: number;
887
+ timer: ReturnType<typeof setTimeout>;
888
+ }
889
+
958
890
  // A finished background continuation waiting to be delivered: the request it
959
891
  // took over and the answer it produced.
960
892
  interface ContinuationDelivery {
@@ -962,6 +894,26 @@ interface ContinuationDelivery {
962
894
  answer: string;
963
895
  }
964
896
 
897
+ interface OwnedForegroundTask {
898
+ phase: "owned";
899
+ epoch: number;
900
+ request: string;
901
+ ownerToken: symbol;
902
+ hostToolStarted: boolean;
903
+ }
904
+
905
+ interface SuspendedForegroundTask {
906
+ phase: "suspended";
907
+ epoch: number;
908
+ request: string;
909
+ suspendedAtMs: number;
910
+ interveningTurns: number;
911
+ resumePending: boolean;
912
+ hostToolStarted: boolean;
913
+ }
914
+
915
+ type ForegroundTaskState = OwnedForegroundTask | SuspendedForegroundTask;
916
+
965
917
  // Appended to an announcement turn's control prompt: the turn the session
966
918
  // starts on its own when a background continuation finishes while the call is
967
919
  // live and nobody is speaking. There is no user utterance behind it, so this
@@ -973,6 +925,12 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
973
925
  return `The user interrupted you earlier, and in the background you finished ${what}. The call is still live and nobody is speaking, so tell them briefly that it is done and give them the result. Do not re-run any tool calls; the work is already complete, and do not repeat it verbatim if it no longer fits. What you produced was:\n\n${answer}`;
974
926
  }
975
927
 
928
+ function buildForegroundTaskResumePrompt(request: string): string {
929
+ const task =
930
+ request.length > 0 ? JSON.stringify(request) : "the unfinished task";
931
+ return `Continue the unfinished task that began with ${task}. Follow the user's latest corrections and all newer conversation context. If the conversation has clearly moved on or the user no longer wants the task, do not continue it and output only ${TASK_STOP_MARKER}, with no spoken words before the marker.`;
932
+ }
933
+
976
934
  // Assemble a leg's model-facing control prompt: the base live-voice rules, the
977
935
  // screen-reveal and setup-flow teaching (both withheld from the front-door leg,
978
936
  // see LIVE_VOICE_SCREEN_REVEAL_TEACHING), plus any pending barge-in merge
@@ -982,12 +940,21 @@ function buildLiveDeliveryNote(request: string, answer: string): string {
982
940
  function buildVoiceControlPrompt(
983
941
  turn: ActiveAssistantTurn,
984
942
  leg: { frontDoor?: boolean },
943
+ sessionControls: readonly LiveVoiceSessionControl[],
944
+ client: { lookFrames: boolean },
985
945
  ): string {
986
946
  let prompt =
987
947
  LIVE_VOICE_CONTROL_PROMPT_BASE +
988
948
  (leg.frontDoor === true
989
949
  ? ""
990
- : LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING);
950
+ : LIVE_VOICE_SCREEN_REVEAL_TEACHING + LIVE_VOICE_SETUP_FLOW_TEACHING) +
951
+ sessionControlTeaching(sessionControls, leg, {
952
+ ...client,
953
+ unfinishedTaskPending:
954
+ turn.foregroundTaskEpoch !== null ||
955
+ turn.interruptedRequest !== null ||
956
+ turn.handedOffRequest !== null,
957
+ });
991
958
  if (turn.language !== undefined) {
992
959
  prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
993
960
  }
@@ -1000,6 +967,9 @@ function buildVoiceControlPrompt(
1000
967
  if (turn.handedOffRequest) {
1001
968
  prompt = `${prompt}\n\n${buildHandoffAnnouncementNote(turn.handedOffRequest)}`;
1002
969
  }
970
+ if (turn.lookFollowUp !== null) {
971
+ prompt = `${prompt}\n\n${lookFollowUpNote(turn.lookFollowUp)}`;
972
+ }
1003
973
  if (turn.continuationDelivery) {
1004
974
  prompt = `${prompt}\n\n${buildLiveDeliveryNote(
1005
975
  turn.continuationDelivery.request,
@@ -1152,6 +1122,20 @@ function foregroundToolContendsWithContinuation(
1152
1122
  return isRefusedInReadOnlyPass(toolName, ownerKind);
1153
1123
  }
1154
1124
 
1125
+ function hostInteractiveToolName(
1126
+ toolName: string,
1127
+ input?: Record<string, unknown>,
1128
+ allowedToolNames?: ReadonlySet<string>,
1129
+ ): string | null {
1130
+ const effectiveToolName =
1131
+ toolName === "skill_execute"
1132
+ ? resolveSkillExecuteInvocation(input ?? {}, allowedToolNames).name
1133
+ : toolName;
1134
+ return getTool(effectiveToolName)?.executionTarget === "host"
1135
+ ? effectiveToolName
1136
+ : null;
1137
+ }
1138
+
1155
1139
  // Upper bound on how long a barge-in waits for the interrupted turn's teardown
1156
1140
  // to settle before giving up on the continuation. The teardown settles once the
1157
1141
  // aborted turn's agent loop reaches its `finally`, which can wait out BOTH the
@@ -1247,6 +1231,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1247
1231
  // take a turn without the microphone. Governs one thing only: whether a
1248
1232
  // missing speech-to-text leg is fatal to startup (see start()).
1249
1233
  private readonly textInput: boolean;
1234
+ // The session controls the client declared it can carry out; the only ones
1235
+ // the model is taught and the only ones a reply's marker can trigger.
1236
+ private readonly sessionControls: readonly LiveVoiceSessionControl[];
1237
+ // The client declared `lookFrames`: it sends a fresh frame for every look
1238
+ // control it carries out, so the session answers a look on its own turn and
1239
+ // teaches the model that the reply asking for one is only the acknowledgement.
1240
+ private readonly lookFrames: boolean;
1241
+ // How often progress updates are spoken, as the user last asked out loud.
1242
+ // Session-scoped: it applies from the next turn to the end of the call.
1243
+ private progressCadence: "fewer" | "normal" = "normal";
1250
1244
  // Whether this session's speech-to-text leg came up. False only when the
1251
1245
  // preflight found it missing and `textInput` let the session open anyway, in
1252
1246
  // which case nothing arms a transcriber and typed turns are the only input.
@@ -1289,22 +1283,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1289
1283
  // assistant turn is audibly speaking: above-gate speech-chunk duration
1290
1284
  // accumulates until it reaches bargeInMinSpeechMs, then the deferred
1291
1285
  // speech_started + barge-in fire (at most once per onset). Brief sub-threshold
1292
- // gaps are tolerated (see BARGE_IN_GAP_TOLERANCE_MS); the run resets on a
1286
+ // gaps are tolerated (see barge-in-guard.ts); the run resets on a
1293
1287
  // single longer continuous silence, or once cumulative tolerated silence
1294
- // exceeds the duty-cycle ceiling (see BARGE_IN_MAX_TOLERATED_SILENCE_RATIO).
1288
+ // exceeds the duty-cycle ceiling (see barge-in-guard.ts).
1295
1289
  // The detector's utterance end discards the guard.
1296
1290
  private pendingBargeIn: {
1297
1291
  // Null when guarding only the post-tts_done drain window (the turn is
1298
1292
  // already finalized but the client is still playing its tail).
1299
1293
  turn: ActiveAssistantTurn | null;
1300
- speechMs: number;
1301
- // Consecutive sub-threshold (non-speech) time since the last speech chunk;
1302
- // resets speechMs once it exceeds BARGE_IN_GAP_TOLERANCE_MS.
1303
- silenceMs: number;
1304
- // Cumulative sub-threshold time over the whole run (not reset by speech
1305
- // chunks); resets speechMs once it exceeds the duty-cycle ceiling so sparse
1306
- // periodic blips cannot sum into a barge-in.
1307
- toleratedSilenceMs: number;
1294
+ // The shared sustained-speech accounting (gap tolerance, duty-cycle
1295
+ // ceiling, threshold).
1296
+ guard: BargeInGuard;
1308
1297
  } | null = null;
1309
1298
  // Estimated wall-clock ms until the client finishes draining the
1310
1299
  // assistant audio sent so far. The server clears the turn right after
@@ -1336,6 +1325,23 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1336
1325
  // answer.
1337
1326
  private pendingAnnouncement: ContinuationDelivery | null = null;
1338
1327
  private announcementTimer: ReturnType<typeof setTimeout> | null = null;
1328
+ // Host-backed work stays on the parent conversation across barge-ins. An
1329
+ // owned task belongs to the tool-capable turn doing the work; a suspended
1330
+ // task is waiting for either a merged escalation or a synthetic resume.
1331
+ private foregroundTaskState: ForegroundTaskState | null = null;
1332
+ private foregroundTaskEpoch = 0;
1333
+ private foregroundTaskResumeTimer: ReturnType<typeof setTimeout> | null =
1334
+ null;
1335
+ private readonly foregroundTaskResumeSilenceMs: number;
1336
+ private readonly foregroundTaskMaxSuspendedMs: number;
1337
+ private readonly foregroundTaskMaxInterveningTurns: number;
1338
+ // A look control sent to a client that declared `lookFrames`, waiting for
1339
+ // the fresh frame the client takes for it. The session answers the look on a
1340
+ // turn of its own once that frame is in the conversation (see
1341
+ // answerLookWhenFloorIsFree). Cleared when the frame lands, when the wait
1342
+ // runs out, and when the session closes.
1343
+ private pendingLook: PendingLook | null = null;
1344
+ private lookFollowUpTimer: ReturnType<typeof setTimeout> | null = null;
1339
1345
  // Set when a continuation actually spawns: the request it took over, so the
1340
1346
  // NEXT turn can tell the user the work is still running. Consumed by that
1341
1347
  // turn; cleared when the continuation finishes (by then the result note
@@ -1350,6 +1356,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1350
1356
  // the distance from it to a keep arriving is the client leg of the frame the
1351
1357
  // onset asked for, measured from the daemon's own clock.
1352
1358
  private lastSpeechStartedAtMs: number | null = null;
1359
+ // How many assistant turns have launched, so a look can tell whether one has
1360
+ // started since its frame landed. A count rather than a time: two launches
1361
+ // and a frame can share a millisecond.
1362
+ private turnsLaunched = 0;
1353
1363
  private readonly maxPendingAudioBytes: number;
1354
1364
  // Set on VAD speech onset; consumed when the first speech chunk is routed
1355
1365
  // to an utterance so the metric lands on the right turn.
@@ -1400,9 +1410,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1400
1410
  // never emits an answer.
1401
1411
  private readonly progressNarrator: VoiceProgressNarrator | null;
1402
1412
  // Complete front-model tunables: the constructor schema-parses the partial
1403
- // option once, so every field carries its `liveVoice.frontModel` schema
1413
+ // option once, so every field carries its `voice.frontModel` schema
1404
1414
  // default when unset.
1405
- private readonly frontModelConfig: LiveVoiceFrontModelConfig;
1415
+ private readonly frontModelConfig: VoiceFrontModelConfig;
1406
1416
  // Complete Flux tunables (the constructor schema-parses the partial option
1407
1417
  // once, so every field carries its `liveVoice.flux` schema default).
1408
1418
  private readonly fluxConfig: LiveVoiceFluxConfig;
@@ -1471,6 +1481,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1471
1481
  defaultDetachTeardownSettleTimeoutMs();
1472
1482
  this.continuationAnnounceSilenceMs =
1473
1483
  options.continuationAnnounceSilenceMs ?? CONTINUATION_ANNOUNCE_SILENCE_MS;
1484
+ this.foregroundTaskResumeSilenceMs =
1485
+ options.foregroundTaskResumeSilenceMs ??
1486
+ FOREGROUND_TASK_RESUME_SILENCE_MS;
1487
+ this.foregroundTaskMaxSuspendedMs =
1488
+ options.foregroundTaskMaxSuspendedMs ?? FOREGROUND_TASK_MAX_SUSPENDED_MS;
1489
+ this.foregroundTaskMaxInterveningTurns =
1490
+ options.foregroundTaskMaxInterveningTurns ??
1491
+ FOREGROUND_TASK_MAX_INTERVENING_TURNS;
1474
1492
  this.emitMetrics = options.emitMetrics ?? false;
1475
1493
  this.createTurnId = options.createTurnId ?? randomUUID;
1476
1494
  this.conversationId =
@@ -1502,7 +1520,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1502
1520
  options.echoDrainSlackMs ?? DEFAULT_ECHO_DRAIN_SLACK_MS;
1503
1521
  this.finalizeGraceMs = options.finalizeGraceMs ?? FINALIZE_GRACE_MS;
1504
1522
  this.progressNarrator = options.progressNarrator ?? null;
1505
- this.frontModelConfig = LiveVoiceFrontModelConfigSchema.parse(
1523
+ this.frontModelConfig = VoiceFrontModelConfigSchema.parse(
1506
1524
  options.frontModelConfig ?? {},
1507
1525
  );
1508
1526
  this.fluxConfig = LiveVoiceFluxConfigSchema.parse(options.fluxConfig ?? {});
@@ -1526,6 +1544,8 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1526
1544
  2 *
1527
1545
  SERVER_VAD_PENDING_AUDIO_MAX_SECONDS;
1528
1546
  this.textInput = context.startFrame.textInput === true;
1547
+ this.sessionControls = context.startFrame.sessionControls ?? [];
1548
+ this.lookFrames = context.startFrame.lookFrames === true;
1529
1549
  }
1530
1550
 
1531
1551
  get finalTranscriptText(): string {
@@ -1774,6 +1794,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1774
1794
  },
1775
1795
  "Sight frame timing",
1776
1796
  );
1797
+ // Only once the row is in: the turn that answers the look reads the
1798
+ // conversation, so the frame has to be there before that turn starts.
1799
+ if (result.ok && frame.timing?.reason === LOOK_FRAME_REASON) {
1800
+ this.lookFrameLanded();
1801
+ }
1777
1802
  if (!result.ok && !this.isClosed) {
1778
1803
  void this.sendFrame({
1779
1804
  type: "error",
@@ -1960,6 +1985,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1960
1985
 
1961
1986
  const shouldEmitSessionEndMetrics = this.state !== "failed";
1962
1987
  this.state = "closed";
1988
+ this.clearForegroundTask("session_closed");
1963
1989
  // Retire the island before the teardown below starts awaiting things. A
1964
1990
  // close can take a while (a pending continuation is delivered first), and
1965
1991
  // an activity left asserting "Speaking…" through it is exactly the stale
@@ -1979,6 +2005,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1979
2005
  // cleared: the announcement is dead either way.
1980
2006
  await this.deliverPendingContinuationToConversation();
1981
2007
  this.clearContinuationAnnouncement();
2008
+ this.clearPendingLook();
1982
2009
  this.stopSessionTranscriber();
1983
2010
  // Detached continuations outlive the call. A deliberate `interrupt()`
1984
2011
  // aborts them; ending the session leaves them running. With no next voice
@@ -2488,7 +2515,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2488
2515
 
2489
2516
  if (this.echoWindowTotalAudioMs === 0) {
2490
2517
  this.echoWindowGuardCarryover =
2491
- this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
2518
+ this.pendingBargeIn !== null && this.pendingBargeIn.guard.speechMs > 0;
2492
2519
  } else if (this.pendingBargeIn === null) {
2493
2520
  this.echoWindowGuardCarryover = false;
2494
2521
  }
@@ -2533,7 +2560,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2533
2560
  );
2534
2561
  if (meanAmplitude > speechThreshold) {
2535
2562
  const guardHasSpeech =
2536
- this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
2563
+ this.pendingBargeIn !== null && this.pendingBargeIn.guard.speechMs > 0;
2537
2564
  if (!guardHasSpeech && this.echoMatchesAssistant(chunk)) {
2538
2565
  this.updateEchoEnergy(meanAmplitude, chunkMs);
2539
2566
  return [{ chunk, classification: "echo" }];
@@ -2831,9 +2858,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2831
2858
  // accumulates (trackBargeInGuard), so no speech is lost either way.
2832
2859
  this.pendingBargeIn = {
2833
2860
  turn: bargeableTurn,
2834
- speechMs: 0,
2835
- silenceMs: 0,
2836
- toleratedSilenceMs: 0,
2861
+ guard: createBargeInGuard(this.bargeInMinSpeechMs),
2837
2862
  };
2838
2863
  return;
2839
2864
  }
@@ -2857,77 +2882,50 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2857
2882
 
2858
2883
  // Advance the sustained-speech barge-in guard by one server-VAD chunk.
2859
2884
  // Speech accumulates toward bargeInMinSpeechMs, short true-silence gaps are
2860
- // tolerated, and classified playback echo resets the run immediately.
2861
- // Longer or mostly silent runs reset through the existing gap limits.
2885
+ // tolerated, and classified playback echo resets the run immediately (the
2886
+ // shared guard owns that accounting). A reset also drops the echo state a
2887
+ // run carried over, so the classifier re-learns the playback level.
2862
2888
  private trackBargeInGuard(
2863
2889
  classification: VadEnergyClassification,
2864
2890
  chunk: Buffer,
2865
2891
  ): void {
2866
- const guard = this.pendingBargeIn;
2867
- if (!guard) {
2892
+ const pending = this.pendingBargeIn;
2893
+ if (!pending) {
2868
2894
  return;
2869
2895
  }
2870
2896
  const chunkMs = pcm16DurationMs(
2871
2897
  chunk.byteLength,
2872
2898
  this.context.startFrame.audio.sampleRate,
2873
2899
  );
2874
- if (classification === "echo") {
2875
- this.resetBargeInGuardRun();
2876
- return;
2877
- }
2878
- if (classification === "silence") {
2879
- guard.silenceMs += chunkMs;
2880
- guard.toleratedSilenceMs += chunkMs;
2881
- // Strictly greater on the per-gap check: a gap of exactly
2882
- // BARGE_IN_GAP_TOLERANCE_MS is still tolerated. The web client batches PCM
2883
- // into 50 ms frames, so a run of ducked frames lands on the boundary
2884
- // exactly (e.g. four frames = 200 ms). The run also resets once its total
2885
- // tolerated silence outweighs the speech by the duty-cycle ceiling, so
2886
- // sparse periodic blips can never sum to the guard.
2887
- if (
2888
- guard.silenceMs > BARGE_IN_GAP_TOLERANCE_MS ||
2889
- guard.toleratedSilenceMs >
2890
- this.bargeInMinSpeechMs * BARGE_IN_MAX_TOLERATED_SILENCE_RATIO
2891
- ) {
2892
- this.resetBargeInGuardRun();
2900
+ const step = pending.guard.track(classification, chunkMs);
2901
+ if (step === "reset") {
2902
+ if (this.echoWindowGuardCarryover) {
2903
+ this.echoWindowGuardCarryover = false;
2904
+ this.echoEnergyEma = 0;
2905
+ this.echoProbeChunks = [];
2893
2906
  }
2894
2907
  return;
2895
2908
  }
2896
- guard.silenceMs = 0;
2897
- guard.speechMs += chunkMs;
2898
- if (guard.speechMs < this.bargeInMinSpeechMs) {
2909
+ if (step !== "fired") {
2899
2910
  return;
2900
2911
  }
2901
2912
  this.pendingBargeIn = null;
2902
2913
  this.assistantPlaybackTailUntilMs = 0;
2903
2914
  this.sendSpeechStarted();
2904
- const { turn } = guard;
2915
+ const { turn } = pending;
2905
2916
  if (turn && turn === this.activeAssistantTurn && !turn.finalized) {
2906
2917
  this.bargeIn(turn);
2907
2918
  }
2908
2919
  }
2909
2920
 
2910
- private resetBargeInGuardRun(): void {
2911
- const guard = this.pendingBargeIn;
2912
- if (!guard) {
2913
- return;
2914
- }
2915
- guard.speechMs = 0;
2916
- guard.silenceMs = 0;
2917
- guard.toleratedSilenceMs = 0;
2918
- if (this.echoWindowGuardCarryover) {
2919
- this.echoWindowGuardCarryover = false;
2920
- this.echoEnergyEma = 0;
2921
- this.echoProbeChunks = [];
2922
- }
2923
- }
2924
-
2925
2921
  private bargeIn(turn: ActiveAssistantTurn): void {
2926
2922
  // Abort synchronously so no tts_audio frame can follow turn_cancelled,
2927
2923
  // and settle the cancelled turn's metrics so the next utterance's marks
2928
2924
  // do not collide with it in the collector. turn_cancelled flushes
2929
2925
  // client playback, so the drain estimate resets with it.
2930
2926
  this.assistantPlaybackTailUntilMs = 0;
2927
+ const keepForegroundTaskOnParent =
2928
+ !turn.assistantCompleted && this.suspendOwnedForegroundTask(turn);
2931
2929
  // Carry the interrupted request into the next turn so it merges with the
2932
2930
  // barge-in utterance rather than being answered as a fresh follow-up.
2933
2931
  const interruptedRequest = turn.utterance.finalTranscriptSegments
@@ -3008,6 +3006,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3008
3006
  this.publishActivity(turn, "");
3009
3007
  await this.sendFrame({ type: "turn_cancelled", turnId: turn.turnId });
3010
3008
  await this.cancelAssistantTurn("barge_in");
3009
+ if (keepForegroundTaskOnParent) {
3010
+ log.info(
3011
+ { turnId: turn.turnId },
3012
+ "Voice duplex continuation skipped for parent-owned foreground task",
3013
+ );
3014
+ return;
3015
+ }
3011
3016
  // Keep the interrupted turn's work alive on a background subagent; the
3012
3017
  // detach waits for its teardown to settle the partial into history before
3013
3018
  // forking.
@@ -3053,16 +3058,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3053
3058
  // already finished, so bargeIn returns it to the stash instead.
3054
3059
  turn.continuationDelivery !== null
3055
3060
  ? "announcement_turn"
3056
- : // The model already finished generating (barge-in during TTS playback
3057
- // of a complete reply): there is nothing to continue, so a
3058
- // continuation would just re-do a finished answer.
3059
- turn.assistantCompleted
3060
- ? "assistant_already_completed"
3061
- : // A stop (interrupt/close) or a superseding invalidation landed
3062
- // during the barge-in teardown: honor it.
3063
- this.detachStopGeneration !== stopGeneration
3064
- ? "invalidated_during_barge_teardown"
3065
- : null;
3061
+ : // Nor over the answer to a look: there is no request behind it
3062
+ // either, and the user talking over it is them moving on.
3063
+ turn.lookFollowUp !== null
3064
+ ? "look_follow_up"
3065
+ : // The model already finished generating (barge-in during TTS playback
3066
+ // of a complete reply): there is nothing to continue, so a
3067
+ // continuation would just re-do a finished answer.
3068
+ turn.assistantCompleted
3069
+ ? "assistant_already_completed"
3070
+ : // A stop (interrupt/close) or a superseding invalidation landed
3071
+ // during the barge-in teardown: honor it.
3072
+ this.detachStopGeneration !== stopGeneration
3073
+ ? "invalidated_during_barge_teardown"
3074
+ : null;
3066
3075
  if (skipReason !== null) {
3067
3076
  log.info(
3068
3077
  { turnId: turn.turnId, skipReason },
@@ -3281,6 +3290,350 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3281
3290
  }
3282
3291
  }
3283
3292
 
3293
+ private foregroundTaskRequestForTurn(turn: ActiveAssistantTurn): string {
3294
+ const currentRequest = turn.utterance.finalTranscriptSegments
3295
+ .join(" ")
3296
+ .trim();
3297
+ return currentRequest || turn.interruptedRequest || "";
3298
+ }
3299
+
3300
+ private createForegroundTaskOwnership(
3301
+ turn: ActiveAssistantTurn,
3302
+ hostToolStarted: boolean,
3303
+ ): OwnedForegroundTask {
3304
+ const task: OwnedForegroundTask = {
3305
+ phase: "owned",
3306
+ epoch: ++this.foregroundTaskEpoch,
3307
+ request: this.foregroundTaskRequestForTurn(turn),
3308
+ ownerToken: turn.token,
3309
+ hostToolStarted,
3310
+ };
3311
+ turn.foregroundTaskEpoch = task.epoch;
3312
+ this.foregroundTaskState = task;
3313
+ return task;
3314
+ }
3315
+
3316
+ private noteEscalatedForegroundTask(turn: ActiveAssistantTurn): void {
3317
+ if (
3318
+ this.activeAssistantTurn?.token !== turn.token ||
3319
+ turn.discardRequested ||
3320
+ turn.abortController.signal.aborted
3321
+ ) {
3322
+ return;
3323
+ }
3324
+ if (
3325
+ this.claimSuspendedForegroundTask(turn) ||
3326
+ this.foregroundTaskState !== null
3327
+ ) {
3328
+ return;
3329
+ }
3330
+
3331
+ const task = this.createForegroundTaskOwnership(turn, false);
3332
+ log.info(
3333
+ { turnId: turn.turnId, epoch: task.epoch },
3334
+ "Escalated live voice turn provisionally owns a foreground task",
3335
+ );
3336
+ }
3337
+
3338
+ private noteHostToolStarted(
3339
+ turn: ActiveAssistantTurn,
3340
+ effectiveToolName: string,
3341
+ ): void {
3342
+ const current = this.foregroundTaskState;
3343
+ if (current?.phase === "owned" && current.ownerToken === turn.token) {
3344
+ if (!current.hostToolStarted) {
3345
+ this.foregroundTaskState = { ...current, hostToolStarted: true };
3346
+ log.info(
3347
+ { turnId: turn.turnId, epoch: current.epoch, effectiveToolName },
3348
+ "Foreground task confirmed by host tool use",
3349
+ );
3350
+ }
3351
+ return;
3352
+ }
3353
+ if (this.claimSuspendedForegroundTask(turn, true)) {
3354
+ return;
3355
+ }
3356
+
3357
+ const task = this.createForegroundTaskOwnership(turn, true);
3358
+ log.info(
3359
+ { turnId: turn.turnId, epoch: task.epoch, effectiveToolName },
3360
+ "Live voice turn started a foreground task",
3361
+ );
3362
+ }
3363
+
3364
+ private claimSuspendedForegroundTask(
3365
+ turn: ActiveAssistantTurn,
3366
+ hostToolStarted = false,
3367
+ ): boolean {
3368
+ const current = this.foregroundTaskState;
3369
+ if (
3370
+ current?.phase !== "suspended" ||
3371
+ turn.foregroundTaskEpoch !== current.epoch ||
3372
+ this.activeAssistantTurn?.token !== turn.token ||
3373
+ turn.discardRequested ||
3374
+ turn.abortController.signal.aborted
3375
+ ) {
3376
+ return false;
3377
+ }
3378
+ this.foregroundTaskState = {
3379
+ phase: "owned",
3380
+ epoch: current.epoch,
3381
+ request: current.request,
3382
+ ownerToken: turn.token,
3383
+ hostToolStarted: current.hostToolStarted || hostToolStarted,
3384
+ };
3385
+ this.clearForegroundTaskResumeTimer();
3386
+ log.info(
3387
+ {
3388
+ turnId: turn.turnId,
3389
+ epoch: current.epoch,
3390
+ hostToolStarted: current.hostToolStarted || hostToolStarted,
3391
+ },
3392
+ "Suspended foreground task claimed by tool-capable turn",
3393
+ );
3394
+ return true;
3395
+ }
3396
+
3397
+ private suspendOwnedForegroundTask(turn: ActiveAssistantTurn): boolean {
3398
+ const current = this.foregroundTaskState;
3399
+ if (current?.phase === "owned" && current.ownerToken === turn.token) {
3400
+ this.foregroundTaskState = {
3401
+ phase: "suspended",
3402
+ epoch: current.epoch,
3403
+ request: current.request,
3404
+ suspendedAtMs: Date.now(),
3405
+ interveningTurns: 0,
3406
+ resumePending: false,
3407
+ hostToolStarted: current.hostToolStarted,
3408
+ };
3409
+ this.clearForegroundTaskResumeTimer();
3410
+ log.info(
3411
+ { turnId: turn.turnId, epoch: current.epoch },
3412
+ "Foreground task suspended by voice barge-in",
3413
+ );
3414
+ return true;
3415
+ }
3416
+ return current !== null;
3417
+ }
3418
+
3419
+ private clearForegroundTask(reason: string): void {
3420
+ const current = this.foregroundTaskState;
3421
+ this.foregroundTaskEpoch += 1;
3422
+ this.foregroundTaskState = null;
3423
+ this.clearForegroundTaskResumeTimer();
3424
+ if (current !== null) {
3425
+ log.info(
3426
+ {
3427
+ epoch: current.epoch,
3428
+ phase: current.phase,
3429
+ hostToolStarted: current.hostToolStarted,
3430
+ reason,
3431
+ },
3432
+ "Live voice foreground task cleared",
3433
+ );
3434
+ }
3435
+ }
3436
+
3437
+ private stopOutstandingInterruptedWork(reason: string): void {
3438
+ this.clearForegroundTask(reason);
3439
+ this.abortDetachedRuns({ reason });
3440
+ }
3441
+
3442
+ private foregroundTaskSuspensionExpired(
3443
+ task: SuspendedForegroundTask,
3444
+ ): "turn_limit" | "time_limit" | null {
3445
+ if (task.interveningTurns > this.foregroundTaskMaxInterveningTurns) {
3446
+ return "turn_limit";
3447
+ }
3448
+ if (Date.now() - task.suspendedAtMs >= this.foregroundTaskMaxSuspendedMs) {
3449
+ return "time_limit";
3450
+ }
3451
+ return null;
3452
+ }
3453
+
3454
+ private queueForegroundTaskResume(turn: ActiveAssistantTurn): void {
3455
+ const current = this.foregroundTaskState;
3456
+ if (
3457
+ current?.phase !== "suspended" ||
3458
+ turn.foregroundTaskEpoch !== current.epoch ||
3459
+ turn.hiddenPrompt ||
3460
+ turn.discardRequested ||
3461
+ turn.speculativePending
3462
+ ) {
3463
+ return;
3464
+ }
3465
+ const next: SuspendedForegroundTask = {
3466
+ ...current,
3467
+ interveningTurns: current.interveningTurns + 1,
3468
+ resumePending: true,
3469
+ };
3470
+ const expired = this.foregroundTaskSuspensionExpired(next);
3471
+ if (expired !== null) {
3472
+ this.clearForegroundTask(`suspended_${expired}`);
3473
+ return;
3474
+ }
3475
+ this.foregroundTaskState = next;
3476
+ if (this.activeAssistantTurn === null) {
3477
+ this.scheduleForegroundTaskResume();
3478
+ }
3479
+ }
3480
+
3481
+ private markForegroundTaskResumePending(epoch: number): boolean {
3482
+ const current = this.foregroundTaskState;
3483
+ if (current?.phase !== "suspended" || current.epoch !== epoch) {
3484
+ return false;
3485
+ }
3486
+ this.foregroundTaskState = { ...current, resumePending: true };
3487
+ return true;
3488
+ }
3489
+
3490
+ private rearmForegroundTaskAfterLegFailure(
3491
+ turn: ActiveAssistantTurn,
3492
+ ): boolean {
3493
+ const epoch = turn.foregroundTaskEpoch;
3494
+ if (epoch === null) {
3495
+ return false;
3496
+ }
3497
+ const current = this.foregroundTaskState;
3498
+ if (current?.phase === "suspended" && current.epoch === epoch) {
3499
+ return this.markForegroundTaskResumePending(epoch);
3500
+ }
3501
+ const snapshot = turn.foregroundTaskSuspensionSnapshot;
3502
+ if (
3503
+ current?.phase !== "owned" ||
3504
+ current.epoch !== epoch ||
3505
+ current.ownerToken !== turn.token ||
3506
+ (snapshot !== null && snapshot.epoch !== epoch)
3507
+ ) {
3508
+ return false;
3509
+ }
3510
+ const next: SuspendedForegroundTask = {
3511
+ ...(snapshot ?? {
3512
+ phase: "suspended",
3513
+ epoch,
3514
+ request: current.request,
3515
+ suspendedAtMs: Date.now(),
3516
+ interveningTurns: 0,
3517
+ hostToolStarted: current.hostToolStarted,
3518
+ }),
3519
+ resumePending: true,
3520
+ };
3521
+ const expired = this.foregroundTaskSuspensionExpired(next);
3522
+ if (expired !== null) {
3523
+ this.clearForegroundTask(`suspended_${expired}`);
3524
+ return false;
3525
+ }
3526
+ this.foregroundTaskState = next;
3527
+ log.info(
3528
+ { turnId: turn.turnId, epoch },
3529
+ "Foreground task re-armed after tool-capable leg failure",
3530
+ );
3531
+ return true;
3532
+ }
3533
+
3534
+ private clearForegroundTaskResumeTimer(): void {
3535
+ if (this.foregroundTaskResumeTimer !== null) {
3536
+ clearTimeout(this.foregroundTaskResumeTimer);
3537
+ this.foregroundTaskResumeTimer = null;
3538
+ }
3539
+ }
3540
+
3541
+ private scheduleFloorCheck(
3542
+ currentTimer: ReturnType<typeof setTimeout> | null,
3543
+ silenceMs: number,
3544
+ check: (blockedBy: string | null) => void,
3545
+ ): ReturnType<typeof setTimeout> {
3546
+ if (currentTimer !== null) {
3547
+ clearTimeout(currentTimer);
3548
+ }
3549
+ const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
3550
+ return setTimeout(
3551
+ () => check(this.sessionTurnFloorBlocker()),
3552
+ drainMs + silenceMs,
3553
+ );
3554
+ }
3555
+
3556
+ private scheduleForegroundTaskResume(): void {
3557
+ const task = this.foregroundTaskState;
3558
+ if (
3559
+ task?.phase !== "suspended" ||
3560
+ !task.resumePending ||
3561
+ this.isClosed ||
3562
+ this.state === "failed"
3563
+ ) {
3564
+ this.clearForegroundTaskResumeTimer();
3565
+ return;
3566
+ }
3567
+ const expired = this.foregroundTaskSuspensionExpired(task);
3568
+ if (expired !== null) {
3569
+ this.clearForegroundTask(`suspended_${expired}`);
3570
+ return;
3571
+ }
3572
+ const epoch = task.epoch;
3573
+ this.foregroundTaskResumeTimer = this.scheduleFloorCheck(
3574
+ this.foregroundTaskResumeTimer,
3575
+ this.foregroundTaskResumeSilenceMs,
3576
+ (blockedBy) => {
3577
+ this.foregroundTaskResumeTimer = null;
3578
+ const current = this.foregroundTaskState;
3579
+ if (
3580
+ current?.phase !== "suspended" ||
3581
+ current.epoch !== epoch ||
3582
+ !current.resumePending
3583
+ ) {
3584
+ return;
3585
+ }
3586
+ const currentExpired = this.foregroundTaskSuspensionExpired(current);
3587
+ if (currentExpired !== null) {
3588
+ this.clearForegroundTask(`suspended_${currentExpired}`);
3589
+ return;
3590
+ }
3591
+ if (blockedBy === null) {
3592
+ void this.resumeForegroundTask(current).catch((err: unknown) => {
3593
+ log.warn({ err, epoch }, "Foreground task resume failed");
3594
+ });
3595
+ return;
3596
+ }
3597
+ if (blockedBy === "session_unavailable") {
3598
+ this.clearForegroundTask("session_unavailable");
3599
+ return;
3600
+ }
3601
+ if (blockedBy !== "turn_active") {
3602
+ this.scheduleForegroundTaskResume();
3603
+ }
3604
+ },
3605
+ );
3606
+ }
3607
+
3608
+ private async resumeForegroundTask(
3609
+ task: SuspendedForegroundTask,
3610
+ ): Promise<void> {
3611
+ const current = this.foregroundTaskState;
3612
+ if (
3613
+ current?.phase !== "suspended" ||
3614
+ current.epoch !== task.epoch ||
3615
+ !current.resumePending
3616
+ ) {
3617
+ return;
3618
+ }
3619
+ this.foregroundTaskState = { ...current, resumePending: false };
3620
+ const started = await this.launchAssistantTurn(
3621
+ createSyntheticUtterance(),
3622
+ buildForegroundTaskResumePrompt(current.request),
3623
+ {
3624
+ hiddenPrompt: true,
3625
+ initialLeg: "escalated",
3626
+ foregroundTaskEpoch: current.epoch,
3627
+ },
3628
+ );
3629
+ if (started) {
3630
+ return;
3631
+ }
3632
+ if (this.markForegroundTaskResumePending(current.epoch)) {
3633
+ this.scheduleForegroundTaskResume();
3634
+ }
3635
+ }
3636
+
3284
3637
  // Abort every background continuation this session started and drop its
3285
3638
  // handle. Called on a hard stop (client interrupt / session close), when a
3286
3639
  // newer barge-in supersedes the detached runs, and on a foreground-wins
@@ -3407,50 +3760,53 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3407
3760
  retry = false,
3408
3761
  drainRearms = 0,
3409
3762
  ): void {
3410
- if (this.announcementTimer) {
3411
- clearTimeout(this.announcementTimer);
3412
- }
3413
- // Queued client-side audio is the previous reply still being audible, so
3414
- // the silence window only begins once it has drained.
3415
- const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
3416
- this.announcementTimer = setTimeout(() => {
3417
- this.announcementTimer = null;
3418
- const blockedBy = this.continuationAnnouncementBlocker();
3419
- if (blockedBy === null) {
3420
- void this.announceContinuation().catch((err: unknown) => {
3421
- log.warn({ err }, "Voice duplex continuation announcement failed");
3422
- });
3423
- return;
3424
- }
3425
- log.debug(
3426
- { conversationId: this.conversationId, blockedBy, retry, drainRearms },
3427
- "Voice duplex continuation announcement deferred",
3428
- );
3429
- // A tail that grew while the timer ran (the reply was still streaming
3430
- // TTS when it was armed) is a wait of known length rather than a busy
3431
- // call, so it re-arms against the new deadline without spending the
3432
- // single retry the busy-call blockers get — otherwise a long reply's
3433
- // playback alone outlasts the budget and the announcement is dropped
3434
- // even though the call goes idle right after. The tail only extends
3435
- // while a turn is active, which blocks earlier, so the re-arms converge;
3436
- // the cap bounds the interleaving where a fresh turn's audio lands
3437
- // between two checks.
3438
- if (
3439
- blockedBy === "playback_draining" &&
3440
- drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
3441
- ) {
3442
- this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
3443
- return;
3444
- }
3445
- if (blockedBy === "turn_active") {
3446
- return;
3447
- }
3448
- if (retry || this.pendingAnnouncement === null) {
3449
- this.pendingAnnouncement = null;
3450
- return;
3451
- }
3452
- this.scheduleContinuationAnnouncement(true, drainRearms);
3453
- }, drainMs + this.continuationAnnounceSilenceMs);
3763
+ this.announcementTimer = this.scheduleFloorCheck(
3764
+ this.announcementTimer,
3765
+ this.continuationAnnounceSilenceMs,
3766
+ () => {
3767
+ this.announcementTimer = null;
3768
+ const blockedBy = this.continuationAnnouncementBlocker();
3769
+ if (blockedBy === null) {
3770
+ void this.announceContinuation().catch((err: unknown) => {
3771
+ log.warn({ err }, "Voice duplex continuation announcement failed");
3772
+ });
3773
+ return;
3774
+ }
3775
+ log.debug(
3776
+ {
3777
+ conversationId: this.conversationId,
3778
+ blockedBy,
3779
+ retry,
3780
+ drainRearms,
3781
+ },
3782
+ "Voice duplex continuation announcement deferred",
3783
+ );
3784
+ // A tail that grew while the timer ran (the reply was still streaming
3785
+ // TTS when it was armed) is a wait of known length rather than a busy
3786
+ // call, so it re-arms against the new deadline without spending the
3787
+ // single retry the busy-call blockers get. Otherwise a long reply's
3788
+ // playback alone outlasts the budget and the announcement is dropped
3789
+ // even though the call goes idle right after. The tail only extends
3790
+ // while a turn is active, which blocks earlier, so the re-arms converge;
3791
+ // the cap bounds the interleaving where a fresh turn's audio lands
3792
+ // between two checks.
3793
+ if (
3794
+ blockedBy === "playback_draining" &&
3795
+ drainRearms < CONTINUATION_ANNOUNCE_MAX_DRAIN_REARMS
3796
+ ) {
3797
+ this.scheduleContinuationAnnouncement(retry, drainRearms + 1);
3798
+ return;
3799
+ }
3800
+ if (blockedBy === "turn_active") {
3801
+ return;
3802
+ }
3803
+ if (retry || this.pendingAnnouncement === null) {
3804
+ this.pendingAnnouncement = null;
3805
+ return;
3806
+ }
3807
+ this.scheduleContinuationAnnouncement(true, drainRearms);
3808
+ },
3809
+ );
3454
3810
  }
3455
3811
 
3456
3812
  /**
@@ -3561,11 +3917,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3561
3917
  // down. One line, not narration: the turn is not working, it is waiting,
3562
3918
  // and it says which, in the turn's spoken language, like every other
3563
3919
  // filler phrase.
3564
- this.enqueueFillerPhrase(
3565
- turn,
3566
- approvalPendingPhraseFor(turn.language),
3567
- this.fixedPhraseLanguage(turn, APPROVAL_PENDING_PHRASE_BY_LANGUAGE),
3568
- );
3920
+ if (
3921
+ this.enqueueFillerPhrase(
3922
+ turn,
3923
+ approvalPendingPhraseFor(turn.language),
3924
+ this.fixedPhraseLanguage(turn, APPROVAL_PENDING_PHRASE_BY_LANGUAGE),
3925
+ )
3926
+ ) {
3927
+ turn.progress.noteFloorHolder();
3928
+ }
3569
3929
  }
3570
3930
 
3571
3931
  /** Clear the wait once a decision lands, so the turn narrates normally again. */
@@ -3636,6 +3996,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3636
3996
  ) {
3637
3997
  this.scheduleContinuationAnnouncement();
3638
3998
  }
3999
+ if (
4000
+ this.foregroundTaskState?.phase === "suspended" &&
4001
+ this.foregroundTaskState.resumePending &&
4002
+ !this.isClosed &&
4003
+ this.state !== "failed"
4004
+ ) {
4005
+ this.scheduleForegroundTaskResume();
4006
+ }
3639
4007
  }
3640
4008
 
3641
4009
  // Why the session must not speak up right now, or null when the call is
@@ -3705,6 +4073,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3705
4073
  // own it leaves manual mode blind to a user who is talking right now and
3706
4074
  // has no text yet — hence the captured-audio flag, which manual ingress
3707
4075
  // sets from the first chunk.
4076
+ //
4077
+ // A partial counts only when it has words. Deepgram Flux sends interim
4078
+ // updates through silence too, each an empty partial, and one of those
4079
+ // would otherwise hold the floor until the user next spoke.
3708
4080
  if (
3709
4081
  utterance !== null &&
3710
4082
  !utterance.completed &&
@@ -3712,13 +4084,129 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3712
4084
  utterance.assistantTurnStarted ||
3713
4085
  (utterance.manualAudioCaptured && opts?.ignoreManualCapture !== true) ||
3714
4086
  utterance.finalTranscriptSegments.length > 0 ||
3715
- utterance.latestPartialText !== null)
4087
+ (utterance.latestPartialText?.trim() ?? "").length > 0)
3716
4088
  ) {
3717
4089
  return "utterance_in_flight";
3718
4090
  }
3719
4091
  return null;
3720
4092
  }
3721
4093
 
4094
+ /**
4095
+ * Wait for the fresh frame a look control asks the client for.
4096
+ *
4097
+ * Only for a client that declared `lookFrames`: any other sends no such
4098
+ * frame, and its look keeps the old shape, where the user's next words are
4099
+ * what the frame gets answered on. A newer look replaces an older one still
4100
+ * waiting, and the wait is bounded so a frame that never comes (a camera
4101
+ * that would not open, a share the desktop refused) cannot turn up minutes
4102
+ * later as a reply to nothing.
4103
+ */
4104
+ private awaitLookFrame(action: LookSessionControl): void {
4105
+ if (!this.lookFrames) {
4106
+ return;
4107
+ }
4108
+ this.clearPendingLook();
4109
+ const timer = setTimeout(() => {
4110
+ if (this.pendingLook?.timer !== timer) {
4111
+ return;
4112
+ }
4113
+ this.pendingLook = null;
4114
+ log.info(
4115
+ { conversationId: this.conversationId, action },
4116
+ "Live voice look dropped: no frame arrived",
4117
+ );
4118
+ }, LOOK_FRAME_WAIT_MS);
4119
+ this.pendingLook = { action, armedAtMs: Date.now(), timer };
4120
+ }
4121
+
4122
+ private clearPendingLook(): void {
4123
+ if (this.pendingLook !== null) {
4124
+ clearTimeout(this.pendingLook.timer);
4125
+ this.pendingLook = null;
4126
+ }
4127
+ if (this.lookFollowUpTimer !== null) {
4128
+ clearTimeout(this.lookFollowUpTimer);
4129
+ this.lookFollowUpTimer = null;
4130
+ }
4131
+ }
4132
+
4133
+ /**
4134
+ * The frame a look asked for is in the conversation: answer the look.
4135
+ *
4136
+ * A look frame with no look waiting (the wait ran out, or the session never
4137
+ * armed one) is only a frame, like any other keep.
4138
+ */
4139
+ private lookFrameLanded(): void {
4140
+ const pending = this.pendingLook;
4141
+ if (pending === null || this.isClosed) {
4142
+ return;
4143
+ }
4144
+ clearTimeout(pending.timer);
4145
+ this.pendingLook = null;
4146
+ this.answerLookWhenFloorIsFree(pending, this.turnsLaunched, 0);
4147
+ }
4148
+
4149
+ /**
4150
+ * Start the turn that answers a look, once nothing else holds the floor.
4151
+ *
4152
+ * The acknowledgement ("taking a look") can still be playing when the frame
4153
+ * lands, and the look's own turn can still be clearing, so both are waited
4154
+ * out, as is the user mid-utterance. A turn that starts once the frame is in
4155
+ * the conversation is not: it reads the frame, so answering the look as well
4156
+ * would answer it twice. A turn that started before the frame landed did not
4157
+ * see it, so the look is still answered once that turn is done.
4158
+ *
4159
+ * `turnsAtFrame` is how many turns had launched when the frame landed.
4160
+ */
4161
+ private answerLookWhenFloorIsFree(
4162
+ look: PendingLook,
4163
+ turnsAtFrame: number,
4164
+ rearms: number,
4165
+ ): void {
4166
+ if (this.lookFollowUpTimer !== null) {
4167
+ clearTimeout(this.lookFollowUpTimer);
4168
+ this.lookFollowUpTimer = null;
4169
+ }
4170
+ const { action, armedAtMs } = look;
4171
+ // A turn launched since the frame landed read it already. Speech that
4172
+ // never became a turn (a cough, noise that transcribed to nothing) is only
4173
+ // waited out.
4174
+ const blockedBy =
4175
+ this.turnsLaunched > turnsAtFrame
4176
+ ? "turn_since_look"
4177
+ : this.sessionTurnFloorBlocker();
4178
+ if (blockedBy === null) {
4179
+ void this.launchAssistantTurn(
4180
+ createSyntheticUtterance(),
4181
+ LOOK_FOLLOW_UP_CONTENT,
4182
+ { lookFollowUp: action, hiddenPrompt: true },
4183
+ ).catch((err: unknown) => {
4184
+ log.warn(
4185
+ { err, conversationId: this.conversationId, action },
4186
+ "Live voice look follow-up failed to start",
4187
+ );
4188
+ });
4189
+ return;
4190
+ }
4191
+ const waitable =
4192
+ blockedBy !== "turn_since_look" && blockedBy !== "session_unavailable";
4193
+ if (!waitable || Date.now() - armedAtMs >= LOOK_ANSWER_DEADLINE_MS) {
4194
+ log.info(
4195
+ { conversationId: this.conversationId, action, blockedBy, rearms },
4196
+ "Live voice look follow-up skipped",
4197
+ );
4198
+ return;
4199
+ }
4200
+ const drainMs = Math.max(0, this.assistantPlaybackTailUntilMs - Date.now());
4201
+ this.lookFollowUpTimer = setTimeout(
4202
+ () => {
4203
+ this.lookFollowUpTimer = null;
4204
+ this.answerLookWhenFloorIsFree(look, turnsAtFrame, rearms + 1);
4205
+ },
4206
+ Math.max(drainMs, LOOK_FOLLOW_UP_REARM_MS),
4207
+ );
4208
+ }
4209
+
3722
4210
  // Speak the finished continuation's result on a turn the session starts
3723
4211
  // itself. The turn rides a synthetic cycle rather than the armed VAD one —
3724
4212
  // `currentUtterance` belongs to the capture loop, and overwriting it would
@@ -4172,13 +4660,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4172
4660
  if (!alreadyReleased) {
4173
4661
  void this.releaseUtterance();
4174
4662
  }
4175
- if (
4176
- this.frontModelConfig.progress.enabled &&
4177
- this.streamTtsAudio &&
4178
- this.progressNarrator
4179
- ) {
4180
- this.armProgressIdleTimer(turn);
4181
- }
4663
+ turn.progress.arm();
4182
4664
  return true;
4183
4665
  }
4184
4666
 
@@ -4914,8 +5396,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
4914
5396
  // the next turn is now stale (the interrupted utterance may be discarded
4915
5397
  // without ever reaching finalizePendingUtterance).
4916
5398
  this.pendingInterruptedRequest = null;
5399
+ this.clearForegroundTask("client_interrupt");
4917
5400
  // ...and it hard-stops any detached background continuations.
4918
5401
  this.abortDetachedRuns({ reason: "client_interrupt" });
5402
+ // ...and a look still waiting to be answered: the user stopped the call
5403
+ // talking, and a reply starting on its own a moment later is not a stop.
5404
+ this.clearPendingLook();
4919
5405
  const utterance = this.currentUtterance;
4920
5406
  this.stopSessionTranscriber();
4921
5407
  if (utterance) {
@@ -5137,6 +5623,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5137
5623
  // Set on an announcement turn: the finished continuation this turn exists
5138
5624
  // to deliver. Its answer goes in the control prompt, not in `content`.
5139
5625
  continuationDelivery?: ContinuationDelivery | null;
5626
+ // Set on the turn that answers a look: which look. Its instruction goes
5627
+ // in the control prompt, not in `content`.
5628
+ lookFollowUp?: LookSessionControl;
5140
5629
  // Unified front-door: dispatch without releasing the utterance. The
5141
5630
  // thinking frame and floor-holding timers are deferred until the leg's
5142
5631
  // leading verdict commits the turn (see commitSpeculativeTurn); a hold
@@ -5146,6 +5635,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5146
5635
  // the row persists and drives the turn but never renders: no echo, and
5147
5636
  // `/messages` filters it after a reload.
5148
5637
  hiddenPrompt?: boolean;
5638
+ // Synthetic host-task resumes start on the tool-capable profile without
5639
+ // passing through the front door.
5640
+ initialLeg?: VoiceRoutingLeg;
5641
+ // The suspended foreground task this synthetic turn resumes.
5642
+ foregroundTaskEpoch?: number;
5149
5643
  },
5150
5644
  ): Promise<boolean> {
5151
5645
  utterance.assistantTurnStarted = true;
@@ -5161,6 +5655,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5161
5655
  this.startMetricsTurnIfNeeded(utterance, turnId);
5162
5656
  this.markAssistantDispatch(utterance, turnId);
5163
5657
  const abortController = new AbortController();
5658
+ const suspendedForegroundTask =
5659
+ this.foregroundTaskState?.phase === "suspended"
5660
+ ? this.foregroundTaskState
5661
+ : null;
5662
+ const foregroundTaskEpoch =
5663
+ opts?.foregroundTaskEpoch ??
5664
+ (opts?.hiddenPrompt !== true
5665
+ ? (suspendedForegroundTask?.epoch ?? null)
5666
+ : null);
5164
5667
  const activeTurn: ActiveAssistantTurn = {
5165
5668
  token,
5166
5669
  turnId,
@@ -5169,23 +5672,46 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5169
5672
  abortController,
5170
5673
  handle: null,
5171
5674
  launchedAtMs: Date.now(),
5172
- progress: {
5173
- ops: [],
5174
- opsSinceNarration: 0,
5175
- // Equal epochs at launch: a turn that has done nothing observable yet
5176
- // has nothing to narrate, so the idle trigger waits for tool activity
5177
- // or the maxSilenceMs heartbeat.
5178
- stateEpoch: 0,
5179
- narratedEpoch: 0,
5180
- updatesSpoken: 0,
5181
- lastFloorHolderAtMs: null,
5182
- lastAudibleAtMs: Date.now(),
5183
- idleTimer: null,
5184
- narrationInFlight: false,
5185
- },
5675
+ progress: createProgressCadence({
5676
+ config: progressConfigForCadence(
5677
+ this.frontModelConfig.progress,
5678
+ this.progressCadence,
5679
+ ),
5680
+ // Without TTS there is nothing to speak (the idle trigger's static
5681
+ // fallback still needs a generation attempt to fall back from).
5682
+ narrator: this.streamTtsAudio ? this.progressNarrator : null,
5683
+ nextFallbackPhraseIndex: () => this.progressPhraseCounter++,
5684
+ host: {
5685
+ turnId,
5686
+ launchedAtMs: Date.now(),
5687
+ signal: abortController.signal,
5688
+ // Narration may speak whenever the live turn is audibly silent and
5689
+ // has not completed. Escalated turns qualify too: the bridge phrase
5690
+ // holds the floor only while its audio plays (covered by the
5691
+ // audible-idle check), and the strong leg's tool loops are the
5692
+ // longest dead air in the system. Nothing is in flight while a
5693
+ // decision is pending, so every phrase narration has would be a lie
5694
+ // about who the call is waiting on: the turn says so once, when it
5695
+ // starts waiting, and is quiet after that.
5696
+ canNarrate: () =>
5697
+ this.isActiveAssistantTurn(token) &&
5698
+ !activeTurn.assistantCompleted &&
5699
+ activeTurn.pendingApproval === null,
5700
+ isAudioIdle: () => this.turnAudioIdle(activeTurn),
5701
+ playbackTailUntilMs: () => this.assistantPlaybackTailUntilMs,
5702
+ transcriptSoFar: () =>
5703
+ activeTurn.utterance.finalTranscriptSegments.join(" ").trim(),
5704
+ language: () => activeTurn.language,
5705
+ deltaEpoch: () => activeTurn.deltaEpoch,
5706
+ speak: (phrase, language) =>
5707
+ this.enqueueFillerPhrase(activeTurn, phrase, language),
5708
+ onNarrationSpoken: () => this.markProgressSpoken(activeTurn),
5709
+ },
5710
+ }),
5186
5711
  assistantCompleted: false,
5187
5712
  ttsDone: false,
5188
5713
  minimizeRequested: false,
5714
+ sessionControlRequested: null,
5189
5715
  activityLabel: "",
5190
5716
  publishedApprovalRequestId: null,
5191
5717
  pendingApproval: null,
@@ -5208,9 +5734,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5208
5734
  consumedAnnouncement: pending?.announcement ?? null,
5209
5735
  pendingContextStopGeneration: this.detachStopGeneration,
5210
5736
  continuationDelivery: opts?.continuationDelivery ?? null,
5737
+ lookFollowUp: opts?.lookFollowUp ?? null,
5211
5738
  hiddenPrompt: opts?.hiddenPrompt === true,
5739
+ foregroundTaskEpoch,
5740
+ foregroundTaskSuspensionSnapshot:
5741
+ suspendedForegroundTask?.epoch === foregroundTaskEpoch
5742
+ ? { ...suspendedForegroundTask }
5743
+ : null,
5212
5744
  deltaEpoch: 0,
5213
- escalationHandedOff: false,
5745
+ frontDoor: null,
5214
5746
  ttsBuffer: "",
5215
5747
  spokenSegments: 0,
5216
5748
  spokenChars: 0,
@@ -5223,6 +5755,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5223
5755
  assistantAudioMimeType: "audio/pcm",
5224
5756
  };
5225
5757
  this.activeAssistantTurn = activeTurn;
5758
+ this.turnsLaunched += 1;
5226
5759
 
5227
5760
  // A speculative turn defers the thinking frame and both floor-holding
5228
5761
  // timers to commitSpeculativeTurn: until the verdict arrives, the pause
@@ -5268,22 +5801,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5268
5801
  // cadence, wherever in the turn it occurs; without TTS or a narrator there
5269
5802
  // is nothing to speak (the idle trigger's static fallback still needs a
5270
5803
  // generation attempt to fall back from).
5271
- if (
5272
- this.frontModelConfig.progress.enabled &&
5273
- this.streamTtsAudio &&
5274
- this.progressNarrator &&
5275
- !opts?.speculative
5276
- ) {
5277
- this.armProgressIdleTimer(activeTurn);
5804
+ if (!opts?.speculative) {
5805
+ activeTurn.progress.arm();
5278
5806
  }
5279
5807
 
5280
- // Front-door leg: a fast model fronts every turn (the `voiceFrontDoor`
5281
- // call site pins it) and may hand off to a quality leg on the escalate
5282
- // verdict.
5808
+ const initialLeg = opts?.initialLeg ?? "front-door";
5283
5809
  const started = await this.startAssistantLeg(activeTurn, {
5284
5810
  content,
5285
- routingLeg: "front-door",
5286
- frontDoor: true,
5811
+ routingLeg: initialLeg,
5812
+ ...(initialLeg === "front-door"
5813
+ ? { frontDoor: true }
5814
+ : { directEscalated: true }),
5287
5815
  // Only this leg: an escalated leg persists its own continuation row, and
5288
5816
  // the frame belongs to the row carrying the user's words.
5289
5817
  ...(activeTurn.turnAttachmentId
@@ -5328,6 +5856,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5328
5856
  overrideProfile?: string;
5329
5857
  routingLeg?: VoiceRoutingLeg;
5330
5858
  frontDoor?: boolean;
5859
+ directEscalated?: boolean;
5331
5860
  spokenEscalationBridge?: string;
5332
5861
  attachments?: readonly string[];
5333
5862
  },
@@ -5337,17 +5866,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5337
5866
  }
5338
5867
  const { token, utterance, turnId } = activeTurn;
5339
5868
 
5340
- // `rawText` accumulates this leg's full stream. A front-door leg starts
5341
- // in `deciding` until its leading tokens classify as hold / escalate /
5342
- // answer: an answer flushes through the shared marker holdback, while an
5343
- // escalation buffers the post-verdict stream into `bridgeRaw` until the
5344
- // bridge is complete, then hands off. A default/escalated leg flushes
5345
- // every delta through the same holdback, so a stray control marker from
5346
- // the main model is stripped instead of spoken.
5869
+ // `rawText` accumulates this leg's spoken stream. A front-door leg runs
5870
+ // through the shared coordinator, which reads its stream through the
5871
+ // verdict machine: released answer text flushes through the shared
5872
+ // marker holdback, while an escalation buffers the post-verdict stream
5873
+ // until the capped bridge is complete, then hands off. A default or
5874
+ // escalated leg flushes every delta through the same holdback, so a
5875
+ // stray control marker from the main model is stripped instead of
5876
+ // spoken.
5347
5877
  let rawText = "";
5348
- let frontDoorStage: "deciding" | "answer" | "bridging" | "handedOff" =
5349
- "deciding";
5350
- let bridgeRaw = "";
5351
5878
 
5352
5879
  const emitLegText = (chunk: string): void => {
5353
5880
  if (chunk.length === 0) {
@@ -5370,20 +5897,55 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5370
5897
  this.bufferAssistantTextForTts(token, chunk);
5371
5898
  };
5372
5899
 
5373
- const flushLegText = createControlMarkerHoldback(activeTurn, emitLegText);
5374
-
5375
- // Hand off once enough of the post-verdict stream has arrived to cap
5376
- // the bridge (sentence terminator or hard cap). Until then nothing is
5377
- // spoken — the bridge goes out in one piece at hand-off, so the audio,
5378
- // the persisted row, and the phrase quoted to the escalated leg are all
5379
- // the same capped text.
5380
- const maybeHandOffBridge = (): void => {
5381
- if (!isEscalationBridgeComplete(bridgeRaw)) {
5382
- return;
5383
- }
5384
- frontDoorStage = "handedOff";
5385
- this.escalateTurn(activeTurn, capEscalationBridge(bridgeRaw));
5386
- };
5900
+ const flushLegText = createControlMarkerHoldback(emitLegText);
5901
+
5902
+ const coordinator = leg.frontDoor
5903
+ ? createFrontDoorLegCoordinator({
5904
+ // The hold branch is only classifiable while the leg is
5905
+ // speculative: its decision rule is the only one that teaches the
5906
+ // hold token.
5907
+ holdEnabled:
5908
+ activeTurn.speculativePending && activeTurn.speculativeHoldAllowed,
5909
+ host: {
5910
+ isLive: () => !activeTurn.finalized,
5911
+ language: () => activeTurn.language,
5912
+ progress: {
5913
+ // The verdict deadline goes with the narration timer: the
5914
+ // hand-off settles the verdict.
5915
+ clear: () => this.clearFillerTimers(activeTurn),
5916
+ noteFloorHolder: () => activeTurn.progress.noteFloorHolder(),
5917
+ arm: () => activeTurn.progress.arm(),
5918
+ },
5919
+ // Hold discards a speculative turn (mid-thought pause, keep
5920
+ // listening); escalate and answer both commit it: utterance
5921
+ // release, thinking frame, and timers all happen inside
5922
+ // commitSpeculativeTurn.
5923
+ onHold: () => {
5924
+ void this.holdSpeculativeTurn(activeTurn);
5925
+ },
5926
+ commit: () =>
5927
+ !activeTurn.speculativePending ||
5928
+ this.commitSpeculativeTurn(activeTurn),
5929
+ onAnswerText: (text) => {
5930
+ rawText += text;
5931
+ flushLegText(rawText);
5932
+ },
5933
+ abortLeg: () => {
5934
+ activeTurn.handle?.abort();
5935
+ activeTurn.handle = null;
5936
+ },
5937
+ speakBridge: (bridge) =>
5938
+ this.speakEscalationBridge(activeTurn, bridge),
5939
+ startEscalatedLeg: (escalated) => {
5940
+ this.noteEscalatedForegroundTask(activeTurn);
5941
+ void this.startAssistantLeg(activeTurn, escalated);
5942
+ },
5943
+ },
5944
+ })
5945
+ : null;
5946
+ if (coordinator !== null) {
5947
+ activeTurn.frontDoor = coordinator;
5948
+ }
5387
5949
 
5388
5950
  try {
5389
5951
  // Latched before the await, not after: this flag only decides whether the
@@ -5418,6 +5980,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5418
5980
  // as telemetry instead, on `voiceTelemetry` below.
5419
5981
  userMessageInterface: "macos",
5420
5982
  assistantMessageInterface: "macos",
5983
+ ...(this.context.startFrame.client === "macos"
5984
+ ? { macosDesktopSession: true }
5985
+ : {}),
5421
5986
  voiceTelemetry: {
5422
5987
  sessionId: this.context.sessionId,
5423
5988
  ...(this.context.startFrame.client
@@ -5427,9 +5992,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5427
5992
  ? { entry: this.context.startFrame.entry }
5428
5993
  : {}),
5429
5994
  },
5430
- voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
5431
- ...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
5432
- }),
5995
+ voiceControlPrompt: buildVoiceControlPrompt(
5996
+ activeTurn,
5997
+ {
5998
+ ...(leg.frontDoor !== undefined
5999
+ ? { frontDoor: leg.frontDoor }
6000
+ : {}),
6001
+ },
6002
+ this.sessionControls,
6003
+ { lookFrames: this.lookFrames },
6004
+ ),
5433
6005
  onApprovalPending: (requestId) => {
5434
6006
  this.revealRoomForPendingApproval(activeTurn, requestId);
5435
6007
  },
@@ -5454,6 +6026,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5454
6026
  ...(leg.spokenEscalationBridge != null
5455
6027
  ? { spokenEscalationBridge: leg.spokenEscalationBridge }
5456
6028
  : {}),
6029
+ ...(leg.directEscalated === true ? { directEscalated: true } : {}),
5457
6030
  // A speculative front-door leg's decision rule includes the hold
5458
6031
  // branch so its leading tokens can be the hold verdict — but only
5459
6032
  // on the utterance's FIRST dispatch. Extension replays (the
@@ -5470,53 +6043,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5470
6043
  if (!this.isForwardingAssistantText(token)) {
5471
6044
  return;
5472
6045
  }
5473
- if (leg.frontDoor) {
5474
- rawText += msg.text;
5475
- if (frontDoorStage === "handedOff") {
5476
- return;
5477
- }
5478
- if (frontDoorStage === "bridging") {
5479
- bridgeRaw += msg.text;
5480
- maybeHandOffBridge();
5481
- return;
5482
- }
6046
+ if (!leg.frontDoor) {
6047
+ this.claimSuspendedForegroundTask(activeTurn);
6048
+ }
6049
+ if (coordinator !== null) {
5483
6050
  // Verdict-first: the leg's leading tokens decide the turn's
5484
- // fate. Hold discards a speculative turn (mid-thought pause,
5485
- // keep listening); escalate and answer both commit it —
5486
- // utterance release, thinking frame, and timers all happen
5487
- // inside commitSpeculativeTurn. The hold branch is only
5488
- // classifiable while the leg is speculative (its decision
5489
- // rule is the only one that teaches the hold token).
5490
- if (frontDoorStage === "deciding") {
5491
- const verdict = classifyFrontDoorLeading(
5492
- rawText.trimStart(),
5493
- activeTurn.speculativePending &&
5494
- activeTurn.speculativeHoldAllowed,
5495
- );
5496
- if (verdict === "pending") {
5497
- return;
5498
- }
5499
- if (verdict === "hold") {
5500
- void this.holdSpeculativeTurn(activeTurn);
5501
- return;
5502
- }
5503
- if (
5504
- activeTurn.speculativePending &&
5505
- !this.commitSpeculativeTurn(activeTurn)
5506
- ) {
5507
- return;
5508
- }
5509
- if (verdict === "escalate") {
5510
- frontDoorStage = "bridging";
5511
- bridgeRaw = rawText
5512
- .trimStart()
5513
- .slice(ESCALATE_VERDICT_TOKEN.length);
5514
- maybeHandOffBridge();
5515
- return;
5516
- }
5517
- frontDoorStage = "answer";
5518
- }
5519
- flushLegText(rawText);
6051
+ // fate, and the coordinator acts on them. The bridge hands
6052
+ // off in one piece once the machine caps it, so the audio,
6053
+ // the persisted row, and the phrase quoted to the escalated
6054
+ // leg are all the same text.
6055
+ coordinator.push(msg.text);
5520
6056
  return;
5521
6057
  }
5522
6058
  // Defensive: speculative legs are always front-door today, but a
@@ -5545,6 +6081,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5545
6081
  ) {
5546
6082
  return;
5547
6083
  }
6084
+ if (!leg.frontDoor) {
6085
+ this.claimSuspendedForegroundTask(current);
6086
+ }
5548
6087
  // A speculative leg that finished without a single delta (empty
5549
6088
  // output, provider hiccup) carries no verdict — fail open to a
5550
6089
  // committed turn so the utterance releases and finalizes like a
@@ -5552,27 +6091,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5552
6091
  if (current.speculativePending) {
5553
6092
  this.commitSpeculativeTurn(current);
5554
6093
  }
5555
- // A front-door leg that stopped mid-bridge (a bare escalate
5556
- // verdict, or a holding phrase with no sentence terminator)
5557
- // hands off now with whatever arrived; the canned fallback
5558
- // covers an empty bridge. A cancellation mid-bridge falls
5559
- // through to normal cancelled finalization instead — a dead
5560
- // turn must not spawn an escalated leg.
5561
- if (
5562
- leg.frontDoor &&
5563
- frontDoorStage === "bridging" &&
5564
- msg.type === "message_complete" &&
5565
- !current.escalationHandedOff
5566
- ) {
5567
- frontDoorStage = "handedOff";
5568
- this.escalateTurn(current, capEscalationBridge(bridgeRaw));
5569
- return;
5570
- }
5571
- // A front-door leg that handed off is finished; the escalated leg
5572
- // drives completion. The front-door leg's own trailing completion
5573
- // (including the generation_cancelled from its abort) is a no-op.
5574
- if (leg.frontDoor && current.escalationHandedOff) {
5575
- return;
6094
+ // A front-door leg that stopped mid-bridge hands off now with
6095
+ // whatever arrived. A cancellation mid-bridge falls through to
6096
+ // normal cancelled finalization instead: a dead turn must not
6097
+ // spawn an escalated leg. A front-door leg that handed off is
6098
+ // finished; the escalated leg drives completion, so this leg's
6099
+ // own trailing completion (including the generation_cancelled
6100
+ // from its abort) is a no-op.
6101
+ if (coordinator !== null) {
6102
+ if (msg.type === "message_complete") {
6103
+ coordinator.complete();
6104
+ }
6105
+ if (coordinator.handedOff) {
6106
+ return;
6107
+ }
5576
6108
  }
5577
6109
  // A held "[…"-tail that never completed a marker is real text —
5578
6110
  // force-flush it before assistantCompleted closes the TTS buffer
@@ -5581,6 +6113,34 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5581
6113
  if (!leg.frontDoor && msg.type === "message_complete") {
5582
6114
  flushLegText(rawText, { force: true });
5583
6115
  }
6116
+ // Read off the leg that finished the reply: a front-door answer
6117
+ // or the escalated leg. A handed-off front-door leg returned
6118
+ // above, so its holding phrase can never end a call.
6119
+ if (msg.type === "message_complete") {
6120
+ const request = requestedSessionControl(
6121
+ rawText,
6122
+ this.sessionControls,
6123
+ );
6124
+ if (request?.action === "updates") {
6125
+ // The session's own control: nothing to send, and nothing to
6126
+ // wait for, since it shapes turns that have not started yet.
6127
+ this.progressCadence = request.cadence;
6128
+ log.info(
6129
+ { turnId, cadence: request.cadence },
6130
+ "Live voice progress cadence changed",
6131
+ );
6132
+ } else if (request?.action === "task_stop") {
6133
+ this.stopOutstandingInterruptedWork("spoken_task_stop");
6134
+ } else {
6135
+ if (request?.action === "end") {
6136
+ this.clearForegroundTask("call_end_requested");
6137
+ }
6138
+ current.sessionControlRequested = request;
6139
+ }
6140
+ }
6141
+ if (leg.frontDoor) {
6142
+ this.queueForegroundTaskResume(current);
6143
+ }
5584
6144
  current.assistantCompleted = true;
5585
6145
  if (msg.type === "generation_cancelled") {
5586
6146
  void this.finalizeAssistantTurn(
@@ -5615,6 +6175,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5615
6175
  if (current?.token !== token) {
5616
6176
  return;
5617
6177
  }
6178
+ const effectiveHostTool = hostInteractiveToolName(
6179
+ toolName,
6180
+ detail?.input,
6181
+ detail?.allowedToolNames,
6182
+ );
6183
+ if (effectiveHostTool !== null) {
6184
+ this.noteHostToolStarted(current, effectiveHostTool);
6185
+ }
5618
6186
  // Foreground wins the workspace: the continuation runs with full
5619
6187
  // subagent abilities (it can write files, run commands), so the
5620
6188
  // moment a live turn starts a consequential tool the two could
@@ -5649,57 +6217,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5649
6217
  toolName,
5650
6218
  });
5651
6219
  }
5652
- // The op counts toward the narration threshold on start (not
5653
- // completion) so a burst of slow tools still trips the ops
5654
- // trigger while they run.
5655
- current.progress.ops.push({
5656
- toolName,
5657
- ...(detail?.toolUseId !== undefined
5658
- ? { toolUseId: detail.toolUseId }
5659
- : {}),
5660
- startedAtMs: Date.now(),
5661
- });
5662
- current.progress.opsSinceNarration += 1;
5663
- current.progress.stateEpoch += 1;
6220
+ current.progress.toolStarted(toolName, detail?.toolUseId);
5664
6221
  this.refreshActivity(current);
5665
6222
  log.debug({ turnId, toolName }, "Live voice turn started tool use");
5666
- this.maybeNarrateProgress(current, "ops");
5667
6223
  },
5668
6224
  tool_result: (event) => {
5669
6225
  const current = this.activeAssistantTurn;
5670
6226
  if (current?.token !== token) {
5671
6227
  return;
5672
6228
  }
5673
- // Match by toolUseId when present; otherwise the last-started
5674
- // incomplete op with the same name (parallel same-name ops
5675
- // resolve newest-first).
5676
- const op =
5677
- (event.toolUseId !== undefined
5678
- ? current.progress.ops.find(
5679
- (o) => o.toolUseId === event.toolUseId,
5680
- )
5681
- : undefined) ??
5682
- findLastIncompleteOp(current.progress.ops, event.toolName);
5683
- // A long-running op finishing is the beat the user has been
5684
- // waiting through: it narrates immediately rather than waiting for
5685
- // `opsThreshold` more ops, which on a one-slow-tool turn never
5686
- // arrive. Short ops stay on the ops trigger — narrating every
5687
- // quick lookup is the chatter this cadence exists to avoid.
5688
- let trigger: "ops" | "op_complete" = "ops";
5689
- if (op) {
5690
- op.completedAtMs = Date.now();
5691
- if (event.isError !== undefined) {
5692
- op.isError = event.isError;
5693
- }
5694
- op.resultPreview = event.resultPreview;
5695
- if (
5696
- op.completedAtMs - op.startedAtMs >=
5697
- this.frontModelConfig.progress.longOpMs
5698
- ) {
5699
- trigger = "op_complete";
5700
- }
5701
- }
5702
- current.progress.stateEpoch += 1;
6229
+ current.progress.toolFinished(event);
5703
6230
  // Showing a surface implies revealing it: the room is a
5704
6231
  // full-screen overlay, so a surface rendered behind it is a
5705
6232
  // surface nobody sees.
@@ -5722,7 +6249,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5722
6249
  }
5723
6250
  }
5724
6251
  this.refreshActivity(current);
5725
- this.maybeNarrateProgress(current, trigger);
5726
6252
  },
5727
6253
  },
5728
6254
  onError: (message) => {
@@ -5743,7 +6269,15 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5743
6269
  if (currentTurn?.token !== token) {
5744
6270
  return;
5745
6271
  }
6272
+ const foregroundTaskResumeRearmed =
6273
+ !leg.frontDoor &&
6274
+ !currentTurn.discardRequested &&
6275
+ !currentTurn.abortController.signal.aborted &&
6276
+ this.rearmForegroundTaskAfterLegFailure(currentTurn);
5746
6277
  await this.finalizeAssistantTurn(currentTurn, "cancelled", "error");
6278
+ if (foregroundTaskResumeRearmed) {
6279
+ this.scheduleForegroundTaskResume();
6280
+ }
5747
6281
  })();
5748
6282
  },
5749
6283
  });
@@ -5779,9 +6313,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5779
6313
  this.clearActiveAssistantTurn(token);
5780
6314
  return true;
5781
6315
  }
6316
+ if (!leg.frontDoor) {
6317
+ this.claimSuspendedForegroundTask(current);
6318
+ }
5782
6319
  // The front-door leg may have handed off before its handle resolved;
5783
6320
  // abort it rather than exposing it as the turn's live handle.
5784
- if (leg.frontDoor && current.escalationHandedOff) {
6321
+ if (coordinator?.handedOff === true) {
5785
6322
  handle.abort();
5786
6323
  return true;
5787
6324
  }
@@ -5793,8 +6330,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5793
6330
  return false;
5794
6331
  }
5795
6332
 
6333
+ const foregroundTaskResumeRearmed =
6334
+ !leg.frontDoor &&
6335
+ leg.directEscalated !== true &&
6336
+ !activeTurn.discardRequested &&
6337
+ !activeTurn.abortController.signal.aborted &&
6338
+ this.rearmForegroundTaskAfterLegFailure(activeTurn);
5796
6339
  this.clearFillerTimers(activeTurn);
5797
6340
  this.clearActiveAssistantTurn(token);
6341
+ if (foregroundTaskResumeRearmed) {
6342
+ this.scheduleForegroundTaskResume();
6343
+ }
5798
6344
  await this.sendFrame({
5799
6345
  type: "error",
5800
6346
  code: LiveVoiceProtocolErrorCode.InvalidField,
@@ -5809,37 +6355,17 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5809
6355
  }
5810
6356
 
5811
6357
  /**
5812
- * Hand the turn from the front-door leg to the strong "escalated" leg after
5813
- * the escalate verdict. Speaks the capped bridge (the leg's own post-verdict
5814
- * holding phrase, or the canned fallback when that was too short) in one
5815
- * piece and force-flushes it so it plays during the strong-model call, then
5816
- * starts the escalated leg on the same ActiveAssistantTurn. Idempotent.
6358
+ * Speak the escalation bridge so the strong-model call has no dead air.
6359
+ * The model's own bridge is real assistant speech (captions + TTS); the
6360
+ * canned fallback stays audio-only, matching the persisted-row hygiene (a
6361
+ * deleted row for a bridge the model never produced).
5817
6362
  */
5818
- private escalateTurn(
6363
+ private speakEscalationBridge(
5819
6364
  activeTurn: ActiveAssistantTurn,
5820
- cappedBridge: string,
6365
+ bridge: SpokenEscalationBridge,
5821
6366
  ): void {
5822
- if (activeTurn.escalationHandedOff || activeTurn.finalized) {
5823
- return;
5824
- }
5825
- activeTurn.escalationHandedOff = true;
5826
- // The escalation bridge below holds the floor, so pending progress
5827
- // narration would only stack a second filler on top of it.
5828
- this.clearFillerTimers(activeTurn);
5829
- // Abort the front-door leg so a model that keeps generating past the
5830
- // bridge cap adds no latency before the escalated leg starts.
5831
- activeTurn.handle?.abort();
5832
- activeTurn.handle = null;
5833
-
5834
- // Speak the bridge so the strong-model call has no dead air. The model's
5835
- // own bridge is real assistant speech (captions + TTS); the canned
5836
- // fallback stays audio-only, matching the persisted-row hygiene (a
5837
- // deleted row for a bridge the model never produced).
5838
- const usesFallbackBridge = cappedBridge.length < MIN_SPOKEN_BRIDGE_CHARS;
5839
- const spokenBridge = usesFallbackBridge
5840
- ? fallbackEscalationBridgeFor(activeTurn.language)
5841
- : cappedBridge;
5842
- if (!usesFallbackBridge) {
6367
+ const { spokenBridge, usesFallback, language } = bridge;
6368
+ if (!usesFallback) {
5843
6369
  this.markFirstAssistantDelta(activeTurn.utterance, activeTurn.turnId);
5844
6370
  this.markAssistantDelta(activeTurn);
5845
6371
  void this.sendFrame(
@@ -5851,46 +6377,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5851
6377
  // otherwise sit buffered until a sentence boundary and leave the
5852
6378
  // caller in silence during the escalated model's call.
5853
6379
  this.flushTtsBuffer(activeTurn.token, true);
5854
- } else {
5855
- // The canned bridge is a fixed localized-table phrase, enqueued
5856
- // directly (it is already one complete sentence) so the segment can
5857
- // carry the "en" override when the table lacks the turn's language.
5858
- const speakable = sanitizeForTts(spokenBridge).trim();
5859
- if (speakable.length > 0) {
5860
- this.enqueueTtsSegment(activeTurn.token, speakable, {
5861
- language: this.fixedPhraseLanguage(
5862
- activeTurn,
5863
- FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
5864
- ),
5865
- });
5866
- }
6380
+ return;
5867
6381
  }
5868
-
5869
- // No overrideProfile here: the bridge pins the escalated leg to the
5870
- // conversation's own profile, the model the caller's typed turns already
5871
- // run on (see voice-triage-escalate.ts). The bridge phrase the caller
5872
- // just heard is handed along so the escalated continuation rule can
5873
- // quote it and ban a re-announcing echo ("Let me check…" twice in a
5874
- // row).
5875
- void this.startAssistantLeg(activeTurn, {
5876
- content: ESCALATION_CONTINUATION_CONTENT,
5877
- routingLeg: "escalated",
5878
- spokenEscalationBridge: spokenBridge,
5879
- });
5880
-
5881
- // Escalated legs run the slowest work in the system (strong-model
5882
- // thinking + tool loops), and the bridge only covers the first couple of
5883
- // seconds of it — exactly the dead air progress narration exists for.
5884
- // Re-arm the idle narration timer (cleared above with the ack): its
5885
- // audible-silence gating means nothing speaks until the bridge audio has
5886
- // fully drained plus a whole idle interval. Acks stay suppressed
5887
- // post-handoff — the bridge already served that role.
5888
- if (
5889
- this.frontModelConfig.progress.enabled &&
5890
- this.streamTtsAudio &&
5891
- this.progressNarrator
5892
- ) {
5893
- this.armProgressIdleTimer(activeTurn);
6382
+ // The canned bridge is a fixed localized-table phrase, enqueued
6383
+ // directly (it is already one complete sentence) so the segment can
6384
+ // carry the "en" override when the table lacks the turn's language.
6385
+ const speakable = sanitizeForTts(spokenBridge).trim();
6386
+ if (speakable.length > 0) {
6387
+ this.enqueueTtsSegment(activeTurn.token, speakable, { language });
5894
6388
  }
5895
6389
  }
5896
6390
 
@@ -5967,61 +6461,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5967
6461
  turn.deltaEpoch += 1;
5968
6462
  }
5969
6463
 
5970
- // Arms (or re-arms) the dead-air narration timer. The countdown measures
5971
- // audible silence — time since the turn's audio last (estimatedly) reached
5972
- // the user's ears — not time since launch, so it covers mid-turn silences
5973
- // for the whole turn. On expiry with audio still pending, or with the
5974
- // silence not yet a full interval old, it re-arms for the remainder; only a
5975
- // full interval of audible silence reaches the narration gatekeeper. The
5976
- // interval is a polling cadence, not a speaking cadence: most ticks find
5977
- // nothing new to report and stay quiet, so what the user hears follows the
5978
- // turn's tool activity (with `maxSilenceMs` as the heartbeat ceiling).
5979
- private armProgressIdleTimer(
5980
- turn: ActiveAssistantTurn,
5981
- delayMs?: number,
5982
- ): void {
5983
- const { token } = turn;
5984
- this.clearProgressIdleTimer(turn);
5985
- turn.progress.idleTimer = setTimeout(() => {
5986
- turn.progress.idleTimer = null;
5987
- if (!this.isActiveAssistantTurn(token) || turn.assistantCompleted) {
5988
- return;
5989
- }
5990
- if (!this.turnAudioIdle(turn)) {
5991
- // Audio is buffered, queued, or still playing: a fresh silence can
5992
- // only be a full interval old one interval from now.
5993
- this.armProgressIdleTimer(turn);
5994
- return;
5995
- }
5996
- const remaining = this.progressIdleDeadlineMs(turn) - Date.now();
5997
- if (remaining > 0) {
5998
- this.armProgressIdleTimer(turn, remaining);
5999
- return;
6000
- }
6001
- this.maybeNarrateProgress(turn, "idle");
6002
- this.armProgressIdleTimer(turn);
6003
- }, delayMs ?? this.frontModelConfig.progress.idleIntervalMs);
6004
- }
6005
-
6006
- // Wall-clock instant the current audible silence turns a full interval old.
6007
- private progressIdleDeadlineMs(turn: ActiveAssistantTurn): number {
6008
- return (
6009
- this.progressSilenceSinceMs(turn) +
6010
- this.frontModelConfig.progress.idleIntervalMs
6011
- );
6012
- }
6013
-
6014
- // When the turn's current audible silence began: the latest of the last
6015
- // emitted segment, the estimated client playback end, and the last enqueued
6016
- // filler.
6017
- private progressSilenceSinceMs(turn: ActiveAssistantTurn): number {
6018
- return Math.max(
6019
- turn.progress.lastAudibleAtMs,
6020
- this.assistantPlaybackTailUntilMs,
6021
- turn.progress.lastFloorHolderAtMs ?? 0,
6022
- );
6023
- }
6024
-
6025
6464
  // No assistant audio is pending or (estimatedly) still playing: nothing
6026
6465
  // buffered toward the next sentence, every queued TTS segment fully
6027
6466
  // emitted, and the client-side playback-tail estimate expired.
@@ -6033,177 +6472,18 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6033
6472
  );
6034
6473
  }
6035
6474
 
6036
- private clearProgressIdleTimer(turn: ActiveAssistantTurn): void {
6037
- if (turn.progress.idleTimer !== null) {
6038
- clearTimeout(turn.progress.idleTimer);
6039
- turn.progress.idleTimer = null;
6040
- }
6041
- }
6042
-
6043
6475
  // Clears progress narration and verdict timers for events that end the
6044
6476
  // current filler lifecycle.
6045
6477
  private clearFillerTimers(turn: ActiveAssistantTurn): void {
6046
- this.clearProgressIdleTimer(turn);
6478
+ turn.progress.clear();
6047
6479
  if (turn.verdictDeadlineTimer !== null) {
6048
6480
  clearTimeout(turn.verdictDeadlineTimer);
6049
6481
  turn.verdictDeadlineTimer = null;
6050
6482
  }
6051
6483
  }
6052
6484
 
6053
- // Narration may speak whenever the live turn is audibly silent and has
6054
- // not completed. Escalated turns qualify too: the bridge phrase holds the
6055
- // floor only while its audio plays (covered by the audible-idle check),
6056
- // and the strong leg's tool loops are the longest dead air in the system.
6057
- private turnCanNarrateProgress(turn: ActiveAssistantTurn): boolean {
6058
- return (
6059
- this.isActiveAssistantTurn(turn.token) &&
6060
- !turn.assistantCompleted &&
6061
- // Nothing is in flight while a decision is pending, so every phrase
6062
- // narration has would be a lie about who the call is waiting on. The
6063
- // turn says so once, when it starts waiting, and is quiet after that.
6064
- turn.pendingApproval === null &&
6065
- this.turnAudioIdle(turn)
6066
- );
6067
- }
6068
-
6069
- // The idle tick has something worth saying when the turn's tool activity has
6070
- // moved since the last narration described it, or when the silence has run
6071
- // past `maxSilenceMs` — the heartbeat ceiling that proves the assistant is
6072
- // still alive on a turn with no observable activity at all. Every other tick
6073
- // stays quiet, so the cadence follows the work rather than the clock.
6074
- private progressIdleHasSomethingToSay(turn: ActiveAssistantTurn): boolean {
6075
- const { progress } = turn;
6076
- if (progress.stateEpoch !== progress.narratedEpoch) {
6077
- return true;
6078
- }
6079
- const silentForMs = Date.now() - this.progressSilenceSinceMs(turn);
6080
- if (silentForMs >= this.frontModelConfig.progress.maxSilenceMs) {
6081
- return true;
6082
- }
6083
- log.debug(
6084
- { turnId: turn.turnId, silentForMs },
6085
- "Live voice progress narration held — nothing new since the last update",
6086
- );
6087
- return false;
6088
- }
6089
-
6090
- // Gatekeeper for progress narration. It speaks only while the turn is
6091
- // audibly silent, with one generation at a time and the configured spacing.
6092
- private maybeNarrateProgress(
6093
- turn: ActiveAssistantTurn,
6094
- trigger: "ops" | "idle" | "op_complete",
6095
- ): void {
6096
- const cfg = this.frontModelConfig.progress;
6097
- const { progress } = turn;
6098
- const progressNarrator = this.progressNarrator;
6099
- if (
6100
- !cfg.enabled ||
6101
- !this.streamTtsAudio ||
6102
- !progressNarrator ||
6103
- !this.turnCanNarrateProgress(turn) ||
6104
- progress.narrationInFlight ||
6105
- (progress.lastFloorHolderAtMs !== null &&
6106
- Date.now() - progress.lastFloorHolderAtMs < cfg.minGapMs) ||
6107
- (trigger === "ops" && progress.opsSinceNarration < cfg.opsThreshold) ||
6108
- (trigger === "idle" && !this.progressIdleHasSomethingToSay(turn))
6109
- ) {
6110
- return;
6111
- }
6112
- void this.speakProgressUpdate(turn, progressNarrator, trigger);
6113
- }
6114
-
6115
- // Generate and speak one audio-only progress narration. On a null result,
6116
- // idle narration falls back to a static phrase while activity-triggered
6117
- // narration stays silent.
6118
- private async speakProgressUpdate(
6119
- turn: ActiveAssistantTurn,
6120
- progressNarrator: VoiceProgressNarrator,
6121
- trigger: "ops" | "idle" | "op_complete",
6122
- ): Promise<void> {
6123
- const { progress } = turn;
6124
- progress.narrationInFlight = true;
6125
- // Any delta that lands while generation is in flight makes the text stale.
6126
- const deltaEpochAtLaunch = turn.deltaEpoch;
6127
- // The activity this update describes. Tool events that land mid-generation
6128
- // are news the generated text cannot carry, so they must leave the idle
6129
- // trigger armed rather than count as already narrated.
6130
- const stateEpochAtLaunch = progress.stateEpoch;
6131
- try {
6132
- const now = Date.now();
6133
- const currentOp = findLastIncompleteOp(progress.ops);
6134
- const input: VoiceProgressTextInput = {
6135
- transcriptSoFar: turn.utterance.finalTranscriptSegments
6136
- .join(" ")
6137
- .trim(),
6138
- completedOps: progress.ops
6139
- .filter(
6140
- (op): op is TurnProgressOp & { completedAtMs: number } =>
6141
- op.completedAtMs !== undefined,
6142
- )
6143
- // `ops` is in start order; the narrator wants completion
6144
- // order, which differs when parallel tools finish out of order.
6145
- .sort((a, b) => a.completedAtMs - b.completedAtMs)
6146
- .map((op) => ({
6147
- toolName: op.toolName,
6148
- ...(op.isError !== undefined ? { isError: op.isError } : {}),
6149
- ...(op.resultPreview !== undefined
6150
- ? { resultPreview: op.resultPreview }
6151
- : {}),
6152
- })),
6153
- currentOp: currentOp
6154
- ? {
6155
- toolName: currentOp.toolName,
6156
- elapsedMs: now - currentOp.startedAtMs,
6157
- }
6158
- : null,
6159
- turnElapsedMs: now - turn.launchedAtMs,
6160
- updateIndex: progress.updatesSpoken + 1,
6161
- ...(turn.language !== undefined ? { languageHint: turn.language } : {}),
6162
- };
6163
- const generated = await progressNarrator
6164
- .generateProgressText(input, turn.abortController.signal)
6165
- // The narrator contract never rejects. Keep test stubs fail-soft too.
6166
- .catch(() => null);
6167
- // Re-check liveness and staleness after the provider call.
6168
- if (
6169
- !this.turnCanNarrateProgress(turn) ||
6170
- turn.deltaEpoch !== deltaEpochAtLaunch ||
6171
- (progress.lastFloorHolderAtMs !== null &&
6172
- Date.now() - progress.lastFloorHolderAtMs <
6173
- this.frontModelConfig.progress.minGapMs)
6174
- ) {
6175
- return;
6176
- }
6177
- let raw = generated;
6178
- // Generated text is in the turn's language; only the static
6179
- // fallback comes from a localized table and may need the "en" override.
6180
- let fillerLanguage: string | undefined;
6181
- if (raw === null) {
6182
- if (trigger !== "idle") {
6183
- return;
6184
- }
6185
- raw = pickProgressPhrase(this.progressPhraseCounter++, turn.language);
6186
- fillerLanguage = this.fixedPhraseLanguage(
6187
- turn,
6188
- PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
6189
- );
6190
- }
6191
- if (!this.enqueueFillerPhrase(turn, raw, fillerLanguage)) {
6192
- return;
6193
- }
6194
- progress.opsSinceNarration = 0;
6195
- progress.narratedEpoch = stateEpochAtLaunch;
6196
- progress.updatesSpoken += 1;
6197
- // Record only when narration audio actually enqueued.
6198
- this.markProgressSpoken(turn);
6199
- // Restart the dead-air countdown from this narration.
6200
- this.armProgressIdleTimer(turn);
6201
- } finally {
6202
- progress.narrationInFlight = false;
6203
- }
6204
- }
6205
-
6206
- // Sanitize and enqueue one progress sentence on the ordered TTS queue.
6485
+ // Sanitize and enqueue one filler sentence (narration, the approval line)
6486
+ // on the ordered TTS queue, audio-only.
6207
6487
  private enqueueFillerPhrase(
6208
6488
  turn: ActiveAssistantTurn,
6209
6489
  raw: string,
@@ -6217,8 +6497,6 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6217
6497
  countsAsFirstSegment: false,
6218
6498
  ...(language !== undefined ? { language } : {}),
6219
6499
  });
6220
- // A spoken filler holds the floor, so narration's minGapMs spaces from it.
6221
- turn.progress.lastFloorHolderAtMs = Date.now();
6222
6500
  return true;
6223
6501
  }
6224
6502
 
@@ -6231,10 +6509,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6231
6509
  turn: ActiveAssistantTurn,
6232
6510
  table: Readonly<Record<string, unknown>>,
6233
6511
  ): string | undefined {
6234
- return turn.language !== undefined &&
6235
- !hasLocalizedEntry(table, turn.language)
6236
- ? "en"
6237
- : undefined;
6512
+ return fixedPhraseLanguage(table, turn.language);
6238
6513
  }
6239
6514
 
6240
6515
  private bufferAssistantTextForTts(token: symbol, text: string): void {
@@ -6277,7 +6552,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6277
6552
  log.info(
6278
6553
  {
6279
6554
  turnId: currentTurn.turnId,
6280
- escalated: currentTurn.escalationHandedOff,
6555
+ escalated: currentTurn.frontDoor?.handedOff ?? false,
6281
6556
  spokenSegments: currentTurn.spokenSegments,
6282
6557
  spokenChars: currentTurn.spokenChars,
6283
6558
  },
@@ -6306,6 +6581,44 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6306
6581
  // leave the last tool it touched on screen through the next silence.
6307
6582
  this.publishActivity(currentTurn, "");
6308
6583
 
6584
+ // Drain-scoped session control, under the same terms as the minimize
6585
+ // below: after the acknowledgement has been spoken, never for a
6586
+ // barged-in turn (talking over "okay, bye" means they are not
6587
+ // leaving), at most once per turn. Ending the call makes revealing
6588
+ // the screen moot, so an end takes the minimize's place.
6589
+ const sessionControl = currentTurn.sessionControlRequested;
6590
+ currentTurn.sessionControlRequested = null;
6591
+ if (
6592
+ sessionControl !== null &&
6593
+ !currentTurn.abortController.signal.aborted
6594
+ ) {
6595
+ log.info(
6596
+ { turnId: currentTurn.turnId, action: sessionControl.action },
6597
+ "Live voice reply requested a session control",
6598
+ );
6599
+ if (sessionControl.action === "end") {
6600
+ currentTurn.minimizeRequested = false;
6601
+ }
6602
+ // Armed before the send, so a client quick enough to answer with
6603
+ // its frame before this await resolves still finds the look waiting.
6604
+ // Never from the turn that answers a look: that turn is the answer,
6605
+ // and a marker it emits anyway must not chain another.
6606
+ if (
6607
+ isLookSessionControl(sessionControl.action) &&
6608
+ currentTurn.lookFollowUp === null
6609
+ ) {
6610
+ this.awaitLookFrame(sessionControl.action);
6611
+ }
6612
+ await this.sendFrame(
6613
+ {
6614
+ type: "session_control",
6615
+ turnId: currentTurn.turnId,
6616
+ ...sessionControl,
6617
+ },
6618
+ () => !this.isClosed,
6619
+ );
6620
+ }
6621
+
6309
6622
  // Drain-scoped minimize: the latched marker is consumed here, after
6310
6623
  // the turn's speech has fully drained — never mid-speech, never for
6311
6624
  // a barged-in turn, at most once per turn.
@@ -6518,7 +6831,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6518
6831
  if (settledTurn?.token === token) {
6519
6832
  // Anchor the dead-air countdown to the end of emission; the playback
6520
6833
  // -tail estimate covers any client-side buffer still draining.
6521
- settledTurn.progress.lastAudibleAtMs = Date.now();
6834
+ settledTurn.progress.noteAudioSettled();
6522
6835
  }
6523
6836
  this.pumpTtsSynthesis(token);
6524
6837
  }
@@ -6835,6 +7148,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
6835
7148
 
6836
7149
  turn.finalized = true;
6837
7150
  this.clearFillerTimers(turn);
7151
+ if (
7152
+ this.foregroundTaskState?.phase === "owned" &&
7153
+ this.foregroundTaskState.ownerToken === turn.token
7154
+ ) {
7155
+ this.clearForegroundTask(
7156
+ status === "completed" ? "owner_completed" : `owner_${reason}`,
7157
+ );
7158
+ }
6838
7159
  turn.utterance.completed = true;
6839
7160
  await this.archiveBufferedAudio({
6840
7161
  turnId: turn.turnId,
@@ -7105,8 +7426,8 @@ export function createLiveVoiceSession(
7105
7426
  const vadConfig = liveVoiceConfig?.vad;
7106
7427
  // Parsed once here into a complete config, shared by endpointing and the
7107
7428
  // session (the constructor's own parse is then a no-op re-validation).
7108
- const frontModelConfig = LiveVoiceFrontModelConfigSchema.parse(
7109
- options.frontModelConfig ?? liveVoiceConfig?.frontModel ?? {},
7429
+ const frontModelConfig = VoiceFrontModelConfigSchema.parse(
7430
+ options.frontModelConfig ?? getConfig().voice?.frontModel ?? {},
7110
7431
  );
7111
7432
  return new LiveVoiceSession(context, {
7112
7433
  ...options,
@@ -7271,10 +7592,10 @@ export async function defaultSpawnBackgroundContinuation(args: {
7271
7592
  // spoken by a session turn, on the session's terms; it was never meant to
7272
7593
  // mean invisible.
7273
7594
  //
7274
- // NOT the conversation's own sender: the voice bridge resets that to a
7275
- // no-op at turn teardown (see voice-session-bridge's clientCallbackInstalled
7276
- // reset), and the detach deliberately waits for that teardown before
7277
- // spawning — so a sender-based route is guaranteed to be dead by the time
7595
+ // NOT a per-turn sender: the conversation's event sink is fixed at
7596
+ // construction (see "Conversation event delivery" in assistant/AGENTS.md),
7597
+ // and the detach deliberately waits for the interrupted turn's bridge
7598
+ // teardown before spawning, so nothing turn-scoped is alive by the time
7278
7599
  // these events fire. `broadcastMessage` is the same path the bridge itself
7279
7600
  // uses to reach an attached web client. The subagent events carry
7280
7601
  // `parentConversationId`, not `conversationId`, so scope explicitly.