@vellumai/assistant 0.12.1 → 0.12.2-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (577) hide show
  1. package/AGENTS.md +4 -0
  2. package/ARCHITECTURE.md +20 -21
  3. package/docs/architecture/turn-actor.md +15 -8
  4. package/docs/guardian-request-flow.md +16 -0
  5. package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
  6. package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
  7. package/openapi.yaml +360 -51
  8. package/package.json +3 -1
  9. package/scripts/smoke-desktop-window-theme.ts +186 -0
  10. package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
  11. package/src/__tests__/anthropic-provider.test.ts +0 -63
  12. package/src/__tests__/app-compiler.test.ts +6 -0
  13. package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
  14. package/src/__tests__/app-dir-path-guard.test.ts +1 -3
  15. package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
  16. package/src/__tests__/assistant-event-hub.test.ts +44 -0
  17. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
  18. package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
  19. package/src/__tests__/computer-use-tools.test.ts +19 -7
  20. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  21. package/src/__tests__/config-schema.test.ts +28 -29
  22. package/src/__tests__/config-watcher.test.ts +23 -8
  23. package/src/__tests__/connection-model-compat.test.ts +2 -2
  24. package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
  25. package/src/__tests__/conversation-pairing.test.ts +26 -0
  26. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
  27. package/src/__tests__/conversation-queue.test.ts +48 -207
  28. package/src/__tests__/conversation-retry-discard.test.ts +54 -1
  29. package/src/__tests__/conversation-runtime-assembly.test.ts +28 -0
  30. package/src/__tests__/conversation-skill-tools.test.ts +27 -0
  31. package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
  32. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
  33. package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
  34. package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
  35. package/src/__tests__/conversation-title-service.test.ts +14 -10
  36. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
  37. package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
  38. package/src/__tests__/credential-record-retire.test.ts +171 -1
  39. package/src/__tests__/credential-routes.test.ts +250 -3
  40. package/src/__tests__/credential-security-invariants.test.ts +5 -3
  41. package/src/__tests__/cu-unified-flow.test.ts +14 -4
  42. package/src/__tests__/document-sync-tags.test.ts +25 -0
  43. package/src/__tests__/drain-kick-guard.test.ts +0 -2
  44. package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
  45. package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
  46. package/src/__tests__/git-porcelain.test.ts +47 -0
  47. package/src/__tests__/heartbeat-service.test.ts +56 -40
  48. package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
  49. package/src/__tests__/host-bash-proxy.test.ts +26 -45
  50. package/src/__tests__/host-browser-proxy.test.ts +6 -17
  51. package/src/__tests__/host-cu-proxy.test.ts +596 -0
  52. package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
  53. package/src/__tests__/host-file-proxy.test.ts +11 -23
  54. package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
  55. package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
  56. package/src/__tests__/llm-catalog-parity.test.ts +25 -0
  57. package/src/__tests__/mcp-auth-routes.test.ts +6 -9
  58. package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
  59. package/src/__tests__/mcp-health-check.test.ts +8 -11
  60. package/src/__tests__/mcp-list-plugin-servers.test.ts +8 -10
  61. package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
  62. package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
  63. package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
  64. package/src/__tests__/messaging-send-tool.test.ts +355 -548
  65. package/src/__tests__/model-intents.test.ts +3 -0
  66. package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
  67. package/src/__tests__/oauth-commands-routes.test.ts +168 -0
  68. package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
  69. package/src/__tests__/oauth-provider-serializer.test.ts +22 -0
  70. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  71. package/src/__tests__/openai-provider.test.ts +1 -0
  72. package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
  73. package/src/__tests__/platform-callback-registration.test.ts +8 -16
  74. package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
  75. package/src/__tests__/plugin-import-boundary-guard.test.ts +14 -0
  76. package/src/__tests__/qdrant-manager.test.ts +132 -0
  77. package/src/__tests__/resolve-app.test.ts +135 -0
  78. package/src/__tests__/scaffold-managed-skill-tool.test.ts +229 -6
  79. package/src/__tests__/script-proxy-certs.test.ts +35 -0
  80. package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
  81. package/src/__tests__/service-account-token.test.ts +247 -0
  82. package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
  83. package/src/__tests__/skill-feature-flags.test.ts +2 -2
  84. package/src/__tests__/skills.test.ts +9 -4
  85. package/src/__tests__/source-fingerprint.test.ts +184 -0
  86. package/src/__tests__/subagent-tool-gate-mode.test.ts +70 -3
  87. package/src/__tests__/subagent-tools.test.ts +3 -2
  88. package/src/__tests__/telephony-tts-guard.test.ts +38 -3
  89. package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
  90. package/src/__tests__/ui-channel-variants.test.ts +1 -56
  91. package/src/__tests__/unicode.test.ts +13 -0
  92. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
  93. package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
  94. package/src/__tests__/workspace-git-service.test.ts +26 -0
  95. package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
  96. package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
  97. package/src/acp/__tests__/acp-credentials.test.ts +13 -0
  98. package/src/acp/__tests__/agent-process.test.ts +258 -4
  99. package/src/acp/__tests__/auth-required.test.ts +96 -0
  100. package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
  101. package/src/acp/__tests__/client-handler.test.ts +90 -2
  102. package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
  103. package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
  104. package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
  105. package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
  106. package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
  107. package/src/acp/__tests__/session-manager.test.ts +895 -4
  108. package/src/acp/acp-claude-oauth.ts +328 -14
  109. package/src/acp/acp-credentials.ts +19 -0
  110. package/src/acp/agent-process.test.ts +4 -1
  111. package/src/acp/agent-process.ts +96 -10
  112. package/src/acp/auth-required.ts +50 -1
  113. package/src/acp/auto-install.test.ts +9 -7
  114. package/src/acp/claude-token-refresh.ts +150 -0
  115. package/src/acp/client-handler.ts +16 -3
  116. package/src/acp/model-config.test.ts +166 -0
  117. package/src/acp/model-config.ts +138 -0
  118. package/src/acp/prepare-agent-env.ts +21 -6
  119. package/src/acp/resolve-agent.test.ts +220 -6
  120. package/src/acp/resolve-agent.ts +88 -23
  121. package/src/acp/session-manager.test.ts +196 -3
  122. package/src/acp/session-manager.ts +441 -18
  123. package/src/acp/types.ts +50 -5
  124. package/src/agent/loop.ts +1 -0
  125. package/src/api/events/acp-session-model-update.test.ts +88 -0
  126. package/src/api/events/acp-session-model-update.ts +46 -0
  127. package/src/api/events/app-files-changed.ts +4 -3
  128. package/src/api/events/notification-intent.ts +2 -0
  129. package/src/api/events/user-message-echo.ts +7 -1
  130. package/src/api/index.ts +6 -0
  131. package/src/api/responses/conversation-message.ts +5 -0
  132. package/src/apps/app-store.ts +1 -1
  133. package/src/apps/resolve-app.ts +81 -0
  134. package/src/apps/source-fingerprint.ts +266 -0
  135. package/src/avatar/identity-avatar.ts +7 -2
  136. package/src/bundler/app-compiler.ts +15 -0
  137. package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
  138. package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
  139. package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
  140. package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
  141. package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
  142. package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
  143. package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
  144. package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
  145. package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +4 -4
  146. package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
  147. package/src/calls/__tests__/voice-control-protocol.test.ts +120 -25
  148. package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
  149. package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
  150. package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
  151. package/src/calls/__tests__/voice-session-bridge.test.ts +56 -1
  152. package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
  153. package/src/calls/barge-in-guard.ts +118 -0
  154. package/src/calls/call-constants.ts +24 -4
  155. package/src/calls/call-controller.ts +499 -199
  156. package/src/calls/call-transport.ts +16 -0
  157. package/src/calls/media-stream-output.ts +32 -0
  158. package/src/calls/media-stream-server.ts +82 -30
  159. package/src/calls/media-stream-stt-session.ts +15 -0
  160. package/src/{live-voice → calls}/progress-narration.ts +2 -2
  161. package/src/calls/resolve-call-tts-provider.ts +33 -2
  162. package/src/calls/tts-call-strategy.ts +9 -5
  163. package/src/calls/types.ts +2 -0
  164. package/src/calls/voice-control-protocol.ts +155 -21
  165. package/src/calls/voice-leg-coordinator.ts +189 -0
  166. package/src/calls/voice-progress-cadence.ts +423 -0
  167. package/src/calls/voice-session-bridge.ts +92 -72
  168. package/src/calls/voice-triage-escalate.ts +178 -59
  169. package/src/cli/commands/__tests__/apps.test.ts +245 -0
  170. package/src/cli/commands/__tests__/bash.test.ts +53 -0
  171. package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
  172. package/src/cli/commands/apps.help.ts +67 -13
  173. package/src/cli/commands/apps.ts +183 -0
  174. package/src/cli/commands/bash.help.ts +8 -13
  175. package/src/cli/commands/bash.ts +7 -25
  176. package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
  177. package/src/cli/commands/channels/index.help.ts +70 -2
  178. package/src/cli/commands/channels/index.ts +16 -3
  179. package/src/cli/commands/channels/send.ts +128 -0
  180. package/src/cli/commands/credentials.help.ts +6 -2
  181. package/src/cli/commands/credentials.ts +2 -2
  182. package/src/cli/commands/mcp.help.ts +8 -8
  183. package/src/cli/commands/mcp.ts +1 -1
  184. package/src/cli/commands/oauth/status.ts +5 -0
  185. package/src/cli/commands/roadmap.help.ts +5 -1
  186. package/src/cli/commands/roadmap.ts +1 -1
  187. package/src/cli/lib/bundled-marketplace.json +103 -90
  188. package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
  189. package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
  190. package/src/config/__tests__/platform-identity.test.ts +255 -0
  191. package/src/config/__tests__/platform-rehydration.test.ts +99 -9
  192. package/src/config/acp-agent-write.ts +43 -0
  193. package/src/config/acp-defaults.test.ts +13 -3
  194. package/src/config/acp-defaults.ts +23 -6
  195. package/src/config/acp-schema.test.ts +81 -0
  196. package/src/config/acp-schema.ts +30 -2
  197. package/src/config/bundled-skills/acp/SKILL.md +22 -14
  198. package/src/config/bundled-skills/acp/TOOLS.json +5 -1
  199. package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
  200. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
  201. package/src/config/bundled-skills/computer-use/SKILL.md +29 -2
  202. package/src/config/bundled-skills/computer-use/TOOLS.json +99 -0
  203. package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
  204. package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
  205. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
  206. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
  207. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  208. package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
  209. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
  210. package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
  211. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
  212. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
  213. package/src/config/bundled-skills/schedule/SKILL.md +3 -4
  214. package/src/config/bundled-skills/screen-annotation/SKILL.md +8 -1
  215. package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
  216. package/src/config/bundled-skills/skill-management/TOOLS.json +6 -2
  217. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
  218. package/src/config/default-profile-catalog.ts +1 -1
  219. package/src/config/env.ts +2 -2
  220. package/src/config/feature-flag-registry.json +33 -13
  221. package/src/config/loader.ts +2 -0
  222. package/src/config/memory-v3-gate.test.ts +31 -0
  223. package/src/config/memory-v3-gate.ts +20 -16
  224. package/src/config/platform-identity.ts +201 -0
  225. package/src/config/platform-rehydration.ts +11 -27
  226. package/src/config/schema.ts +2 -2
  227. package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
  228. package/src/config/schemas/__tests__/voice.test.ts +148 -0
  229. package/src/config/schemas/calls.ts +1 -28
  230. package/src/config/schemas/live-voice.ts +1 -173
  231. package/src/config/schemas/mcp.ts +1 -1
  232. package/src/config/schemas/memory-retrospective.ts +10 -0
  233. package/src/config/schemas/services.ts +18 -0
  234. package/src/config/schemas/skills.ts +0 -14
  235. package/src/config/schemas/voice.ts +150 -0
  236. package/src/credential-execution/process-manager.test.ts +18 -19
  237. package/src/credential-health/credential-health-service.ts +11 -17
  238. package/src/daemon/__tests__/conversation-tool-setup.test.ts +66 -121
  239. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +4 -1
  240. package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
  241. package/src/daemon/app-change-notify.ts +45 -0
  242. package/src/daemon/config-watcher.ts +13 -194
  243. package/src/daemon/conversation-error.ts +3 -2
  244. package/src/daemon/conversation-media-retry.ts +2 -1
  245. package/src/daemon/conversation-process.ts +35 -64
  246. package/src/daemon/conversation-runtime-assembly.ts +20 -7
  247. package/src/daemon/conversation-skill-tools.ts +13 -31
  248. package/src/daemon/conversation-store.ts +1 -1
  249. package/src/daemon/conversation-surfaces.ts +21 -10
  250. package/src/daemon/conversation-tool-setup.ts +45 -59
  251. package/src/daemon/conversation.ts +0 -3
  252. package/src/daemon/dictation-profile-store.ts +2 -1
  253. package/src/daemon/handlers/shared.ts +2 -1
  254. package/src/daemon/handlers/skills.ts +6 -3
  255. package/src/daemon/host-bash-proxy.ts +8 -12
  256. package/src/daemon/host-browser-proxy.ts +15 -1
  257. package/src/daemon/host-cu-proxy.ts +224 -19
  258. package/src/daemon/host-file-proxy.ts +8 -10
  259. package/src/daemon/host-proxy-preactivation.ts +2 -1
  260. package/src/daemon/host-transfer-proxy.ts +14 -18
  261. package/src/daemon/lifecycle.ts +0 -5
  262. package/src/daemon/mcp-reload-service.ts +5 -5
  263. package/src/daemon/message-types/sync.ts +1 -0
  264. package/src/daemon/orphan-reaper.ts +1 -1
  265. package/src/daemon/providers-setup.ts +1 -1
  266. package/src/daemon/reaction-history-render.ts +2 -1
  267. package/src/daemon/shutdown-handlers.ts +1 -9
  268. package/src/daemon/skill-memory-refresh.ts +64 -0
  269. package/src/daemon/tool-side-effects.ts +7 -18
  270. package/src/desktop/__tests__/fake-desktop.ts +122 -0
  271. package/src/desktop/desktop-chrome-policy.test.ts +79 -0
  272. package/src/desktop/desktop-chrome-policy.ts +47 -0
  273. package/src/desktop/desktop-chrome-session.ts +37 -0
  274. package/src/desktop/desktop-dependencies.test.ts +102 -0
  275. package/src/desktop/desktop-dependencies.ts +308 -0
  276. package/src/desktop/desktop-feature.test.ts +18 -0
  277. package/src/desktop/desktop-feature.ts +19 -0
  278. package/src/desktop/desktop-panel-config.test.ts +135 -0
  279. package/src/desktop/desktop-panel-config.ts +149 -0
  280. package/src/desktop/desktop-session-manager.test.ts +730 -0
  281. package/src/desktop/desktop-session-manager.ts +850 -0
  282. package/src/desktop/desktop-stream-bridge.test.ts +395 -0
  283. package/src/desktop/desktop-stream-bridge.ts +214 -0
  284. package/src/desktop/desktop-wallpaper.test.ts +191 -0
  285. package/src/desktop/desktop-wallpaper.ts +113 -0
  286. package/src/desktop/desktop-window-drag.ts +177 -0
  287. package/src/desktop/desktop-window-manager-config.test.ts +230 -0
  288. package/src/desktop/desktop-window-manager-config.ts +238 -0
  289. package/src/desktop/desktop-window-theme.ts +170 -0
  290. package/src/heartbeat/heartbeat-service.ts +21 -3
  291. package/src/home/__tests__/feed-writer.test.ts +22 -0
  292. package/src/home/feed-writer.ts +11 -2
  293. package/src/hooks/types.ts +33 -0
  294. package/src/i18n/AGENTS.md +22 -0
  295. package/src/i18n/__tests__/i18n.test.ts +108 -0
  296. package/src/i18n/index.ts +16 -0
  297. package/src/i18n/locales.ts +93 -0
  298. package/src/i18n/messages.ts +61 -0
  299. package/src/i18n/title.ts +25 -0
  300. package/src/inbound/platform-callback-registration.ts +4 -5
  301. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +121 -0
  302. package/src/live-voice/__tests__/live-voice-events.test.ts +8 -7
  303. package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
  304. package/src/live-voice/__tests__/live-voice-progress.test.ts +94 -15
  305. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
  306. package/src/live-voice/__tests__/live-voice-vad.test.ts +2 -2
  307. package/src/live-voice/__tests__/protocol.test.ts +39 -0
  308. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
  309. package/src/live-voice/__tests__/session-controls.test.ts +79 -0
  310. package/src/live-voice/live-voice-photo.ts +25 -6
  311. package/src/live-voice/live-voice-session.ts +297 -699
  312. package/src/live-voice/protocol.ts +60 -2
  313. package/src/live-voice/session-controls.ts +111 -0
  314. package/src/mcp/__tests__/effective-config.test.ts +2 -2
  315. package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
  316. package/src/mcp/effective-config.ts +3 -3
  317. package/src/mcp/mcp-header-store.ts +29 -45
  318. package/src/mcp/spec-schema.ts +111 -0
  319. package/src/mcp/startup.ts +8 -4
  320. package/src/mcp/workspace-mcp-config.ts +188 -0
  321. package/src/messaging/provider.ts +10 -3
  322. package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
  323. package/src/messaging/providers/callback-routing.ts +18 -0
  324. package/src/messaging/providers/channel-transport.ts +56 -0
  325. package/src/messaging/providers/discord/transport.ts +17 -0
  326. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
  327. package/src/messaging/providers/slack/adapter.ts +7 -44
  328. package/src/messaging/providers/slack/transport.ts +16 -0
  329. package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
  330. package/src/messaging/providers/telegram-bot/client.ts +0 -26
  331. package/src/messaging/providers/telegram-bot/transport.ts +20 -0
  332. package/src/messaging/providers/whatsapp/adapter.ts +1 -48
  333. package/src/messaging/providers/whatsapp/transport.ts +18 -0
  334. package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
  335. package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
  336. package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
  337. package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
  338. package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
  339. package/src/notifications/adapters/macos.ts +3 -0
  340. package/src/notifications/assistant-reply-producer.ts +8 -3
  341. package/src/notifications/conversation-pairing.ts +36 -16
  342. package/src/notifications/delivered-post-record.ts +11 -1
  343. package/src/notifications/guardian-feed-projection.ts +125 -5
  344. package/src/notifications/home-feed-side-effect.ts +55 -7
  345. package/src/notifications/schedule-result-producer.ts +66 -22
  346. package/src/oauth/AGENTS.md +2 -0
  347. package/src/oauth/__tests__/seed-providers-managed.test.ts +149 -0
  348. package/src/oauth/byo-connection.test.ts +105 -0
  349. package/src/oauth/byo-connection.ts +36 -6
  350. package/src/oauth/connection-resolver.test.ts +100 -0
  351. package/src/oauth/connection-resolver.ts +120 -3
  352. package/src/oauth/provider-serializer.ts +9 -0
  353. package/src/oauth/scope-utils.ts +24 -0
  354. package/src/oauth/seed-providers.ts +314 -27
  355. package/src/outbound-proxy/certs.ts +3 -5
  356. package/src/permissions/checker.test.ts +11 -6
  357. package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
  358. package/src/persistence/conversation-crud.ts +71 -63
  359. package/src/persistence/conversation-lineage.ts +12 -4
  360. package/src/persistence/conversation-queries.ts +2 -2
  361. package/src/persistence/conversation-title-placeholders.ts +23 -0
  362. package/src/persistence/conversation-title-service.ts +9 -26
  363. package/src/persistence/embeddings/qdrant-manager.ts +163 -18
  364. package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
  365. package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
  366. package/src/persistence/message-cursor.ts +78 -0
  367. package/src/persistence/message-reads.ts +30 -0
  368. package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
  369. package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
  370. package/src/persistence/schema/memory-core.ts +5 -0
  371. package/src/platform/client.test.ts +4 -2
  372. package/src/platform/client.ts +2 -10
  373. package/src/plugin-api/constants.ts +2 -0
  374. package/src/plugin-api/index.ts +1 -0
  375. package/src/plugin-api/types.ts +2 -0
  376. package/src/plugins/defaults/index.ts +2 -0
  377. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
  378. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
  379. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
  380. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
  381. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +20 -10
  382. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +5 -0
  383. package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
  384. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
  385. package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
  386. package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
  387. package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -2
  388. package/src/plugins/defaults/memory/context-search/format.ts +3 -2
  389. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +6 -3
  390. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +3 -2
  391. package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -7
  392. package/src/plugins/defaults/memory/hooks/init.ts +10 -0
  393. package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
  394. package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
  395. package/src/plugins/defaults/memory/host-utils.ts +10 -0
  396. package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
  397. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
  398. package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
  399. package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
  400. package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
  401. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
  402. package/src/plugins/defaults/memory/memory-retrospective-job.ts +12 -3
  403. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  404. package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
  405. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
  406. package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
  407. package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
  408. package/src/plugins/defaults/memory/startup.ts +28 -2
  409. package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
  410. package/src/plugins/defaults/memory/substrate/page-index.ts +6 -1
  411. package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
  412. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
  413. package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -2
  414. package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
  415. package/src/plugins/defaults/memory/v2/reranker.ts +6 -1
  416. package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
  417. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
  418. package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
  419. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +30 -0
  420. package/src/plugins/defaults/memory/v3/card.ts +3 -2
  421. package/src/plugins/defaults/memory/v3/pool-select.ts +5 -3
  422. package/src/plugins/defaults/memory/v3/sections.ts +9 -3
  423. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +5 -2
  424. package/src/plugins/defaults/memory/worker.ts +6 -0
  425. package/src/plugins/mcp-servers.ts +15 -75
  426. package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
  427. package/src/providers/__tests__/retry-callsite.test.ts +41 -0
  428. package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
  429. package/src/providers/anthropic/client.ts +0 -47
  430. package/src/providers/gemini/client.ts +9 -23
  431. package/src/providers/gemini-thought-signature.test.ts +63 -0
  432. package/src/providers/gemini-thought-signature.ts +44 -0
  433. package/src/providers/inference/__tests__/endpoint-probe.test.ts +42 -3
  434. package/src/providers/inference/auth.ts +2 -3
  435. package/src/providers/inference/connection-availability.ts +27 -8
  436. package/src/providers/inference/endpoint-probe.ts +10 -1
  437. package/src/providers/inference/resolve-auth.ts +22 -4
  438. package/src/providers/inference/service-account-token.ts +248 -0
  439. package/src/providers/model-catalog.ts +34 -0
  440. package/src/providers/model-intents.ts +10 -5
  441. package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
  442. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
  443. package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
  444. package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
  445. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
  446. package/src/providers/openai/chat-completions-provider.ts +212 -79
  447. package/src/providers/openai/codex-models.ts +0 -4
  448. package/src/providers/openai/google-thought-signature.ts +320 -0
  449. package/src/providers/opencode/client.test.ts +73 -4
  450. package/src/providers/opencode/client.ts +60 -3
  451. package/src/providers/openrouter/client.ts +20 -0
  452. package/src/providers/outbound-request-sanitize.ts +96 -0
  453. package/src/providers/retry.ts +30 -19
  454. package/src/providers/unparseable-tool-args.ts +5 -1
  455. package/src/runtime/AGENTS.md +3 -1
  456. package/src/runtime/__tests__/channel-send.test.ts +443 -0
  457. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +89 -0
  458. package/src/runtime/__tests__/host-observe.test.ts +40 -1
  459. package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
  460. package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
  461. package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
  462. package/src/runtime/agent-wake.ts +2 -1
  463. package/src/runtime/assistant-event-hub.ts +21 -3
  464. package/src/runtime/auth/same-actor.ts +38 -3
  465. package/src/runtime/channel-send.ts +316 -0
  466. package/src/runtime/host-observe.ts +11 -1
  467. package/src/runtime/http-server.ts +160 -151
  468. package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
  469. package/src/runtime/routes/__tests__/acp-routes.test.ts +159 -4
  470. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +275 -0
  471. package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
  472. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
  473. package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
  474. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
  475. package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
  476. package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
  477. package/src/runtime/routes/acp-routes.test.ts +15 -0
  478. package/src/runtime/routes/acp-routes.ts +72 -9
  479. package/src/runtime/routes/app-management-routes.ts +90 -2
  480. package/src/runtime/routes/app-routes.ts +1 -1
  481. package/src/runtime/routes/channel-send-routes.ts +85 -0
  482. package/src/runtime/routes/conversation-list-routes.ts +14 -3
  483. package/src/runtime/routes/conversation-query-routes.ts +40 -0
  484. package/src/runtime/routes/conversation-routes.ts +16 -3
  485. package/src/runtime/routes/credential-routes.ts +20 -14
  486. package/src/runtime/routes/desktop-setup-routes.test.ts +74 -0
  487. package/src/runtime/routes/desktop-setup-routes.ts +35 -0
  488. package/src/runtime/routes/diagnostics-routes.ts +61 -4
  489. package/src/runtime/routes/events-routes.ts +4 -0
  490. package/src/runtime/routes/host-cu-routes.ts +6 -0
  491. package/src/runtime/routes/index.ts +4 -2
  492. package/src/runtime/routes/log-export-routes.ts +2 -0
  493. package/src/runtime/routes/mcp-auth-routes.ts +39 -47
  494. package/src/runtime/routes/migration-routes.ts +6 -20
  495. package/src/runtime/routes/oauth-commands-routes.ts +50 -0
  496. package/src/runtime/routes/oauth-providers.ts +7 -0
  497. package/src/runtime/routes/platform-routes.ts +18 -15
  498. package/src/runtime/routes/workspace-routes.test.ts +55 -26
  499. package/src/runtime/routes/workspace-routes.ts +46 -63
  500. package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
  501. package/src/runtime/routes/workspace-tree-walk.ts +240 -0
  502. package/src/runtime/routes/workspace-utils.ts +17 -0
  503. package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
  504. package/src/runtime/services/conversation-serializer.ts +16 -3
  505. package/src/runtime/sync/resource-sync-events.ts +2 -1
  506. package/src/schedule/run-script.ts +2 -1
  507. package/src/schedule/schedule-store.ts +9 -2
  508. package/src/schedule/worker.ts +1 -1
  509. package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
  510. package/src/security/ces-http-record-backend.ts +120 -0
  511. package/src/security/ces-rpc-record-backend.ts +7 -0
  512. package/src/security/untrusted-content.ts +3 -1
  513. package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
  514. package/src/skills/workspace-skill-md-signature.ts +50 -0
  515. package/src/subagent/consult-context.ts +12 -25
  516. package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
  517. package/src/tools/acp/list-agents.test.ts +3 -1
  518. package/src/tools/acp/spawn.test.ts +146 -11
  519. package/src/tools/acp/spawn.ts +21 -3
  520. package/src/tools/acp/status.test.ts +94 -0
  521. package/src/tools/acp/status.ts +40 -3
  522. package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
  523. package/src/tools/browser/browser-execution.ts +1 -1
  524. package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
  525. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
  526. package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
  527. package/src/tools/browser/network-recorder.ts +2 -1
  528. package/src/tools/capability-offer.test.ts +22 -2
  529. package/src/tools/capability-offer.ts +16 -1
  530. package/src/tools/computer-use/definitions.ts +121 -2
  531. package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
  532. package/src/tools/credentials/metadata-store.ts +100 -0
  533. package/src/tools/filesystem/search.ts +2 -1
  534. package/src/tools/host-filesystem/edit.test.ts +2 -1
  535. package/src/tools/host-filesystem/edit.ts +2 -2
  536. package/src/tools/host-filesystem/read.test.ts +2 -1
  537. package/src/tools/host-filesystem/read.ts +2 -2
  538. package/src/tools/host-filesystem/transfer.test.ts +2 -1
  539. package/src/tools/host-filesystem/transfer.ts +2 -2
  540. package/src/tools/host-filesystem/write.test.ts +2 -1
  541. package/src/tools/host-filesystem/write.ts +2 -2
  542. package/src/tools/host-terminal/host-shell.test.ts +16 -0
  543. package/src/tools/host-terminal/host-shell.ts +4 -20
  544. package/src/tools/network/web-search.ts +6 -1
  545. package/src/tools/policy-context.ts +6 -7
  546. package/src/tools/skills/scaffold-managed.ts +61 -3
  547. package/src/tools/subagent/spawn.ts +5 -8
  548. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +57 -0
  549. package/src/tools/terminal/sanitized-bash.ts +105 -0
  550. package/src/tools/ui-surface/channel-variants.ts +10 -59
  551. package/src/tools/watch/watch-retro-report.ts +0 -8
  552. package/src/types/host-capabilities.ts +5 -3
  553. package/src/util/__tests__/language-subtag.test.ts +23 -1
  554. package/src/util/ca-bundle.ts +13 -0
  555. package/src/util/language-subtag.ts +16 -0
  556. package/src/util/platform.ts +8 -0
  557. package/src/util/unicode.ts +15 -1
  558. package/src/watch/watch-retro.ts +6 -8
  559. package/src/watch/watch-timeline.ts +2 -2
  560. package/src/workflows/run-manager.ts +2 -1
  561. package/src/workspace/git-porcelain.ts +66 -0
  562. package/src/workspace/git-service.ts +6 -55
  563. package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
  564. package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
  565. package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
  566. package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
  567. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
  568. package/src/workspace/migrations/registry.ts +6 -0
  569. package/src/__tests__/app-source-watcher.test.ts +0 -218
  570. package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
  571. package/src/__tests__/speaker-identification.test.ts +0 -56
  572. package/src/calls/speaker-identification.ts +0 -233
  573. package/src/daemon/app-source-watcher.ts +0 -218
  574. package/src/messaging/providers/whatsapp/client.ts +0 -29
  575. package/src/runtime/routes/debug-bash-routes.ts +0 -173
  576. package/src/util/fs-watcher-error.ts +0 -36
  577. /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
@@ -98,4 +98,20 @@ export interface CallTransport {
98
98
  * this — the controller then treats playback as already drained.
99
99
  */
100
100
  awaitPlaybackDrained?(): Promise<void>;
101
+
102
+ /**
103
+ * Whether no assistant speech is buffered, queued, in flight, or
104
+ * (estimatedly) still playing to the caller. Progress narration speaks
105
+ * only into audible silence, so it asks before every update. Transports
106
+ * that play speech synchronously may omit this; the controller then
107
+ * treats the transport as idle.
108
+ */
109
+ isPlaybackIdle?(): boolean;
110
+
111
+ /**
112
+ * Wall-clock estimate of when audio already sent finishes playing to the
113
+ * caller, 0 when nothing is playing. Anchors the dead-air countdown for
114
+ * progress narration to when the caller last heard something.
115
+ */
116
+ playbackTailUntilMs?(): number;
101
117
  }
@@ -327,6 +327,14 @@ export class MediaStreamOutput implements CallTransport {
327
327
  /** True when the queue drain loop is actively running. */
328
328
  private draining = false;
329
329
 
330
+ /**
331
+ * Estimated wall-clock instant the audio already sent to Twilio finishes
332
+ * playing to the caller. Frames leave faster than real time and Twilio
333
+ * buffers them, so the only local view of "still audible" is this running
334
+ * sum of sent frame durations; a `clear` drops the buffer and resets it.
335
+ */
336
+ private playbackTailAtMs = 0;
337
+
330
338
  /** Abort controller for the currently in-flight synthesis/fetch. */
331
339
  private activePlaybackAbort: AbortController | null = null;
332
340
 
@@ -659,6 +667,7 @@ export class MediaStreamOutput implements CallTransport {
659
667
  return;
660
668
  }
661
669
  this.sendClearCommand();
670
+ this.playbackTailAtMs = 0;
662
671
  // Twilio drops its buffered audio (and the marks within it) on `clear`, so
663
672
  // any end-of-turn mark already sent will never echo. Treat those as drained
664
673
  // so a pending end-call drain wait resolves instead of stalling on the cap.
@@ -716,6 +725,24 @@ export class MediaStreamOutput implements CallTransport {
716
725
  return this.playbackQueue.length;
717
726
  }
718
727
 
728
+ /**
729
+ * Whether no speech is buffered toward a sentence, queued, being
730
+ * synthesized or fetched, or (estimatedly) still playing at Twilio.
731
+ */
732
+ isPlaybackIdle(): boolean {
733
+ return (
734
+ this.textBuffer.length === 0 &&
735
+ this.playbackQueue.length === 0 &&
736
+ !this.draining &&
737
+ Date.now() >= this.playbackTailAtMs
738
+ );
739
+ }
740
+
741
+ /** See {@link CallTransport.playbackTailUntilMs}. */
742
+ playbackTailUntilMs(): number {
743
+ return this.playbackTailAtMs;
744
+ }
745
+
719
746
  /**
720
747
  * Runtime check for closed state. Used instead of direct property access
721
748
  * in async methods because TypeScript's control flow analysis cannot
@@ -772,6 +799,7 @@ export class MediaStreamOutput implements CallTransport {
772
799
  this.playbackQueue.length = 0;
773
800
  this.playbackVersion++;
774
801
  this.audioStartCallback = null;
802
+ this.playbackTailAtMs = 0;
775
803
  if (this.activePlaybackAbort) {
776
804
  this.activePlaybackAbort.abort();
777
805
  this.activePlaybackAbort = null;
@@ -854,6 +882,10 @@ export class MediaStreamOutput implements CallTransport {
854
882
  for (const frame of frames) {
855
883
  this.sendAudioPayload(frame);
856
884
  }
885
+ const durationMs =
886
+ (frames.length * MULAW_FRAME_SIZE * 1000) / TELEPHONY_SAMPLE_RATE_HZ;
887
+ this.playbackTailAtMs =
888
+ Math.max(Date.now(), this.playbackTailAtMs) + durationMs;
857
889
  }
858
890
 
859
891
  /**
@@ -47,6 +47,11 @@ import { addMessage } from "../persistence/conversation-crud.js";
47
47
  import { resolveGuardianName } from "../prompts/user-reference.js";
48
48
  import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
49
49
  import { getLogger } from "../util/logger.js";
50
+ import {
51
+ type BargeInGuard,
52
+ createBargeInGuard,
53
+ DEFAULT_BARGE_IN_MIN_SPEECH_MS,
54
+ } from "./barge-in-guard.js";
50
55
  import { CallController } from "./call-controller.js";
51
56
  import {
52
57
  formatDuration,
@@ -188,6 +193,8 @@ export class MediaStreamCallSession {
188
193
  private bargeInAccepted = 0;
189
194
  /** Number of barge-in attempts that were ignored (assistant not speaking). */
190
195
  private bargeInIgnored = 0;
196
+ /** Armed by caller speech while a turn or its tail is interruptible. */
197
+ private bargeInGuard: BargeInGuard | null = null;
191
198
  /** Number of turn-start transitions detected by the STT session. */
192
199
  private turnStarts = 0;
193
200
  /** Number of transcript finals produced (non-empty). */
@@ -215,6 +222,9 @@ export class MediaStreamCallSession {
215
222
  // Create STT session with callbacks wired to the controller.
216
223
  const callbacks: MediaStreamSttSessionCallbacks = {
217
224
  onSpeechStart: () => this.handleSpeechStart(),
225
+ onMediaFrame: (hasSpeech, durationMs) =>
226
+ this.handleMediaFrame(hasSpeech, durationMs),
227
+ onSpeechEnd: () => this.handleSpeechEnd(),
218
228
  onTranscriptFinal: (text, durationMs) =>
219
229
  this.handleTranscriptFinal(text, durationMs),
220
230
  onDtmf: (digit) => this.handleDtmf(digit),
@@ -768,38 +778,80 @@ export class MediaStreamCallSession {
768
778
 
769
779
  private handleSpeechStart(): void {
770
780
  this.turnStarts++;
781
+ }
771
782
 
772
- // Barge-in: clear queued outbound audio and abort the in-flight LLM
773
- // turn only when the assistant is actively speaking. Uses the gated
774
- // handleBargeIn path so initial inbound audio frames do not cancel a
775
- // still-starting initial turn.
776
- //
777
- // clearAudio runs via the onAccepted hook so it only fires when the
778
- // barge-in passes the speaking gate an ignored barge-in (controller
779
- // idle/processing) must not flush queued/in-flight TTS such as a
780
- // buffered greeting. The hook runs before handleInterrupt so the
781
- // end-of-turn mark it enqueues survives the queue flush.
782
- if (this.output && this.controller) {
783
- const output = this.output;
784
- const accepted = this.controller.handleBargeIn(() => output.clearAudio());
785
- if (accepted) {
786
- this.bargeInAccepted++;
787
- log.info(
788
- { callSessionId: this.callSessionId },
789
- "Media-stream barge-in accepted — cleared outbound audio",
790
- );
791
- } else {
792
- // No turn to abort, but a completed turn's tail can still be
793
- // playing from Twilio's buffer — flush only that buffer so the
794
- // caller isn't talked over. Queued speech that hasn't reached
795
- // Twilio yet (greeting, handoff prompt) is preserved.
796
- output.clearBufferedAudio();
797
- this.bargeInIgnored++;
798
- log.debug(
799
- { callSessionId: this.callSessionId },
800
- "Media-stream barge-in ignored — assistant not speaking",
801
- );
783
+ // Sustained-speech barge-in guard, fed by every inbound frame. Speech only
784
+ // counts while there is something to interrupt: an assistant turn in
785
+ // flight (thinking or speaking) or a completed turn's tail still playing
786
+ // from Twilio's buffer. Outside that window the guard is dropped, so
787
+ // speech from the caller's own utterance never carries into a turn that
788
+ // starts before the local VAD has ended the utterance (a streaming final
789
+ // can start the turn first). A speech frame inside the window arms the
790
+ // guard; a fired guard reaches the controller. A line click, a cough, or
791
+ // TTS bleed never gets that far.
792
+ private handleMediaFrame(hasSpeech: boolean, durationMs: number): void {
793
+ if (!this.output || !this.controller) {
794
+ return;
795
+ }
796
+ const bargeable =
797
+ this.controller.getState() !== "idle" || !this.output.isPlaybackIdle();
798
+ if (!bargeable) {
799
+ this.bargeInGuard = null;
800
+ return;
801
+ }
802
+ if (this.bargeInGuard === null) {
803
+ if (!hasSpeech) {
804
+ return;
802
805
  }
806
+ this.bargeInGuard = createBargeInGuard(DEFAULT_BARGE_IN_MIN_SPEECH_MS);
807
+ }
808
+ const step = this.bargeInGuard.track(
809
+ hasSpeech ? "speech" : "silence",
810
+ durationMs,
811
+ );
812
+ if (step === "fired") {
813
+ this.bargeInGuard = null;
814
+ this.fireBargeIn();
815
+ }
816
+ }
817
+
818
+ private handleSpeechEnd(): void {
819
+ // An untripped guard was noise or a fragment: the run never reached the
820
+ // threshold before the caller's turn ended.
821
+ this.bargeInGuard = null;
822
+ }
823
+
824
+ // Sustained caller speech landed: clear queued outbound audio and abort
825
+ // the in-flight LLM turn when one is in flight (thinking or speaking).
826
+ //
827
+ // clearAudio runs via the onAccepted hook so it only fires when the
828
+ // barge-in passes the controller's gate; an ignored barge-in (controller
829
+ // idle) must not flush queued/in-flight TTS such as a buffered greeting.
830
+ // The hook runs before handleInterrupt so the end-of-turn mark it
831
+ // enqueues survives the queue flush.
832
+ private fireBargeIn(): void {
833
+ if (!this.output || !this.controller) {
834
+ return;
835
+ }
836
+ const output = this.output;
837
+ const accepted = this.controller.handleBargeIn(() => output.clearAudio());
838
+ if (accepted) {
839
+ this.bargeInAccepted++;
840
+ log.info(
841
+ { callSessionId: this.callSessionId },
842
+ "Media-stream barge-in accepted: cleared outbound audio",
843
+ );
844
+ } else {
845
+ // No turn to abort, but a completed turn's tail can still be
846
+ // playing from Twilio's buffer, so flush only that buffer so the
847
+ // caller isn't talked over. Queued speech that hasn't reached
848
+ // Twilio yet (greeting, handoff prompt) is preserved.
849
+ output.clearBufferedAudio();
850
+ this.bargeInIgnored++;
851
+ log.debug(
852
+ { callSessionId: this.callSessionId },
853
+ "Media-stream barge-in ignored: no turn in flight",
854
+ );
803
855
  }
804
856
  }
805
857
 
@@ -120,6 +120,16 @@ export interface MediaStreamSttSessionCallbacks {
120
120
  /** Called when the turn detector transitions to active (first speech-bearing chunk). */
121
121
  onSpeechStart?: () => void;
122
122
 
123
+ /**
124
+ * Called for every inbound frame with the local VAD's verdict on it and
125
+ * the frame's duration. Drives the sustained-speech barge-in guard, which
126
+ * needs the silence frames inside a detector turn as well as the speech.
127
+ */
128
+ onMediaFrame?: (hasSpeech: boolean, durationMs: number) => void;
129
+
130
+ /** Called when the turn detector ends the caller's turn (silence or cap). */
131
+ onSpeechEnd?: () => void;
132
+
123
133
  /**
124
134
  * Called when a completed caller utterance has been transcribed.
125
135
  *
@@ -255,6 +265,7 @@ export class MediaStreamSttSession {
255
265
  this.callbacks.onSpeechStart?.();
256
266
  },
257
267
  onTurnEnd: (reason, durationMs) => {
268
+ this.callbacks.onSpeechEnd?.();
258
269
  void this.handleTurnEnd(reason, durationMs);
259
270
  },
260
271
  });
@@ -377,6 +388,10 @@ export class MediaStreamSttSession {
377
388
  const raw = Buffer.from(event.media.payload, "base64");
378
389
  const hasSpeech = detectSpeechActivity(raw);
379
390
  this.turnDetector.onMediaChunk(hasSpeech);
391
+ this.callbacks.onMediaFrame?.(
392
+ hasSpeech,
393
+ raw.length / (TELEPHONY_SOURCE_SAMPLE_RATE / 1000),
394
+ );
380
395
 
381
396
  if (this.mode === "batch") {
382
397
  this.currentTurnChunks.push(event.media.payload);
@@ -5,7 +5,7 @@
5
5
  * The voice front door owns hold, escalation, and direct-response behavior.
6
6
  */
7
7
 
8
- import type { LiveVoiceProgressConfig } from "../config/schemas/live-voice.js";
8
+ import type { VoiceProgressConfig } from "../config/schemas/voice.js";
9
9
  import {
10
10
  extractToolUse,
11
11
  getConfiguredProvider,
@@ -216,7 +216,7 @@ export function effectiveSpokenTextMaxChars(
216
216
  }
217
217
 
218
218
  export function createVoiceProgressNarrator(options: {
219
- config: LiveVoiceProgressConfig;
219
+ config: VoiceProgressConfig;
220
220
  /** Provider resolver, injectable for tests. */
221
221
  getProvider?: () => Promise<Provider | null>;
222
222
  }): VoiceProgressNarrator {
@@ -8,6 +8,7 @@
8
8
  */
9
9
 
10
10
  import { loadConfig } from "../config/loader.js";
11
+ import { resolveEffectiveSpeechProviders } from "../config/managed-speech-defaults.js";
11
12
  import {
12
13
  getCatalogProvider,
13
14
  getTtsProvider,
@@ -73,6 +74,13 @@ export interface ResolveCallTtsOptions {
73
74
  /**
74
75
  * Resolve the active TTS provider via the global provider abstraction.
75
76
  *
77
+ * The provider is the effective one after managed-speech defaulting
78
+ * ({@link resolveEffectiveSpeechProviders}): a configured BYOK provider
79
+ * whose credential does not resolve is stood in for by managed speech
80
+ * while the platform connection is usable, the same substitution live
81
+ * voice makes, so a fresh Vellum connection speaks on the phone with zero
82
+ * TTS configuration.
83
+ *
76
84
  * The native-vs-synthesized decision is driven by the catalog's
77
85
  * `callMode` field via {@link resolveCallStrategy}. Providers with
78
86
  * `callMode: "synthesized-play"` have their audio streamed through the
@@ -94,10 +102,11 @@ export async function resolveCallTtsProvider(
94
102
  ): Promise<ResolvedCallTts> {
95
103
  try {
96
104
  const config = loadConfig();
97
- const resolved = resolveTtsConfig(config);
105
+ const effectiveProviderId = await effectiveTtsProviderId(config);
106
+ const resolved = resolveTtsConfig(config, effectiveProviderId);
98
107
 
99
108
  // Use the catalog's callMode to decide the call path.
100
- const strategy = resolveCallStrategy(config);
109
+ const strategy = resolveCallStrategy(config, effectiveProviderId);
101
110
 
102
111
  let providerId = resolved.provider;
103
112
  let useSynthesizedPath = strategy.callMode === "synthesized-play";
@@ -175,6 +184,28 @@ export async function resolveCallTtsProvider(
175
184
  }
176
185
  }
177
186
 
187
+ /**
188
+ * The TTS provider the call actually uses after managed-speech defaulting.
189
+ * A failure inside the managed lookup (platform client, credential store)
190
+ * keeps the configured provider: a managed-speech hiccup must not demote a
191
+ * working BYOK provider, and the outer resolver's own fallback is the
192
+ * silent native path.
193
+ */
194
+ async function effectiveTtsProviderId(
195
+ config: ReturnType<typeof loadConfig>,
196
+ ): Promise<TtsProviderId | undefined> {
197
+ try {
198
+ const { tts } = await resolveEffectiveSpeechProviders(config);
199
+ return tts;
200
+ } catch (err) {
201
+ log.warn(
202
+ { err },
203
+ "Managed-speech defaulting failed for call TTS; keeping the configured provider",
204
+ );
205
+ return undefined;
206
+ }
207
+ }
208
+
178
209
  /**
179
210
  * Decode a resolved call audio format into the provider request format
180
211
  * and the audio-store format.
@@ -40,17 +40,21 @@ export interface TtsCallStrategy {
40
40
  }
41
41
 
42
42
  /**
43
- * Resolve the call strategy for the currently configured TTS provider.
43
+ * Resolve the call strategy for the active TTS provider.
44
44
  *
45
- * Reads the active provider from config via {@link resolveTtsConfig},
46
- * then looks up the provider's `callMode` in the catalog.
45
+ * Reads the provider from config via {@link resolveTtsConfig} (or takes
46
+ * `providerOverride`, the provider a caller already resolved after
47
+ * managed-speech defaulting), then looks up its `callMode` in the catalog.
47
48
  *
48
49
  * Falls back to `native-twilio` with `"elevenlabs"` when the config
49
50
  * or catalog is unavailable (e.g. test mocks, pre-migration configs).
50
51
  */
51
- export function resolveCallStrategy(config: AssistantConfig): TtsCallStrategy {
52
+ export function resolveCallStrategy(
53
+ config: AssistantConfig,
54
+ providerOverride?: TtsProviderId,
55
+ ): TtsCallStrategy {
52
56
  try {
53
- const resolved = resolveTtsConfig(config);
57
+ const resolved = resolveTtsConfig(config, providerOverride);
54
58
  const catalogEntry = getCatalogProvider(resolved.provider);
55
59
  return {
56
60
  providerId: catalogEntry.id,
@@ -11,6 +11,8 @@ export type CallEventType =
11
11
  | "call_connected"
12
12
  | "caller_spoke"
13
13
  | "assistant_spoke"
14
+ | "tool_use_started"
15
+ | "tool_use_completed"
14
16
  | "user_question_asked"
15
17
  | "user_answered"
16
18
  | "user_instruction_relayed"
@@ -29,14 +29,37 @@ export const HOLD_VERDICT_TOKEN = "[0]";
29
29
  export const ESCALATE_VERDICT_TOKEN = "[1]";
30
30
 
31
31
  /**
32
- * Emitted inline by the main/escalated voice leg to ask the live-voice
33
- * client to minimize the voice room and reveal the screen behind it
34
- * (e.g. after creating an app worth showing). Advisory: the daemon
35
- * translates it into a `minimize_room` server frame after the turn's TTS
36
- * drains; it is never spoken and never persisted.
32
+ * Room-minimize token, kept in the marker table so it is stripped from
33
+ * speech and from persisted rows. No prompt teaches it: the live-voice
34
+ * session decides a minimize from whether a ui-surface tool ran during the
35
+ * turn (`revealsUiSurface` on `tool_result`) and sends the `minimize_room`
36
+ * frame itself, so a model that emits this token regardless moves nothing.
37
37
  */
38
38
  export const MINIMIZE_ROOM_MARKER = "[-1]";
39
39
 
40
+ /**
41
+ * Mic-mute session control for live voice: `[MUTE]` mutes until the user
42
+ * unmutes, `[MUTE:<seconds>]` mutes for that long. Like {@link END_CALL_MARKER}
43
+ * on a live-voice session, it is acted on only at the very end of a reply
44
+ * (see {@link parseTerminalSessionControl}), after the acknowledgement that
45
+ * precedes it has been spoken.
46
+ */
47
+ export const MUTE_MARKER = "[MUTE]";
48
+ const MUTE_MARKER_PREFIX = "[MUTE:";
49
+
50
+ /** Longest timed mute a marker can ask for; longer asks mute until unmuted. */
51
+ export const MAX_TIMED_MUTE_SECONDS = 3600;
52
+
53
+ /**
54
+ * Progress-update cadence for the rest of a live-voice session:
55
+ * {@link FEWER_UPDATES_MARKER} after "don't give me updates so often",
56
+ * {@link NORMAL_UPDATES_MARKER} to go back. Terminal like the other session
57
+ * controls, but carried out by the session itself: narration is the daemon's.
58
+ */
59
+ export const FEWER_UPDATES_MARKER = "[UPDATES:FEWER]";
60
+ export const NORMAL_UPDATES_MARKER = "[UPDATES:NORMAL]";
61
+ const UPDATES_MARKER_PREFIX = "[UPDATES:";
62
+
40
63
  // ---------------------------------------------------------------------------
41
64
  // Regexes
42
65
  // ---------------------------------------------------------------------------
@@ -57,6 +80,8 @@ const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
57
80
  const HOLD_VERDICT_TOKEN_REGEX = /\[0\]/g;
58
81
  const ESCALATE_VERDICT_TOKEN_REGEX = /\[1\]/g;
59
82
  const MINIMIZE_ROOM_MARKER_REGEX = /\[-1\]/g;
83
+ const MUTE_MARKER_REGEX = /\[MUTE(?::\s*[^\]]*)?\]/g;
84
+ const UPDATES_MARKER_REGEX = /\[UPDATES:\s*[^\]]*\]/g;
60
85
  const GUARDIAN_TIMEOUT_MARKER_REGEX = /\[GUARDIAN_TIMEOUT\]/g;
61
86
  const GUARDIAN_UNAVAILABLE_MARKER_REGEX = /\[GUARDIAN_UNAVAILABLE\]/g;
62
87
 
@@ -182,6 +207,8 @@ export function stripInternalSpeechMarkers(text: string): string {
182
207
  .replace(HOLD_VERDICT_TOKEN_REGEX, "")
183
208
  .replace(ESCALATE_VERDICT_TOKEN_REGEX, "")
184
209
  .replace(MINIMIZE_ROOM_MARKER_REGEX, "")
210
+ .replace(MUTE_MARKER_REGEX, "")
211
+ .replace(UPDATES_MARKER_REGEX, "")
185
212
  .replace(GUARDIAN_TIMEOUT_MARKER_REGEX, "")
186
213
  .replace(GUARDIAN_UNAVAILABLE_MARKER_REGEX, "");
187
214
  return result;
@@ -192,9 +219,9 @@ export function stripInternalSpeechMarkers(text: string): string {
192
219
  // ---------------------------------------------------------------------------
193
220
 
194
221
  /**
195
- * All known control marker prefixes. Used by couldBeControlMarker to detect
196
- * whether a buffer that starts with `[` might be the beginning of a control
197
- * marker (and should therefore be held rather than flushed to TTS).
222
+ * All known control marker prefixes. Used by isIncompleteControlMarkerTail to
223
+ * detect whether a buffer that starts with `[` might be the beginning of a
224
+ * control marker (and should therefore be held rather than flushed to TTS).
198
225
  */
199
226
  const CONTROL_MARKER_STRINGS = [
200
227
  "[ASK_GUARDIAN_APPROVAL:",
@@ -207,23 +234,13 @@ const CONTROL_MARKER_STRINGS = [
207
234
  "[0]",
208
235
  "[1]",
209
236
  "[-1]",
237
+ MUTE_MARKER,
238
+ MUTE_MARKER_PREFIX,
239
+ UPDATES_MARKER_PREFIX,
210
240
  "[GUARDIAN_TIMEOUT]",
211
241
  "[GUARDIAN_UNAVAILABLE]",
212
242
  ];
213
243
 
214
- /**
215
- * Check whether `text` could be a partial or complete control marker.
216
- *
217
- * Returns true if any known marker string is a prefix of `text`
218
- * (text starts with the marker) or `text` is a prefix of a marker
219
- * (the marker starts with text — i.e. text is still being streamed).
220
- */
221
- export function couldBeControlMarker(text: string): boolean {
222
- return CONTROL_MARKER_STRINGS.some(
223
- (marker) => marker.startsWith(text) || text.startsWith(marker),
224
- );
225
- }
226
-
227
244
  // Colon-style markers whose bodies terminate at the first "]" — their strip
228
245
  // regexes are non-greedy (`.+?\]`), so the first bracket IS the terminator.
229
246
  // ASK_GUARDIAN_APPROVAL is deliberately absent: its balanced-JSON body may
@@ -233,6 +250,8 @@ const FIRST_BRACKET_TERMINATED_PREFIXES = [
233
250
  "[ASK_GUARDIAN:",
234
251
  "[USER_ANSWERED:",
235
252
  "[USER_INSTRUCTION:",
253
+ MUTE_MARKER_PREFIX,
254
+ UPDATES_MARKER_PREFIX,
236
255
  ];
237
256
 
238
257
  const GUARDIAN_APPROVAL_PREFIX = "[ASK_GUARDIAN_APPROVAL:";
@@ -267,3 +286,118 @@ export function isIncompleteControlMarkerTail(tail: string): boolean {
267
286
  }
268
287
  return false;
269
288
  }
289
+
290
+ /**
291
+ * Control-marker hygiene for one model leg's streamed text, shared by the
292
+ * phone call controller and the live-voice session. The returned flush is
293
+ * called with the leg's cumulative raw text so far and forwards, through
294
+ * `emit`, the stripped ({@link stripInternalSpeechMarkers}) prefix that has
295
+ * not been emitted yet and cannot contain a still-streaming control marker:
296
+ * the flush stops at the first "[" whose tail is an incomplete marker
297
+ * ({@link isIncompleteControlMarkerTail}) and holds from there until a later
298
+ * delta completes or disproves it. `force` (leg completion) emits the held
299
+ * tail so real text that merely resembles a marker prefix is not dropped.
300
+ *
301
+ * The scan runs forward from the emitted boundary, not from the last "[", so
302
+ * brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
303
+ * in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
304
+ * its terminator. Markers are stripped, never acted on: a consumer that acts
305
+ * on a marker reads the leg's full raw text for it separately.
306
+ */
307
+ export function createControlMarkerHoldback(
308
+ emit: (chunk: string) => void,
309
+ ): (raw: string, opts?: { force?: boolean }) => void {
310
+ let emitted = 0;
311
+ return (raw, opts) => {
312
+ let safeEnd = raw.length;
313
+ if (opts?.force !== true) {
314
+ for (
315
+ let i = raw.indexOf("[", emitted);
316
+ i !== -1;
317
+ i = raw.indexOf("[", i + 1)
318
+ ) {
319
+ if (isIncompleteControlMarkerTail(raw.slice(i))) {
320
+ safeEnd = i;
321
+ break;
322
+ }
323
+ }
324
+ }
325
+ if (safeEnd > emitted) {
326
+ const chunk = stripInternalSpeechMarkers(raw.slice(emitted, safeEnd));
327
+ emitted = safeEnd;
328
+ if (chunk.length > 0) {
329
+ emit(chunk);
330
+ }
331
+ }
332
+ };
333
+ }
334
+
335
+ // ---------------------------------------------------------------------------
336
+ // Live-voice session controls
337
+ // ---------------------------------------------------------------------------
338
+
339
+ /**
340
+ * A session control a live-voice reply asked for with a terminal marker:
341
+ * `end` from {@link END_CALL_MARKER}, `mute` from {@link MUTE_MARKER} or its
342
+ * timed form, `updates` from the progress-cadence markers.
343
+ */
344
+ export type SessionControlRequest =
345
+ | { readonly action: "end" }
346
+ | { readonly action: "mute"; readonly durationMs?: number }
347
+ | { readonly action: "updates"; readonly cadence: "fewer" | "normal" };
348
+
349
+ const TERMINAL_SESSION_CONTROL_REGEX =
350
+ /(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
351
+
352
+ /**
353
+ * The session control a reply ends with, or null.
354
+ *
355
+ * **Terminal position only.** The marker follows the spoken acknowledgement
356
+ * ("Okay, talk soon."), so a reply that merely mentions a marker mid-text, or
357
+ * parrots one from history before carrying on, controls nothing. This is the
358
+ * same rule the minimize marker's transcript hygiene applies.
359
+ *
360
+ * A timed mute whose body is not a positive number of seconds degrades to an
361
+ * untimed mute rather than to nothing: the user asked to be muted, and muting
362
+ * until they unmute is the conservative reading of a garbled duration. Asks
363
+ * past {@link MAX_TIMED_MUTE_SECONDS} degrade the same way.
364
+ */
365
+ export function parseTerminalSessionControl(
366
+ text: string,
367
+ ): SessionControlRequest | null {
368
+ const match = TERMINAL_SESSION_CONTROL_REGEX.exec(text);
369
+ if (!match) {
370
+ return null;
371
+ }
372
+ if (match[1] === END_CALL_MARKER) {
373
+ return { action: "end" };
374
+ }
375
+ if (match[2] !== undefined) {
376
+ return {
377
+ action: "updates",
378
+ cadence: match[2] === "FEWER" ? "fewer" : "normal",
379
+ };
380
+ }
381
+ const seconds = match[3] === undefined ? NaN : Number(match[3].trim());
382
+ if (
383
+ Number.isFinite(seconds) &&
384
+ seconds > 0 &&
385
+ seconds <= MAX_TIMED_MUTE_SECONDS
386
+ ) {
387
+ return { action: "mute", durationMs: Math.round(seconds * 1000) };
388
+ }
389
+ return { action: "mute" };
390
+ }
391
+
392
+ /**
393
+ * Length of the marker a row's text ends with, ignoring trailing whitespace,
394
+ * when it is one the transcript hygiene pass strips from the persisted row
395
+ * (the minimize marker or a session control); 0 otherwise.
396
+ */
397
+ export function terminalControlMarkerLength(text: string): number {
398
+ const trimmed = text.trimEnd();
399
+ if (trimmed.endsWith(MINIMIZE_ROOM_MARKER)) {
400
+ return MINIMIZE_ROOM_MARKER.length;
401
+ }
402
+ return TERMINAL_SESSION_CONTROL_REGEX.exec(trimmed)?.[1]?.length ?? 0;
403
+ }