@vellumai/assistant 0.12.1 → 0.12.2-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (548) hide show
  1. package/AGENTS.md +4 -0
  2. package/ARCHITECTURE.md +20 -21
  3. package/docs/architecture/turn-actor.md +15 -8
  4. package/docs/guardian-request-flow.md +16 -0
  5. package/openapi.yaml +327 -52
  6. package/package.json +3 -1
  7. package/scripts/smoke-desktop-window-theme.ts +186 -0
  8. package/src/__tests__/always-loaded-tools-guard.test.ts +18 -10
  9. package/src/__tests__/anthropic-provider.test.ts +0 -63
  10. package/src/__tests__/app-compiler.test.ts +6 -0
  11. package/src/__tests__/app-control-tool-schemas.test.ts +5 -2
  12. package/src/__tests__/app-dir-path-guard.test.ts +1 -3
  13. package/src/__tests__/assistant-event-hub-machine-name.test.ts +6 -7
  14. package/src/__tests__/assistant-event-hub.test.ts +44 -0
  15. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
  16. package/src/__tests__/computer-use-skill-proxy-bridge.test.ts +5 -2
  17. package/src/__tests__/computer-use-tools.test.ts +19 -7
  18. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  19. package/src/__tests__/config-schema.test.ts +28 -29
  20. package/src/__tests__/config-watcher.test.ts +23 -8
  21. package/src/__tests__/connection-model-compat.test.ts +2 -2
  22. package/src/__tests__/conversation-key-store-disk-view.test.ts +1 -1
  23. package/src/__tests__/conversation-pairing.test.ts +26 -0
  24. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +4 -10
  25. package/src/__tests__/conversation-queue.test.ts +48 -207
  26. package/src/__tests__/conversation-retry-discard.test.ts +54 -1
  27. package/src/__tests__/conversation-skill-tools.test.ts +27 -0
  28. package/src/__tests__/conversation-surfaces-app-control.test.ts +8 -3
  29. package/src/__tests__/conversation-surfaces-task-progress.test.ts +67 -0
  30. package/src/__tests__/conversation-title-service.test.ts +14 -10
  31. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +3 -5
  32. package/src/__tests__/conversation-tool-setup-attribution.test.ts +51 -0
  33. package/src/__tests__/credential-record-retire.test.ts +171 -1
  34. package/src/__tests__/credential-routes.test.ts +212 -3
  35. package/src/__tests__/credential-security-invariants.test.ts +4 -2
  36. package/src/__tests__/cu-unified-flow.test.ts +8 -2
  37. package/src/__tests__/document-sync-tags.test.ts +25 -0
  38. package/src/__tests__/drain-kick-guard.test.ts +0 -2
  39. package/src/__tests__/drain-requeue-on-contention.test.ts +0 -6
  40. package/src/__tests__/evict-conversations-for-reload.test.ts +1 -1
  41. package/src/__tests__/git-porcelain.test.ts +47 -0
  42. package/src/__tests__/heartbeat-service.test.ts +56 -40
  43. package/src/__tests__/helpers/set-workspace-mcp.ts +11 -0
  44. package/src/__tests__/host-bash-proxy.test.ts +26 -45
  45. package/src/__tests__/host-browser-proxy.test.ts +6 -17
  46. package/src/__tests__/host-cu-proxy.test.ts +539 -0
  47. package/src/__tests__/host-file-proxy-targeted.test.ts +23 -0
  48. package/src/__tests__/host-file-proxy.test.ts +11 -23
  49. package/src/__tests__/host-transfer-proxy-targeted.test.ts +30 -9
  50. package/src/__tests__/list-messages-camera-frame.test.ts +121 -0
  51. package/src/__tests__/llm-catalog-parity.test.ts +25 -0
  52. package/src/__tests__/mcp-auth-routes.test.ts +6 -9
  53. package/src/__tests__/mcp-config-secret-boundary.test.ts +78 -136
  54. package/src/__tests__/mcp-health-check.test.ts +8 -11
  55. package/src/__tests__/mcp-list-plugin-servers.test.ts +8 -10
  56. package/src/__tests__/message-deleted-hook-dispatch.test.ts +119 -0
  57. package/src/__tests__/messages-after-tiebreaker.test.ts +91 -0
  58. package/src/__tests__/messages-read-boundary-guard.test.ts +1 -1
  59. package/src/__tests__/messaging-send-tool.test.ts +355 -548
  60. package/src/__tests__/model-intents.test.ts +3 -0
  61. package/src/__tests__/notification-vellum-adapter.test.ts +85 -2
  62. package/src/__tests__/oauth-commands-routes.test.ts +113 -0
  63. package/src/__tests__/oauth-provider-profiles.test.ts +18 -0
  64. package/src/__tests__/openai-provider.test.ts +1 -0
  65. package/src/__tests__/outlook-messaging-provider.test.ts +6 -6
  66. package/src/__tests__/platform-callback-registration.test.ts +8 -16
  67. package/src/__tests__/platform-client-verify-credential.test.ts +2 -0
  68. package/src/__tests__/plugin-import-boundary-guard.test.ts +14 -0
  69. package/src/__tests__/qdrant-manager.test.ts +132 -0
  70. package/src/__tests__/resolve-app.test.ts +135 -0
  71. package/src/__tests__/scaffold-managed-skill-tool.test.ts +229 -6
  72. package/src/__tests__/script-proxy-certs.test.ts +35 -0
  73. package/src/__tests__/service-account-token.test.ts +247 -0
  74. package/src/__tests__/skill-feature-flags-integration.test.ts +1 -1
  75. package/src/__tests__/skill-feature-flags.test.ts +2 -2
  76. package/src/__tests__/skills.test.ts +9 -4
  77. package/src/__tests__/source-fingerprint.test.ts +184 -0
  78. package/src/__tests__/subagent-tool-gate-mode.test.ts +19 -3
  79. package/src/__tests__/subagent-tools.test.ts +3 -2
  80. package/src/__tests__/telephony-tts-guard.test.ts +38 -3
  81. package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
  82. package/src/__tests__/unicode.test.ts +13 -0
  83. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +4 -0
  84. package/src/__tests__/watch-retro-tool-availability.test.ts +8 -17
  85. package/src/__tests__/workspace-git-service.test.ts +26 -0
  86. package/src/__tests__/workspace-migration-154-repair-retired-codex-gpt-5-4-model-ids.test.ts +336 -0
  87. package/src/acp/__tests__/agent-process.test.ts +258 -4
  88. package/src/acp/__tests__/auth-required.test.ts +96 -0
  89. package/src/acp/__tests__/client-handler.test.ts +90 -2
  90. package/src/acp/__tests__/helpers/acp-model-option.ts +51 -0
  91. package/src/acp/__tests__/session-manager-persistence.test.ts +5 -1
  92. package/src/acp/__tests__/session-manager-resume.test.ts +270 -6
  93. package/src/acp/__tests__/session-manager-usage.test.ts +4 -2
  94. package/src/acp/__tests__/session-manager.test.ts +895 -4
  95. package/src/acp/agent-process.test.ts +4 -1
  96. package/src/acp/agent-process.ts +96 -10
  97. package/src/acp/auth-required.ts +50 -1
  98. package/src/acp/auto-install.test.ts +9 -7
  99. package/src/acp/client-handler.ts +16 -3
  100. package/src/acp/model-config.test.ts +166 -0
  101. package/src/acp/model-config.ts +138 -0
  102. package/src/acp/resolve-agent.test.ts +220 -6
  103. package/src/acp/resolve-agent.ts +88 -23
  104. package/src/acp/session-manager.test.ts +196 -3
  105. package/src/acp/session-manager.ts +441 -18
  106. package/src/acp/types.ts +50 -5
  107. package/src/agent/loop.ts +1 -0
  108. package/src/api/events/acp-session-model-update.test.ts +88 -0
  109. package/src/api/events/acp-session-model-update.ts +46 -0
  110. package/src/api/events/app-files-changed.ts +4 -3
  111. package/src/api/events/notification-intent.ts +2 -0
  112. package/src/api/events/user-message-echo.ts +7 -1
  113. package/src/api/index.ts +6 -0
  114. package/src/api/responses/conversation-message.ts +5 -0
  115. package/src/apps/app-store.ts +1 -1
  116. package/src/apps/resolve-app.ts +81 -0
  117. package/src/apps/source-fingerprint.ts +266 -0
  118. package/src/avatar/identity-avatar.ts +7 -2
  119. package/src/bundler/app-compiler.ts +15 -0
  120. package/src/calls/__tests__/barge-in-guard.test.ts +81 -0
  121. package/src/{__tests__ → calls/__tests__}/call-controller.test.ts +503 -82
  122. package/src/{__tests__ → calls/__tests__}/media-stream-audio-transcode.test.ts +1 -1
  123. package/src/{__tests__ → calls/__tests__}/media-stream-output.test.ts +37 -8
  124. package/src/{__tests__ → calls/__tests__}/media-stream-parser.test.ts +1 -1
  125. package/src/{__tests__ → calls/__tests__}/media-stream-server-integration.test.ts +150 -57
  126. package/src/{__tests__ → calls/__tests__}/media-stream-stt-session.test.ts +5 -5
  127. package/src/{__tests__ → calls/__tests__}/media-turn-detector.test.ts +1 -1
  128. package/src/{live-voice → calls}/__tests__/progress-narration.test.ts +4 -4
  129. package/src/{live-voice → calls}/__tests__/progress-phrases.test.ts +1 -1
  130. package/src/calls/__tests__/voice-control-protocol.test.ts +58 -25
  131. package/src/calls/__tests__/voice-leg-coordinator.test.ts +229 -0
  132. package/src/calls/__tests__/voice-progress-cadence.test.ts +274 -0
  133. package/src/{__tests__/voice-session-bridge.test.ts → calls/__tests__/voice-session-bridge-turn-flow.test.ts} +20 -12
  134. package/src/calls/__tests__/voice-session-bridge.test.ts +37 -1
  135. package/src/calls/__tests__/voice-triage-escalate.test.ts +105 -0
  136. package/src/calls/barge-in-guard.ts +118 -0
  137. package/src/calls/call-constants.ts +24 -4
  138. package/src/calls/call-controller.ts +499 -199
  139. package/src/calls/call-transport.ts +16 -0
  140. package/src/calls/media-stream-output.ts +32 -0
  141. package/src/calls/media-stream-server.ts +82 -30
  142. package/src/calls/media-stream-stt-session.ts +15 -0
  143. package/src/{live-voice → calls}/progress-narration.ts +2 -2
  144. package/src/calls/resolve-call-tts-provider.ts +33 -2
  145. package/src/calls/tts-call-strategy.ts +9 -5
  146. package/src/calls/types.ts +2 -0
  147. package/src/calls/voice-control-protocol.ts +53 -21
  148. package/src/calls/voice-leg-coordinator.ts +189 -0
  149. package/src/calls/voice-progress-cadence.ts +423 -0
  150. package/src/calls/voice-session-bridge.ts +59 -46
  151. package/src/calls/voice-triage-escalate.ts +175 -56
  152. package/src/cli/commands/__tests__/apps.test.ts +245 -0
  153. package/src/cli/commands/__tests__/bash.test.ts +53 -0
  154. package/src/cli/commands/__tests__/roadmap.help.test.ts +19 -0
  155. package/src/cli/commands/apps.help.ts +67 -13
  156. package/src/cli/commands/apps.ts +183 -0
  157. package/src/cli/commands/bash.help.ts +8 -13
  158. package/src/cli/commands/bash.ts +7 -25
  159. package/src/cli/commands/channels/__tests__/send.test.ts +218 -0
  160. package/src/cli/commands/channels/index.help.ts +70 -2
  161. package/src/cli/commands/channels/index.ts +16 -3
  162. package/src/cli/commands/channels/send.ts +128 -0
  163. package/src/cli/commands/credentials.help.ts +6 -2
  164. package/src/cli/commands/credentials.ts +2 -2
  165. package/src/cli/commands/mcp.help.ts +8 -8
  166. package/src/cli/commands/mcp.ts +1 -1
  167. package/src/cli/commands/roadmap.help.ts +5 -1
  168. package/src/cli/commands/roadmap.ts +1 -1
  169. package/src/cli/lib/bundled-marketplace.json +103 -90
  170. package/src/config/__tests__/default-profile-catalog.test.ts +22 -0
  171. package/src/config/__tests__/memory-retrospective-schema.test.ts +15 -1
  172. package/src/config/__tests__/platform-identity.test.ts +255 -0
  173. package/src/config/__tests__/platform-rehydration.test.ts +99 -9
  174. package/src/config/acp-agent-write.ts +43 -0
  175. package/src/config/acp-defaults.test.ts +13 -3
  176. package/src/config/acp-defaults.ts +23 -6
  177. package/src/config/acp-schema.test.ts +81 -0
  178. package/src/config/acp-schema.ts +30 -2
  179. package/src/config/bundled-skills/acp/SKILL.md +22 -14
  180. package/src/config/bundled-skills/acp/TOOLS.json +5 -1
  181. package/src/config/bundled-skills/app-builder/SKILL.md +12 -9
  182. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +1 -1
  183. package/src/config/bundled-skills/computer-use/SKILL.md +20 -2
  184. package/src/config/bundled-skills/computer-use/TOOLS.json +99 -0
  185. package/src/config/bundled-skills/computer-use/tools/computer-use-sequence.ts +12 -0
  186. package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -1
  187. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -1
  188. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -1
  189. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  190. package/src/config/bundled-skills/messaging/TOOLS.json +3 -3
  191. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +67 -139
  192. package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +1 -1
  193. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +2 -1
  194. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +2 -1
  195. package/src/config/bundled-skills/schedule/SKILL.md +3 -4
  196. package/src/config/bundled-skills/screen-annotation/SKILL.md +8 -1
  197. package/src/config/bundled-skills/screen-annotation/tools/screen-annotation.test.ts +4 -1
  198. package/src/config/bundled-skills/skill-management/TOOLS.json +6 -2
  199. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +5 -2
  200. package/src/config/default-profile-catalog.ts +1 -1
  201. package/src/config/env.ts +2 -2
  202. package/src/config/feature-flag-registry.json +24 -12
  203. package/src/config/loader.ts +1 -0
  204. package/src/config/memory-v3-gate.test.ts +31 -0
  205. package/src/config/memory-v3-gate.ts +20 -16
  206. package/src/config/platform-identity.ts +201 -0
  207. package/src/config/platform-rehydration.ts +11 -27
  208. package/src/config/schema.ts +2 -2
  209. package/src/config/schemas/__tests__/live-voice.test.ts +8 -158
  210. package/src/config/schemas/__tests__/voice.test.ts +148 -0
  211. package/src/config/schemas/calls.ts +1 -28
  212. package/src/config/schemas/live-voice.ts +1 -173
  213. package/src/config/schemas/mcp.ts +1 -1
  214. package/src/config/schemas/memory-retrospective.ts +10 -0
  215. package/src/config/schemas/services.ts +8 -0
  216. package/src/config/schemas/skills.ts +0 -14
  217. package/src/config/schemas/voice.ts +150 -0
  218. package/src/credential-execution/process-manager.test.ts +18 -19
  219. package/src/credential-health/credential-health-service.ts +11 -17
  220. package/src/daemon/__tests__/conversation-tool-setup.test.ts +66 -121
  221. package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +4 -1
  222. package/src/daemon/__tests__/skill-memory-refresh-mtime.test.ts +133 -0
  223. package/src/daemon/app-change-notify.ts +45 -0
  224. package/src/daemon/config-watcher.ts +13 -194
  225. package/src/daemon/conversation-error.ts +3 -2
  226. package/src/daemon/conversation-media-retry.ts +2 -1
  227. package/src/daemon/conversation-process.ts +35 -64
  228. package/src/daemon/conversation-runtime-assembly.ts +2 -1
  229. package/src/daemon/conversation-skill-tools.ts +13 -31
  230. package/src/daemon/conversation-store.ts +1 -1
  231. package/src/daemon/conversation-surfaces.ts +15 -9
  232. package/src/daemon/conversation-tool-setup.ts +37 -43
  233. package/src/daemon/conversation.ts +0 -3
  234. package/src/daemon/dictation-profile-store.ts +2 -1
  235. package/src/daemon/handlers/shared.ts +2 -1
  236. package/src/daemon/handlers/skills.ts +6 -3
  237. package/src/daemon/host-bash-proxy.ts +8 -12
  238. package/src/daemon/host-browser-proxy.ts +15 -1
  239. package/src/daemon/host-cu-proxy.ts +186 -19
  240. package/src/daemon/host-file-proxy.ts +8 -10
  241. package/src/daemon/host-proxy-preactivation.ts +2 -1
  242. package/src/daemon/host-transfer-proxy.ts +14 -18
  243. package/src/daemon/lifecycle.ts +0 -5
  244. package/src/daemon/mcp-reload-service.ts +5 -5
  245. package/src/daemon/message-types/sync.ts +1 -0
  246. package/src/daemon/orphan-reaper.ts +1 -1
  247. package/src/daemon/providers-setup.ts +1 -1
  248. package/src/daemon/reaction-history-render.ts +2 -1
  249. package/src/daemon/shutdown-handlers.ts +1 -9
  250. package/src/daemon/skill-memory-refresh.ts +64 -0
  251. package/src/daemon/tool-side-effects.ts +7 -18
  252. package/src/desktop/__tests__/fake-desktop.ts +122 -0
  253. package/src/desktop/desktop-chrome-policy.test.ts +79 -0
  254. package/src/desktop/desktop-chrome-policy.ts +47 -0
  255. package/src/desktop/desktop-chrome-session.ts +37 -0
  256. package/src/desktop/desktop-dependencies.test.ts +102 -0
  257. package/src/desktop/desktop-dependencies.ts +308 -0
  258. package/src/desktop/desktop-feature.test.ts +18 -0
  259. package/src/desktop/desktop-feature.ts +19 -0
  260. package/src/desktop/desktop-panel-config.test.ts +135 -0
  261. package/src/desktop/desktop-panel-config.ts +149 -0
  262. package/src/desktop/desktop-session-manager.test.ts +730 -0
  263. package/src/desktop/desktop-session-manager.ts +850 -0
  264. package/src/desktop/desktop-stream-bridge.test.ts +395 -0
  265. package/src/desktop/desktop-stream-bridge.ts +214 -0
  266. package/src/desktop/desktop-wallpaper.test.ts +191 -0
  267. package/src/desktop/desktop-wallpaper.ts +113 -0
  268. package/src/desktop/desktop-window-drag.ts +177 -0
  269. package/src/desktop/desktop-window-manager-config.test.ts +230 -0
  270. package/src/desktop/desktop-window-manager-config.ts +238 -0
  271. package/src/desktop/desktop-window-theme.ts +170 -0
  272. package/src/heartbeat/heartbeat-service.ts +21 -3
  273. package/src/home/__tests__/feed-writer.test.ts +22 -0
  274. package/src/home/feed-writer.ts +11 -2
  275. package/src/hooks/types.ts +33 -0
  276. package/src/i18n/AGENTS.md +22 -0
  277. package/src/i18n/__tests__/i18n.test.ts +108 -0
  278. package/src/i18n/index.ts +16 -0
  279. package/src/i18n/locales.ts +93 -0
  280. package/src/i18n/messages.ts +61 -0
  281. package/src/i18n/title.ts +25 -0
  282. package/src/inbound/platform-callback-registration.ts +4 -5
  283. package/src/live-voice/__tests__/live-voice-metrics.test.ts +0 -3
  284. package/src/live-voice/__tests__/live-voice-progress.test.ts +15 -15
  285. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +238 -0
  286. package/src/live-voice/__tests__/live-voice-vad.test.ts +2 -2
  287. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +13 -69
  288. package/src/live-voice/live-voice-photo.ts +25 -6
  289. package/src/live-voice/live-voice-session.ts +213 -693
  290. package/src/live-voice/protocol.ts +0 -2
  291. package/src/mcp/__tests__/effective-config.test.ts +2 -2
  292. package/src/mcp/__tests__/workspace-mcp-config.test.ts +135 -0
  293. package/src/mcp/effective-config.ts +3 -3
  294. package/src/mcp/mcp-header-store.ts +29 -45
  295. package/src/mcp/spec-schema.ts +111 -0
  296. package/src/mcp/startup.ts +8 -4
  297. package/src/mcp/workspace-mcp-config.ts +188 -0
  298. package/src/messaging/provider.ts +10 -3
  299. package/src/messaging/providers/__tests__/proactive-address.test.ts +95 -0
  300. package/src/messaging/providers/callback-routing.ts +18 -0
  301. package/src/messaging/providers/channel-transport.ts +56 -0
  302. package/src/messaging/providers/discord/transport.ts +17 -0
  303. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +6 -17
  304. package/src/messaging/providers/slack/adapter.ts +7 -44
  305. package/src/messaging/providers/slack/transport.ts +16 -0
  306. package/src/messaging/providers/telegram-bot/adapter.ts +1 -48
  307. package/src/messaging/providers/telegram-bot/client.ts +0 -26
  308. package/src/messaging/providers/telegram-bot/transport.ts +20 -0
  309. package/src/messaging/providers/whatsapp/adapter.ts +1 -48
  310. package/src/messaging/providers/whatsapp/transport.ts +18 -0
  311. package/src/monitoring/__tests__/workspace-git-heartbeat.test.ts +8 -0
  312. package/src/notifications/__tests__/assistant-reply-producer.test.ts +55 -0
  313. package/src/notifications/__tests__/guardian-feed-projection.test.ts +158 -2
  314. package/src/notifications/__tests__/home-feed-side-effect.test.ts +153 -3
  315. package/src/notifications/__tests__/schedule-result-producer.test.ts +99 -3
  316. package/src/notifications/adapters/macos.ts +3 -0
  317. package/src/notifications/assistant-reply-producer.ts +8 -3
  318. package/src/notifications/conversation-pairing.ts +36 -16
  319. package/src/notifications/delivered-post-record.ts +11 -1
  320. package/src/notifications/guardian-feed-projection.ts +125 -5
  321. package/src/notifications/home-feed-side-effect.ts +55 -7
  322. package/src/notifications/schedule-result-producer.ts +66 -22
  323. package/src/oauth/AGENTS.md +2 -0
  324. package/src/oauth/__tests__/seed-providers-managed.test.ts +54 -0
  325. package/src/oauth/byo-connection.test.ts +105 -0
  326. package/src/oauth/byo-connection.ts +36 -6
  327. package/src/oauth/connection-resolver.test.ts +73 -0
  328. package/src/oauth/connection-resolver.ts +95 -2
  329. package/src/oauth/scope-utils.ts +24 -0
  330. package/src/oauth/seed-providers.ts +204 -25
  331. package/src/outbound-proxy/certs.ts +3 -5
  332. package/src/permissions/checker.test.ts +11 -6
  333. package/src/persistence/compaction-log-store-clickhouse.ts +3 -3
  334. package/src/persistence/conversation-crud.ts +71 -63
  335. package/src/persistence/conversation-lineage.ts +12 -4
  336. package/src/persistence/conversation-queries.ts +2 -2
  337. package/src/persistence/conversation-title-placeholders.ts +23 -0
  338. package/src/persistence/conversation-title-service.ts +9 -26
  339. package/src/persistence/embeddings/qdrant-manager.ts +163 -18
  340. package/src/persistence/llm-request-log-sink-clickhouse.ts +3 -3
  341. package/src/persistence/llm-request-log-source-clickhouse.ts +3 -3
  342. package/src/persistence/message-cursor.ts +78 -0
  343. package/src/persistence/message-reads.ts +30 -0
  344. package/src/persistence/migrations/157-invite-contact-id.ts +18 -6
  345. package/src/persistence/migrations/__tests__/157-invite-contact-id.test.ts +149 -0
  346. package/src/persistence/schema/memory-core.ts +5 -0
  347. package/src/platform/client.test.ts +4 -2
  348. package/src/platform/client.ts +2 -10
  349. package/src/plugin-api/constants.ts +2 -0
  350. package/src/plugin-api/index.ts +1 -0
  351. package/src/plugin-api/types.ts +2 -0
  352. package/src/plugins/defaults/index.ts +2 -0
  353. package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +46 -0
  354. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-backfill.test.ts +118 -0
  355. package/src/plugins/defaults/memory/__tests__/memory-retrospective-cursor-preserve.test.ts +83 -0
  356. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +9 -5
  357. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +20 -10
  358. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +5 -0
  359. package/src/plugins/defaults/memory/__tests__/memory-retrospective-state.test.ts +219 -0
  360. package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +31 -0
  361. package/src/plugins/defaults/memory/__tests__/memory-retrospective-trigger-check.test.ts +63 -0
  362. package/src/plugins/defaults/memory/__tests__/shutdown-hook.test.ts +4 -0
  363. package/src/plugins/defaults/memory/context-search/agent-runner.ts +3 -2
  364. package/src/plugins/defaults/memory/context-search/format.ts +3 -2
  365. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +6 -3
  366. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +3 -2
  367. package/src/plugins/defaults/memory/graph/capability-seed.ts +6 -7
  368. package/src/plugins/defaults/memory/hooks/init.ts +10 -0
  369. package/src/plugins/defaults/memory/hooks/message-deleted.ts +21 -0
  370. package/src/plugins/defaults/memory/hooks/shutdown.ts +2 -0
  371. package/src/plugins/defaults/memory/host-utils.ts +10 -0
  372. package/src/plugins/defaults/memory/jobs-worker.ts +13 -0
  373. package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +34 -60
  374. package/src/plugins/defaults/memory/memory-retrospective-cursor-backfill.ts +126 -0
  375. package/src/plugins/defaults/memory/memory-retrospective-cursor-preserve.ts +79 -0
  376. package/src/plugins/defaults/memory/memory-retrospective-cursor.ts +28 -0
  377. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +3 -5
  378. package/src/plugins/defaults/memory/memory-retrospective-job.ts +12 -3
  379. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  380. package/src/plugins/defaults/memory/memory-retrospective-state.ts +312 -71
  381. package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +2 -1
  382. package/src/plugins/defaults/memory/memory-retrospective-trigger-check.ts +5 -3
  383. package/src/plugins/defaults/memory/skill-md-mtime-poll.ts +9 -0
  384. package/src/plugins/defaults/memory/startup.ts +28 -2
  385. package/src/plugins/defaults/memory/substrate/__tests__/skill-content.test.ts +7 -9
  386. package/src/plugins/defaults/memory/substrate/page-index.ts +6 -1
  387. package/src/plugins/defaults/memory/substrate/skill-content.ts +2 -2
  388. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +4 -2
  389. package/src/plugins/defaults/memory/v1/graph/extraction.ts +4 -2
  390. package/src/plugins/defaults/memory/v1/graph/retriever.ts +4 -1
  391. package/src/plugins/defaults/memory/v2/reranker.ts +6 -1
  392. package/src/plugins/defaults/memory/v3/__tests__/card.test.ts +24 -0
  393. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +125 -0
  394. package/src/plugins/defaults/memory/v3/__tests__/sections.test.ts +26 -0
  395. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +30 -0
  396. package/src/plugins/defaults/memory/v3/card.ts +3 -2
  397. package/src/plugins/defaults/memory/v3/pool-select.ts +5 -3
  398. package/src/plugins/defaults/memory/v3/sections.ts +9 -3
  399. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +5 -2
  400. package/src/plugins/defaults/memory/worker.ts +6 -0
  401. package/src/plugins/mcp-servers.ts +15 -75
  402. package/src/providers/__tests__/connection-model-compat.test.ts +3 -3
  403. package/src/providers/__tests__/retry-callsite.test.ts +41 -0
  404. package/src/providers/__tests__/retry-outbound-sanitize.test.ts +176 -0
  405. package/src/providers/anthropic/client.ts +0 -47
  406. package/src/providers/gemini/client.ts +9 -23
  407. package/src/providers/gemini-thought-signature.test.ts +63 -0
  408. package/src/providers/gemini-thought-signature.ts +44 -0
  409. package/src/providers/inference/__tests__/endpoint-probe.test.ts +42 -3
  410. package/src/providers/inference/auth.ts +2 -3
  411. package/src/providers/inference/connection-availability.ts +27 -8
  412. package/src/providers/inference/endpoint-probe.ts +10 -1
  413. package/src/providers/inference/resolve-auth.ts +22 -4
  414. package/src/providers/inference/service-account-token.ts +248 -0
  415. package/src/providers/model-catalog.ts +34 -0
  416. package/src/providers/model-intents.ts +10 -5
  417. package/src/providers/openai/__tests__/chat-completions-output-token-limit.test.ts +141 -0
  418. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +146 -18
  419. package/src/providers/openai/__tests__/chat-completions-provider-thought-signature.test.ts +613 -0
  420. package/src/providers/openai/__tests__/google-thought-signature.test.ts +235 -0
  421. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +83 -1
  422. package/src/providers/openai/chat-completions-provider.ts +212 -79
  423. package/src/providers/openai/codex-models.ts +0 -4
  424. package/src/providers/openai/google-thought-signature.ts +320 -0
  425. package/src/providers/opencode/client.test.ts +73 -4
  426. package/src/providers/opencode/client.ts +60 -3
  427. package/src/providers/openrouter/client.ts +20 -0
  428. package/src/providers/outbound-request-sanitize.ts +96 -0
  429. package/src/providers/retry.ts +30 -19
  430. package/src/providers/unparseable-tool-args.ts +5 -1
  431. package/src/runtime/AGENTS.md +3 -1
  432. package/src/runtime/__tests__/channel-send.test.ts +443 -0
  433. package/src/runtime/__tests__/desktop-stream-upgrade.test.ts +89 -0
  434. package/src/runtime/__tests__/host-observe.test.ts +40 -1
  435. package/src/runtime/__tests__/live-voice-upgrade.test.ts +58 -0
  436. package/src/runtime/__tests__/runtime-ws-test-utils.ts +72 -0
  437. package/src/runtime/__tests__/watch-stream-upgrade.test.ts +59 -0
  438. package/src/runtime/agent-wake.ts +2 -1
  439. package/src/runtime/assistant-event-hub.ts +21 -3
  440. package/src/runtime/auth/same-actor.ts +38 -3
  441. package/src/runtime/channel-send.ts +316 -0
  442. package/src/runtime/host-observe.ts +11 -1
  443. package/src/runtime/http-server.ts +160 -151
  444. package/src/runtime/routes/__tests__/acp-routes.test.ts +159 -4
  445. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +205 -0
  446. package/src/runtime/routes/__tests__/channel-send-routes.test.ts +127 -0
  447. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +108 -1
  448. package/src/runtime/routes/__tests__/default-provider-routes.test.ts +2 -2
  449. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +107 -0
  450. package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +16 -12
  451. package/src/runtime/routes/acp-routes.test.ts +15 -0
  452. package/src/runtime/routes/acp-routes.ts +72 -9
  453. package/src/runtime/routes/app-management-routes.ts +73 -0
  454. package/src/runtime/routes/app-routes.ts +1 -1
  455. package/src/runtime/routes/channel-send-routes.ts +85 -0
  456. package/src/runtime/routes/conversation-list-routes.ts +14 -3
  457. package/src/runtime/routes/conversation-query-routes.ts +40 -0
  458. package/src/runtime/routes/conversation-routes.ts +16 -3
  459. package/src/runtime/routes/credential-routes.ts +19 -10
  460. package/src/runtime/routes/desktop-setup-routes.test.ts +74 -0
  461. package/src/runtime/routes/desktop-setup-routes.ts +35 -0
  462. package/src/runtime/routes/diagnostics-routes.ts +61 -4
  463. package/src/runtime/routes/events-routes.ts +4 -0
  464. package/src/runtime/routes/host-cu-routes.ts +6 -0
  465. package/src/runtime/routes/index.ts +4 -2
  466. package/src/runtime/routes/log-export-routes.ts +2 -0
  467. package/src/runtime/routes/mcp-auth-routes.ts +39 -47
  468. package/src/runtime/routes/migration-routes.ts +6 -20
  469. package/src/runtime/routes/oauth-commands-routes.ts +36 -0
  470. package/src/runtime/routes/platform-routes.ts +18 -15
  471. package/src/runtime/routes/workspace-routes.test.ts +55 -26
  472. package/src/runtime/routes/workspace-routes.ts +46 -63
  473. package/src/runtime/routes/workspace-tree-walk.test.ts +244 -0
  474. package/src/runtime/routes/workspace-tree-walk.ts +240 -0
  475. package/src/runtime/routes/workspace-utils.ts +17 -0
  476. package/src/runtime/services/__tests__/conversation-serializer.test.ts +28 -0
  477. package/src/runtime/services/conversation-serializer.ts +16 -3
  478. package/src/runtime/sync/resource-sync-events.ts +2 -1
  479. package/src/schedule/run-script.ts +2 -1
  480. package/src/schedule/schedule-store.ts +9 -2
  481. package/src/schedule/worker.ts +1 -1
  482. package/src/security/__tests__/ces-http-record-backend.test.ts +137 -0
  483. package/src/security/ces-http-record-backend.ts +120 -0
  484. package/src/security/ces-rpc-record-backend.ts +7 -0
  485. package/src/security/untrusted-content.ts +3 -1
  486. package/src/skills/__tests__/workspace-skill-md-signature.test.ts +60 -0
  487. package/src/skills/workspace-skill-md-signature.ts +50 -0
  488. package/src/subagent/consult-context.ts +12 -25
  489. package/src/tools/__tests__/side-effect-abort-guard.test.ts +1 -0
  490. package/src/tools/acp/list-agents.test.ts +3 -1
  491. package/src/tools/acp/spawn.test.ts +146 -11
  492. package/src/tools/acp/spawn.ts +21 -3
  493. package/src/tools/acp/status.test.ts +94 -0
  494. package/src/tools/acp/status.ts +40 -3
  495. package/src/tools/app-control/skill-proxy-bridge.ts +2 -1
  496. package/src/tools/browser/browser-execution.ts +1 -1
  497. package/src/tools/browser/cdp-client/accessibility-snapshot.ts +9 -3
  498. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +2 -1
  499. package/src/tools/browser/cdp-client/extension-cdp-client.ts +3 -2
  500. package/src/tools/browser/network-recorder.ts +2 -1
  501. package/src/tools/capability-offer.test.ts +22 -2
  502. package/src/tools/capability-offer.ts +16 -1
  503. package/src/tools/computer-use/definitions.ts +121 -2
  504. package/src/tools/computer-use/skill-proxy-bridge.ts +2 -1
  505. package/src/tools/credentials/metadata-store.ts +100 -0
  506. package/src/tools/filesystem/search.ts +2 -1
  507. package/src/tools/host-filesystem/edit.test.ts +2 -1
  508. package/src/tools/host-filesystem/edit.ts +2 -2
  509. package/src/tools/host-filesystem/read.test.ts +2 -1
  510. package/src/tools/host-filesystem/read.ts +2 -2
  511. package/src/tools/host-filesystem/transfer.test.ts +2 -1
  512. package/src/tools/host-filesystem/transfer.ts +2 -2
  513. package/src/tools/host-filesystem/write.test.ts +2 -1
  514. package/src/tools/host-filesystem/write.ts +2 -2
  515. package/src/tools/host-terminal/host-shell.test.ts +16 -0
  516. package/src/tools/host-terminal/host-shell.ts +4 -20
  517. package/src/tools/network/web-search.ts +6 -1
  518. package/src/tools/policy-context.ts +6 -7
  519. package/src/tools/skills/scaffold-managed.ts +61 -3
  520. package/src/tools/subagent/spawn.ts +5 -8
  521. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +57 -0
  522. package/src/tools/terminal/sanitized-bash.ts +105 -0
  523. package/src/tools/watch/watch-retro-report.ts +0 -8
  524. package/src/types/host-capabilities.ts +5 -3
  525. package/src/util/__tests__/language-subtag.test.ts +23 -1
  526. package/src/util/ca-bundle.ts +13 -0
  527. package/src/util/language-subtag.ts +16 -0
  528. package/src/util/platform.ts +8 -0
  529. package/src/util/unicode.ts +15 -1
  530. package/src/watch/watch-timeline.ts +2 -2
  531. package/src/workflows/run-manager.ts +2 -1
  532. package/src/workspace/git-porcelain.ts +66 -0
  533. package/src/workspace/git-service.ts +6 -55
  534. package/src/workspace/migrations/154-repair-retired-codex-gpt-5-4-model-ids.ts +258 -0
  535. package/src/workspace/migrations/155-move-front-model-config-to-voice.ts +78 -0
  536. package/src/workspace/migrations/156-extract-workspace-mcp-json.ts +214 -0
  537. package/src/workspace/migrations/__tests__/155-move-front-model-config-to-voice.test.ts +102 -0
  538. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +199 -0
  539. package/src/workspace/migrations/registry.ts +6 -0
  540. package/src/__tests__/app-source-watcher.test.ts +0 -218
  541. package/src/__tests__/config-watcher-skill-reseed.test.ts +0 -359
  542. package/src/__tests__/speaker-identification.test.ts +0 -56
  543. package/src/calls/speaker-identification.ts +0 -233
  544. package/src/daemon/app-source-watcher.ts +0 -218
  545. package/src/messaging/providers/whatsapp/client.ts +0 -29
  546. package/src/runtime/routes/debug-bash-routes.ts +0 -173
  547. package/src/util/fs-watcher-error.ts +0 -36
  548. /package/src/{live-voice → calls}/progress-phrases.ts +0 -0
@@ -98,4 +98,20 @@ export interface CallTransport {
98
98
  * this — the controller then treats playback as already drained.
99
99
  */
100
100
  awaitPlaybackDrained?(): Promise<void>;
101
+
102
+ /**
103
+ * Whether no assistant speech is buffered, queued, in flight, or
104
+ * (estimatedly) still playing to the caller. Progress narration speaks
105
+ * only into audible silence, so it asks before every update. Transports
106
+ * that play speech synchronously may omit this; the controller then
107
+ * treats the transport as idle.
108
+ */
109
+ isPlaybackIdle?(): boolean;
110
+
111
+ /**
112
+ * Wall-clock estimate of when audio already sent finishes playing to the
113
+ * caller, 0 when nothing is playing. Anchors the dead-air countdown for
114
+ * progress narration to when the caller last heard something.
115
+ */
116
+ playbackTailUntilMs?(): number;
101
117
  }
@@ -327,6 +327,14 @@ export class MediaStreamOutput implements CallTransport {
327
327
  /** True when the queue drain loop is actively running. */
328
328
  private draining = false;
329
329
 
330
+ /**
331
+ * Estimated wall-clock instant the audio already sent to Twilio finishes
332
+ * playing to the caller. Frames leave faster than real time and Twilio
333
+ * buffers them, so the only local view of "still audible" is this running
334
+ * sum of sent frame durations; a `clear` drops the buffer and resets it.
335
+ */
336
+ private playbackTailAtMs = 0;
337
+
330
338
  /** Abort controller for the currently in-flight synthesis/fetch. */
331
339
  private activePlaybackAbort: AbortController | null = null;
332
340
 
@@ -659,6 +667,7 @@ export class MediaStreamOutput implements CallTransport {
659
667
  return;
660
668
  }
661
669
  this.sendClearCommand();
670
+ this.playbackTailAtMs = 0;
662
671
  // Twilio drops its buffered audio (and the marks within it) on `clear`, so
663
672
  // any end-of-turn mark already sent will never echo. Treat those as drained
664
673
  // so a pending end-call drain wait resolves instead of stalling on the cap.
@@ -716,6 +725,24 @@ export class MediaStreamOutput implements CallTransport {
716
725
  return this.playbackQueue.length;
717
726
  }
718
727
 
728
+ /**
729
+ * Whether no speech is buffered toward a sentence, queued, being
730
+ * synthesized or fetched, or (estimatedly) still playing at Twilio.
731
+ */
732
+ isPlaybackIdle(): boolean {
733
+ return (
734
+ this.textBuffer.length === 0 &&
735
+ this.playbackQueue.length === 0 &&
736
+ !this.draining &&
737
+ Date.now() >= this.playbackTailAtMs
738
+ );
739
+ }
740
+
741
+ /** See {@link CallTransport.playbackTailUntilMs}. */
742
+ playbackTailUntilMs(): number {
743
+ return this.playbackTailAtMs;
744
+ }
745
+
719
746
  /**
720
747
  * Runtime check for closed state. Used instead of direct property access
721
748
  * in async methods because TypeScript's control flow analysis cannot
@@ -772,6 +799,7 @@ export class MediaStreamOutput implements CallTransport {
772
799
  this.playbackQueue.length = 0;
773
800
  this.playbackVersion++;
774
801
  this.audioStartCallback = null;
802
+ this.playbackTailAtMs = 0;
775
803
  if (this.activePlaybackAbort) {
776
804
  this.activePlaybackAbort.abort();
777
805
  this.activePlaybackAbort = null;
@@ -854,6 +882,10 @@ export class MediaStreamOutput implements CallTransport {
854
882
  for (const frame of frames) {
855
883
  this.sendAudioPayload(frame);
856
884
  }
885
+ const durationMs =
886
+ (frames.length * MULAW_FRAME_SIZE * 1000) / TELEPHONY_SAMPLE_RATE_HZ;
887
+ this.playbackTailAtMs =
888
+ Math.max(Date.now(), this.playbackTailAtMs) + durationMs;
857
889
  }
858
890
 
859
891
  /**
@@ -47,6 +47,11 @@ import { addMessage } from "../persistence/conversation-crud.js";
47
47
  import { resolveGuardianName } from "../prompts/user-reference.js";
48
48
  import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
49
49
  import { getLogger } from "../util/logger.js";
50
+ import {
51
+ type BargeInGuard,
52
+ createBargeInGuard,
53
+ DEFAULT_BARGE_IN_MIN_SPEECH_MS,
54
+ } from "./barge-in-guard.js";
50
55
  import { CallController } from "./call-controller.js";
51
56
  import {
52
57
  formatDuration,
@@ -188,6 +193,8 @@ export class MediaStreamCallSession {
188
193
  private bargeInAccepted = 0;
189
194
  /** Number of barge-in attempts that were ignored (assistant not speaking). */
190
195
  private bargeInIgnored = 0;
196
+ /** Armed by caller speech while a turn or its tail is interruptible. */
197
+ private bargeInGuard: BargeInGuard | null = null;
191
198
  /** Number of turn-start transitions detected by the STT session. */
192
199
  private turnStarts = 0;
193
200
  /** Number of transcript finals produced (non-empty). */
@@ -215,6 +222,9 @@ export class MediaStreamCallSession {
215
222
  // Create STT session with callbacks wired to the controller.
216
223
  const callbacks: MediaStreamSttSessionCallbacks = {
217
224
  onSpeechStart: () => this.handleSpeechStart(),
225
+ onMediaFrame: (hasSpeech, durationMs) =>
226
+ this.handleMediaFrame(hasSpeech, durationMs),
227
+ onSpeechEnd: () => this.handleSpeechEnd(),
218
228
  onTranscriptFinal: (text, durationMs) =>
219
229
  this.handleTranscriptFinal(text, durationMs),
220
230
  onDtmf: (digit) => this.handleDtmf(digit),
@@ -768,38 +778,80 @@ export class MediaStreamCallSession {
768
778
 
769
779
  private handleSpeechStart(): void {
770
780
  this.turnStarts++;
781
+ }
771
782
 
772
- // Barge-in: clear queued outbound audio and abort the in-flight LLM
773
- // turn only when the assistant is actively speaking. Uses the gated
774
- // handleBargeIn path so initial inbound audio frames do not cancel a
775
- // still-starting initial turn.
776
- //
777
- // clearAudio runs via the onAccepted hook so it only fires when the
778
- // barge-in passes the speaking gate an ignored barge-in (controller
779
- // idle/processing) must not flush queued/in-flight TTS such as a
780
- // buffered greeting. The hook runs before handleInterrupt so the
781
- // end-of-turn mark it enqueues survives the queue flush.
782
- if (this.output && this.controller) {
783
- const output = this.output;
784
- const accepted = this.controller.handleBargeIn(() => output.clearAudio());
785
- if (accepted) {
786
- this.bargeInAccepted++;
787
- log.info(
788
- { callSessionId: this.callSessionId },
789
- "Media-stream barge-in accepted — cleared outbound audio",
790
- );
791
- } else {
792
- // No turn to abort, but a completed turn's tail can still be
793
- // playing from Twilio's buffer — flush only that buffer so the
794
- // caller isn't talked over. Queued speech that hasn't reached
795
- // Twilio yet (greeting, handoff prompt) is preserved.
796
- output.clearBufferedAudio();
797
- this.bargeInIgnored++;
798
- log.debug(
799
- { callSessionId: this.callSessionId },
800
- "Media-stream barge-in ignored — assistant not speaking",
801
- );
783
+ // Sustained-speech barge-in guard, fed by every inbound frame. Speech only
784
+ // counts while there is something to interrupt: an assistant turn in
785
+ // flight (thinking or speaking) or a completed turn's tail still playing
786
+ // from Twilio's buffer. Outside that window the guard is dropped, so
787
+ // speech from the caller's own utterance never carries into a turn that
788
+ // starts before the local VAD has ended the utterance (a streaming final
789
+ // can start the turn first). A speech frame inside the window arms the
790
+ // guard; a fired guard reaches the controller. A line click, a cough, or
791
+ // TTS bleed never gets that far.
792
+ private handleMediaFrame(hasSpeech: boolean, durationMs: number): void {
793
+ if (!this.output || !this.controller) {
794
+ return;
795
+ }
796
+ const bargeable =
797
+ this.controller.getState() !== "idle" || !this.output.isPlaybackIdle();
798
+ if (!bargeable) {
799
+ this.bargeInGuard = null;
800
+ return;
801
+ }
802
+ if (this.bargeInGuard === null) {
803
+ if (!hasSpeech) {
804
+ return;
802
805
  }
806
+ this.bargeInGuard = createBargeInGuard(DEFAULT_BARGE_IN_MIN_SPEECH_MS);
807
+ }
808
+ const step = this.bargeInGuard.track(
809
+ hasSpeech ? "speech" : "silence",
810
+ durationMs,
811
+ );
812
+ if (step === "fired") {
813
+ this.bargeInGuard = null;
814
+ this.fireBargeIn();
815
+ }
816
+ }
817
+
818
+ private handleSpeechEnd(): void {
819
+ // An untripped guard was noise or a fragment: the run never reached the
820
+ // threshold before the caller's turn ended.
821
+ this.bargeInGuard = null;
822
+ }
823
+
824
+ // Sustained caller speech landed: clear queued outbound audio and abort
825
+ // the in-flight LLM turn when one is in flight (thinking or speaking).
826
+ //
827
+ // clearAudio runs via the onAccepted hook so it only fires when the
828
+ // barge-in passes the controller's gate; an ignored barge-in (controller
829
+ // idle) must not flush queued/in-flight TTS such as a buffered greeting.
830
+ // The hook runs before handleInterrupt so the end-of-turn mark it
831
+ // enqueues survives the queue flush.
832
+ private fireBargeIn(): void {
833
+ if (!this.output || !this.controller) {
834
+ return;
835
+ }
836
+ const output = this.output;
837
+ const accepted = this.controller.handleBargeIn(() => output.clearAudio());
838
+ if (accepted) {
839
+ this.bargeInAccepted++;
840
+ log.info(
841
+ { callSessionId: this.callSessionId },
842
+ "Media-stream barge-in accepted: cleared outbound audio",
843
+ );
844
+ } else {
845
+ // No turn to abort, but a completed turn's tail can still be
846
+ // playing from Twilio's buffer, so flush only that buffer so the
847
+ // caller isn't talked over. Queued speech that hasn't reached
848
+ // Twilio yet (greeting, handoff prompt) is preserved.
849
+ output.clearBufferedAudio();
850
+ this.bargeInIgnored++;
851
+ log.debug(
852
+ { callSessionId: this.callSessionId },
853
+ "Media-stream barge-in ignored: no turn in flight",
854
+ );
803
855
  }
804
856
  }
805
857
 
@@ -120,6 +120,16 @@ export interface MediaStreamSttSessionCallbacks {
120
120
  /** Called when the turn detector transitions to active (first speech-bearing chunk). */
121
121
  onSpeechStart?: () => void;
122
122
 
123
+ /**
124
+ * Called for every inbound frame with the local VAD's verdict on it and
125
+ * the frame's duration. Drives the sustained-speech barge-in guard, which
126
+ * needs the silence frames inside a detector turn as well as the speech.
127
+ */
128
+ onMediaFrame?: (hasSpeech: boolean, durationMs: number) => void;
129
+
130
+ /** Called when the turn detector ends the caller's turn (silence or cap). */
131
+ onSpeechEnd?: () => void;
132
+
123
133
  /**
124
134
  * Called when a completed caller utterance has been transcribed.
125
135
  *
@@ -255,6 +265,7 @@ export class MediaStreamSttSession {
255
265
  this.callbacks.onSpeechStart?.();
256
266
  },
257
267
  onTurnEnd: (reason, durationMs) => {
268
+ this.callbacks.onSpeechEnd?.();
258
269
  void this.handleTurnEnd(reason, durationMs);
259
270
  },
260
271
  });
@@ -377,6 +388,10 @@ export class MediaStreamSttSession {
377
388
  const raw = Buffer.from(event.media.payload, "base64");
378
389
  const hasSpeech = detectSpeechActivity(raw);
379
390
  this.turnDetector.onMediaChunk(hasSpeech);
391
+ this.callbacks.onMediaFrame?.(
392
+ hasSpeech,
393
+ raw.length / (TELEPHONY_SOURCE_SAMPLE_RATE / 1000),
394
+ );
380
395
 
381
396
  if (this.mode === "batch") {
382
397
  this.currentTurnChunks.push(event.media.payload);
@@ -5,7 +5,7 @@
5
5
  * The voice front door owns hold, escalation, and direct-response behavior.
6
6
  */
7
7
 
8
- import type { LiveVoiceProgressConfig } from "../config/schemas/live-voice.js";
8
+ import type { VoiceProgressConfig } from "../config/schemas/voice.js";
9
9
  import {
10
10
  extractToolUse,
11
11
  getConfiguredProvider,
@@ -216,7 +216,7 @@ export function effectiveSpokenTextMaxChars(
216
216
  }
217
217
 
218
218
  export function createVoiceProgressNarrator(options: {
219
- config: LiveVoiceProgressConfig;
219
+ config: VoiceProgressConfig;
220
220
  /** Provider resolver, injectable for tests. */
221
221
  getProvider?: () => Promise<Provider | null>;
222
222
  }): VoiceProgressNarrator {
@@ -8,6 +8,7 @@
8
8
  */
9
9
 
10
10
  import { loadConfig } from "../config/loader.js";
11
+ import { resolveEffectiveSpeechProviders } from "../config/managed-speech-defaults.js";
11
12
  import {
12
13
  getCatalogProvider,
13
14
  getTtsProvider,
@@ -73,6 +74,13 @@ export interface ResolveCallTtsOptions {
73
74
  /**
74
75
  * Resolve the active TTS provider via the global provider abstraction.
75
76
  *
77
+ * The provider is the effective one after managed-speech defaulting
78
+ * ({@link resolveEffectiveSpeechProviders}): a configured BYOK provider
79
+ * whose credential does not resolve is stood in for by managed speech
80
+ * while the platform connection is usable, the same substitution live
81
+ * voice makes, so a fresh Vellum connection speaks on the phone with zero
82
+ * TTS configuration.
83
+ *
76
84
  * The native-vs-synthesized decision is driven by the catalog's
77
85
  * `callMode` field via {@link resolveCallStrategy}. Providers with
78
86
  * `callMode: "synthesized-play"` have their audio streamed through the
@@ -94,10 +102,11 @@ export async function resolveCallTtsProvider(
94
102
  ): Promise<ResolvedCallTts> {
95
103
  try {
96
104
  const config = loadConfig();
97
- const resolved = resolveTtsConfig(config);
105
+ const effectiveProviderId = await effectiveTtsProviderId(config);
106
+ const resolved = resolveTtsConfig(config, effectiveProviderId);
98
107
 
99
108
  // Use the catalog's callMode to decide the call path.
100
- const strategy = resolveCallStrategy(config);
109
+ const strategy = resolveCallStrategy(config, effectiveProviderId);
101
110
 
102
111
  let providerId = resolved.provider;
103
112
  let useSynthesizedPath = strategy.callMode === "synthesized-play";
@@ -175,6 +184,28 @@ export async function resolveCallTtsProvider(
175
184
  }
176
185
  }
177
186
 
187
+ /**
188
+ * The TTS provider the call actually uses after managed-speech defaulting.
189
+ * A failure inside the managed lookup (platform client, credential store)
190
+ * keeps the configured provider: a managed-speech hiccup must not demote a
191
+ * working BYOK provider, and the outer resolver's own fallback is the
192
+ * silent native path.
193
+ */
194
+ async function effectiveTtsProviderId(
195
+ config: ReturnType<typeof loadConfig>,
196
+ ): Promise<TtsProviderId | undefined> {
197
+ try {
198
+ const { tts } = await resolveEffectiveSpeechProviders(config);
199
+ return tts;
200
+ } catch (err) {
201
+ log.warn(
202
+ { err },
203
+ "Managed-speech defaulting failed for call TTS; keeping the configured provider",
204
+ );
205
+ return undefined;
206
+ }
207
+ }
208
+
178
209
  /**
179
210
  * Decode a resolved call audio format into the provider request format
180
211
  * and the audio-store format.
@@ -40,17 +40,21 @@ export interface TtsCallStrategy {
40
40
  }
41
41
 
42
42
  /**
43
- * Resolve the call strategy for the currently configured TTS provider.
43
+ * Resolve the call strategy for the active TTS provider.
44
44
  *
45
- * Reads the active provider from config via {@link resolveTtsConfig},
46
- * then looks up the provider's `callMode` in the catalog.
45
+ * Reads the provider from config via {@link resolveTtsConfig} (or takes
46
+ * `providerOverride`, the provider a caller already resolved after
47
+ * managed-speech defaulting), then looks up its `callMode` in the catalog.
47
48
  *
48
49
  * Falls back to `native-twilio` with `"elevenlabs"` when the config
49
50
  * or catalog is unavailable (e.g. test mocks, pre-migration configs).
50
51
  */
51
- export function resolveCallStrategy(config: AssistantConfig): TtsCallStrategy {
52
+ export function resolveCallStrategy(
53
+ config: AssistantConfig,
54
+ providerOverride?: TtsProviderId,
55
+ ): TtsCallStrategy {
52
56
  try {
53
- const resolved = resolveTtsConfig(config);
57
+ const resolved = resolveTtsConfig(config, providerOverride);
54
58
  const catalogEntry = getCatalogProvider(resolved.provider);
55
59
  return {
56
60
  providerId: catalogEntry.id,
@@ -11,6 +11,8 @@ export type CallEventType =
11
11
  | "call_connected"
12
12
  | "caller_spoke"
13
13
  | "assistant_spoke"
14
+ | "tool_use_started"
15
+ | "tool_use_completed"
14
16
  | "user_question_asked"
15
17
  | "user_answered"
16
18
  | "user_instruction_relayed"
@@ -29,11 +29,11 @@ export const HOLD_VERDICT_TOKEN = "[0]";
29
29
  export const ESCALATE_VERDICT_TOKEN = "[1]";
30
30
 
31
31
  /**
32
- * Emitted inline by the main/escalated voice leg to ask the live-voice
33
- * client to minimize the voice room and reveal the screen behind it
34
- * (e.g. after creating an app worth showing). Advisory: the daemon
35
- * translates it into a `minimize_room` server frame after the turn's TTS
36
- * drains; it is never spoken and never persisted.
32
+ * Room-minimize token, kept in the marker table so it is stripped from
33
+ * speech and from persisted rows. No prompt teaches it: the live-voice
34
+ * session decides a minimize from whether a ui-surface tool ran during the
35
+ * turn (`revealsUiSurface` on `tool_result`) and sends the `minimize_room`
36
+ * frame itself, so a model that emits this token regardless moves nothing.
37
37
  */
38
38
  export const MINIMIZE_ROOM_MARKER = "[-1]";
39
39
 
@@ -192,9 +192,9 @@ export function stripInternalSpeechMarkers(text: string): string {
192
192
  // ---------------------------------------------------------------------------
193
193
 
194
194
  /**
195
- * All known control marker prefixes. Used by couldBeControlMarker to detect
196
- * whether a buffer that starts with `[` might be the beginning of a control
197
- * marker (and should therefore be held rather than flushed to TTS).
195
+ * All known control marker prefixes. Used by isIncompleteControlMarkerTail to
196
+ * detect whether a buffer that starts with `[` might be the beginning of a
197
+ * control marker (and should therefore be held rather than flushed to TTS).
198
198
  */
199
199
  const CONTROL_MARKER_STRINGS = [
200
200
  "[ASK_GUARDIAN_APPROVAL:",
@@ -211,19 +211,6 @@ const CONTROL_MARKER_STRINGS = [
211
211
  "[GUARDIAN_UNAVAILABLE]",
212
212
  ];
213
213
 
214
- /**
215
- * Check whether `text` could be a partial or complete control marker.
216
- *
217
- * Returns true if any known marker string is a prefix of `text`
218
- * (text starts with the marker) or `text` is a prefix of a marker
219
- * (the marker starts with text — i.e. text is still being streamed).
220
- */
221
- export function couldBeControlMarker(text: string): boolean {
222
- return CONTROL_MARKER_STRINGS.some(
223
- (marker) => marker.startsWith(text) || text.startsWith(marker),
224
- );
225
- }
226
-
227
214
  // Colon-style markers whose bodies terminate at the first "]" — their strip
228
215
  // regexes are non-greedy (`.+?\]`), so the first bracket IS the terminator.
229
216
  // ASK_GUARDIAN_APPROVAL is deliberately absent: its balanced-JSON body may
@@ -267,3 +254,48 @@ export function isIncompleteControlMarkerTail(tail: string): boolean {
267
254
  }
268
255
  return false;
269
256
  }
257
+
258
+ /**
259
+ * Control-marker hygiene for one model leg's streamed text, shared by the
260
+ * phone call controller and the live-voice session. The returned flush is
261
+ * called with the leg's cumulative raw text so far and forwards, through
262
+ * `emit`, the stripped ({@link stripInternalSpeechMarkers}) prefix that has
263
+ * not been emitted yet and cannot contain a still-streaming control marker:
264
+ * the flush stops at the first "[" whose tail is an incomplete marker
265
+ * ({@link isIncompleteControlMarkerTail}) and holds from there until a later
266
+ * delta completes or disproves it. `force` (leg completion) emits the held
267
+ * tail so real text that merely resembles a marker prefix is not dropped.
268
+ *
269
+ * The scan runs forward from the emitted boundary, not from the last "[", so
270
+ * brackets INSIDE a streaming marker body (a JSON array or "]"-bearing string
271
+ * in ASK_GUARDIAN_APPROVAL) can neither mask the marker's start nor pass as
272
+ * its terminator. Markers are stripped, never acted on: a consumer that acts
273
+ * on a marker reads the leg's full raw text for it separately.
274
+ */
275
+ export function createControlMarkerHoldback(
276
+ emit: (chunk: string) => void,
277
+ ): (raw: string, opts?: { force?: boolean }) => void {
278
+ let emitted = 0;
279
+ return (raw, opts) => {
280
+ let safeEnd = raw.length;
281
+ if (opts?.force !== true) {
282
+ for (
283
+ let i = raw.indexOf("[", emitted);
284
+ i !== -1;
285
+ i = raw.indexOf("[", i + 1)
286
+ ) {
287
+ if (isIncompleteControlMarkerTail(raw.slice(i))) {
288
+ safeEnd = i;
289
+ break;
290
+ }
291
+ }
292
+ }
293
+ if (safeEnd > emitted) {
294
+ const chunk = stripInternalSpeechMarkers(raw.slice(emitted, safeEnd));
295
+ emitted = safeEnd;
296
+ if (chunk.length > 0) {
297
+ emit(chunk);
298
+ }
299
+ }
300
+ };
301
+ }
@@ -0,0 +1,189 @@
1
+ /**
2
+ * The front-door leg coordinator, shared by the live-voice session and the
3
+ * phone call controller.
4
+ *
5
+ * A triaged voice turn opens on a fast, toolless front-door leg whose
6
+ * leading tokens carry its verdict (see `voice-triage-escalate.ts`). The
7
+ * coordinator reads that leg's raw stream through the verdict machine and
8
+ * sequences what follows: an answer is released as speech, a hold discards
9
+ * a speculative turn, and an escalate verdict hands the turn off to the
10
+ * escalated leg with a spoken bridge. The hand-off order is the part that
11
+ * must not drift between drivers: pause narration, abort the front-door
12
+ * leg, resolve and speak the bridge, mark it as the floor holder, start the
13
+ * escalated leg, re-arm narration. A leg that was cancelled never hands off.
14
+ *
15
+ * Everything transport-specific comes in through {@link FrontDoorLegHost}:
16
+ * how answer text and the bridge are spoken, how a leg is started or
17
+ * aborted, and the speculative hold and commit that only live voice has.
18
+ */
19
+
20
+ import type { ProgressCadence } from "./voice-progress-cadence.js";
21
+ import {
22
+ createFrontDoorVerdictMachine,
23
+ ESCALATION_CONTINUATION_CONTENT,
24
+ resolveSpokenEscalationBridge,
25
+ } from "./voice-triage-escalate.js";
26
+
27
+ /** The phrase spoken across the hand-off, as {@link resolveSpokenEscalationBridge} settles it. */
28
+ export type SpokenEscalationBridge = ReturnType<
29
+ typeof resolveSpokenEscalationBridge
30
+ >;
31
+
32
+ /** The leg that answers for real after the front-door leg handed off. */
33
+ export interface EscalatedLeg {
34
+ content: string;
35
+ routingLeg: "escalated";
36
+ /**
37
+ * The bridge the listener heard, quoted to the escalated continuation
38
+ * rule so the quality model does not re-announce it.
39
+ */
40
+ spokenEscalationBridge: string;
41
+ }
42
+
43
+ /**
44
+ * The escalated leg for a hand-off whose bridge was `spokenBridge`. The
45
+ * bridge pins this leg to the conversation's own profile, so no profile
46
+ * override rides along.
47
+ */
48
+ export function escalatedLegFor(spokenBridge: string): EscalatedLeg {
49
+ return {
50
+ content: ESCALATION_CONTINUATION_CONTENT,
51
+ routingLeg: "escalated",
52
+ spokenEscalationBridge: spokenBridge,
53
+ };
54
+ }
55
+
56
+ /** The driver's half of a front-door leg. */
57
+ export interface FrontDoorLegHost {
58
+ /**
59
+ * Whether the turn is still live: not cancelled, not finalized. A dead
60
+ * turn must not spawn an escalated leg.
61
+ */
62
+ isLive(): boolean;
63
+ /** The turn's spoken language, for the canned bridge. */
64
+ language(): string | undefined;
65
+ /**
66
+ * Progress narration to pause across the hand-off: the bridge holds the
67
+ * floor, and audible-silence gating keeps narration quiet until it has
68
+ * played on the escalated leg.
69
+ */
70
+ progress?: Pick<ProgressCadence, "clear" | "noteFloorHolder" | "arm">;
71
+ /**
72
+ * Speculative legs only: the model judged the speaker mid-thought, so the
73
+ * leg is discarded and listening continues.
74
+ */
75
+ onHold?(): void;
76
+ /**
77
+ * Speculative legs only: commit the turn before acting on an answer or
78
+ * escalate verdict. Returns false when the turn is gone, which stops the
79
+ * verdict from being acted on.
80
+ */
81
+ commit?(): boolean;
82
+ /**
83
+ * Answer text released by the verdict machine, in stream order. On the
84
+ * transition it includes the leading text held while the verdict was
85
+ * pending. The driver runs its own control-marker holdback over it.
86
+ */
87
+ onAnswerText(text: string): void;
88
+ /**
89
+ * Abort the front-door leg so a model that keeps generating past the
90
+ * bridge cap adds no latency before the escalated leg starts.
91
+ */
92
+ abortLeg(): void;
93
+ /**
94
+ * Speak the bridge so the strong-model call has no dead air. The model's
95
+ * own bridge is real assistant speech; the canned fallback is audio-only,
96
+ * matching the row the bridge's transcript hygiene deletes for it.
97
+ */
98
+ speakBridge(bridge: SpokenEscalationBridge): void;
99
+ /** Start the escalated leg on the same turn. */
100
+ startEscalatedLeg(leg: EscalatedLeg): void;
101
+ }
102
+
103
+ export interface FrontDoorLegCoordinator {
104
+ /** Whether this leg handed the turn off to the escalated leg. */
105
+ readonly handedOff: boolean;
106
+ /** Feed one raw delta of the front-door leg's stream. */
107
+ push(text: string): void;
108
+ /**
109
+ * The leg completed normally (not cancelled). A leg that stopped
110
+ * mid-bridge (a bare escalate verdict, or a holding phrase with no
111
+ * sentence terminator) hands off now with what arrived; the canned
112
+ * fallback covers an empty bridge. Returns whether the leg handed off.
113
+ */
114
+ complete(): boolean;
115
+ }
116
+
117
+ export function createFrontDoorLegCoordinator(options: {
118
+ /**
119
+ * Whether the leg's decision rule teaches the hold token: true only for
120
+ * speculative legs, whose leading tokens are interpreted.
121
+ */
122
+ holdEnabled: boolean;
123
+ host: FrontDoorLegHost;
124
+ }): FrontDoorLegCoordinator {
125
+ const { host } = options;
126
+ const verdict = createFrontDoorVerdictMachine(options.holdEnabled);
127
+ let handedOff = false;
128
+
129
+ const handOff = (cappedBridge: string): void => {
130
+ if (handedOff || !host.isLive()) {
131
+ return;
132
+ }
133
+ handedOff = true;
134
+ host.progress?.clear();
135
+ host.abortLeg();
136
+ const bridge = resolveSpokenEscalationBridge(cappedBridge, host.language());
137
+ host.speakBridge(bridge);
138
+ // The bridge is the turn's spoken acknowledgement: narration keeps its
139
+ // minimum gap from it rather than following it back to back.
140
+ host.progress?.noteFloorHolder();
141
+ host.startEscalatedLeg(escalatedLegFor(bridge.spokenBridge));
142
+ // The escalated leg runs the slowest work in the system, and the bridge
143
+ // only covers its first couple of seconds: the dead air narration
144
+ // exists for.
145
+ host.progress?.arm();
146
+ };
147
+
148
+ return {
149
+ get handedOff() {
150
+ return handedOff;
151
+ },
152
+ push(text) {
153
+ if (handedOff) {
154
+ return;
155
+ }
156
+ const step = verdict.push(text);
157
+ if (step.kind === "hold") {
158
+ host.onHold?.();
159
+ return;
160
+ }
161
+ if (
162
+ (step.kind === "escalate" || step.kind === "answer") &&
163
+ host.commit !== undefined &&
164
+ !host.commit()
165
+ ) {
166
+ return;
167
+ }
168
+ if (step.kind === "answer") {
169
+ host.onAnswerText(step.text);
170
+ return;
171
+ }
172
+ if (
173
+ (step.kind === "escalate" || step.kind === "bridge") &&
174
+ step.bridge !== null
175
+ ) {
176
+ handOff(step.bridge);
177
+ }
178
+ },
179
+ complete() {
180
+ if (!handedOff && host.isLive()) {
181
+ const step = verdict.finish();
182
+ if (step.kind === "bridge") {
183
+ handOff(step.bridge);
184
+ }
185
+ }
186
+ return handedOff;
187
+ },
188
+ };
189
+ }