@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (826) hide show
  1. package/ARCHITECTURE.md +69 -9
  2. package/docs/flux-turn-detection-spike.md +243 -0
  3. package/docs/stt-provider-onboarding.md +3 -1
  4. package/docs/trusted-contact-access.md +9 -5
  5. package/docs/vellum-doctor.md +195 -0
  6. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  7. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  8. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  9. package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
  10. package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
  11. package/node_modules/@vellumai/environments/package.json +2 -1
  12. package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
  13. package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
  14. package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
  15. package/node_modules/@vellumai/environments/src/index.ts +17 -6
  16. package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
  17. package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
  18. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  19. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  20. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  21. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
  22. package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
  23. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
  24. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
  25. package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
  26. package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
  27. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
  28. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
  29. package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  30. package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  31. package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  32. package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
  33. package/openapi.yaml +619 -47
  34. package/package.json +1 -1
  35. package/scripts/voice-ttft-spike.ts +3 -3
  36. package/src/__tests__/agent-image-optimize.test.ts +4 -4
  37. package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
  38. package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
  39. package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
  40. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
  41. package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
  42. package/src/__tests__/answered-question-persistence.test.ts +240 -0
  43. package/src/__tests__/anthropic-provider.test.ts +165 -30
  44. package/src/__tests__/app-control-flow.test.ts +9 -8
  45. package/src/__tests__/assistant-event-hub.test.ts +48 -0
  46. package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
  47. package/src/__tests__/attachments-store.test.ts +130 -68
  48. package/src/__tests__/attachments.test.ts +36 -36
  49. package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
  50. package/src/__tests__/call-controller.test.ts +120 -0
  51. package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
  52. package/src/__tests__/canned-reply-release.test.ts +86 -0
  53. package/src/__tests__/channel-availability-routes.test.ts +14 -1
  54. package/src/__tests__/channel-guardian.test.ts +59 -29
  55. package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
  56. package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
  57. package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
  58. package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
  59. package/src/__tests__/config-loader-backfill.test.ts +19 -0
  60. package/src/__tests__/config-schema.test.ts +150 -7
  61. package/src/__tests__/consult-deadline.test.ts +65 -0
  62. package/src/__tests__/conversation-agent-loop.test.ts +438 -2
  63. package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
  64. package/src/__tests__/conversation-disk-view.test.ts +4 -4
  65. package/src/__tests__/conversation-error.test.ts +22 -3
  66. package/src/__tests__/conversation-fork-crud.test.ts +5 -1
  67. package/src/__tests__/conversation-fork-referential.test.ts +352 -0
  68. package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
  69. package/src/__tests__/conversation-group-tools.test.ts +32 -2
  70. package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
  71. package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
  72. package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
  73. package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
  74. package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
  75. package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
  76. package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
  77. package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
  78. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
  79. package/src/__tests__/conversation-queue.test.ts +433 -1
  80. package/src/__tests__/conversation-retry-route.test.ts +3 -1
  81. package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
  82. package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
  83. package/src/__tests__/conversation-store.test.ts +26 -10
  84. package/src/__tests__/conversation-summarize-route.test.ts +1 -1
  85. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
  86. package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
  87. package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
  88. package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
  89. package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
  90. package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
  91. package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
  92. package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
  93. package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
  94. package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
  95. package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
  96. package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
  97. package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
  98. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
  99. package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
  100. package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
  101. package/src/__tests__/credential-prompt-route.test.ts +54 -7
  102. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  103. package/src/__tests__/cu-unified-flow.test.ts +89 -28
  104. package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
  105. package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
  106. package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
  107. package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
  108. package/src/__tests__/disk-pressure-guard.test.ts +55 -55
  109. package/src/__tests__/disk-pressure-routes.test.ts +10 -10
  110. package/src/__tests__/disk-pressure-tools.test.ts +2 -2
  111. package/src/__tests__/disk-usage.test.ts +35 -20
  112. package/src/__tests__/document-sync-tags.test.ts +379 -0
  113. package/src/__tests__/document-update-default-surface.test.ts +28 -0
  114. package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
  115. package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
  116. package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
  117. package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
  118. package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
  119. package/src/__tests__/file-list-tool.test.ts +12 -10
  120. package/src/__tests__/file-ops-service.test.ts +151 -44
  121. package/src/__tests__/file-read-container-boundary.test.ts +137 -0
  122. package/src/__tests__/file-read-tool.test.ts +21 -9
  123. package/src/__tests__/file-url-path-guard.test.ts +83 -0
  124. package/src/__tests__/filesystem-tools.test.ts +82 -66
  125. package/src/__tests__/guardian-outbound-http.test.ts +8 -8
  126. package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
  127. package/src/__tests__/helpers/hub-clients.ts +42 -0
  128. package/src/__tests__/helpers/mock-conversation.ts +46 -0
  129. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
  130. package/src/__tests__/history-repair.test.ts +112 -12
  131. package/src/__tests__/host-bash-proxy.test.ts +10 -5
  132. package/src/__tests__/host-bash-routes.test.ts +67 -0
  133. package/src/__tests__/host-browser-proxy.test.ts +2 -2
  134. package/src/__tests__/host-cu-proxy.test.ts +3 -7
  135. package/src/__tests__/host-file-proxy.test.ts +3 -7
  136. package/src/__tests__/host-file-read-tool.test.ts +5 -1
  137. package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
  138. package/src/__tests__/http-conversation-lineage.test.ts +2 -5
  139. package/src/__tests__/http-user-message-parity.test.ts +44 -0
  140. package/src/__tests__/identity-routes.test.ts +13 -13
  141. package/src/__tests__/image-conversion.test.ts +29 -26
  142. package/src/__tests__/image-source-path-reinject.test.ts +2 -2
  143. package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
  144. package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
  145. package/src/__tests__/introduction-card-resolver.test.ts +6 -1
  146. package/src/__tests__/list-messages-attachments.test.ts +56 -32
  147. package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
  148. package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
  149. package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
  150. package/src/__tests__/list-messages-page-latest.test.ts +61 -61
  151. package/src/__tests__/list-messages-provider-error.test.ts +6 -6
  152. package/src/__tests__/list-messages-queued.test.ts +49 -6
  153. package/src/__tests__/list-messages-system-card.test.ts +4 -4
  154. package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
  155. package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
  156. package/src/__tests__/llm-usage-store.test.ts +55 -0
  157. package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
  158. package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
  159. package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
  160. package/src/__tests__/media-stream-output.test.ts +175 -0
  161. package/src/__tests__/media-stream-server-integration.test.ts +36 -0
  162. package/src/__tests__/media-stream-stt-session.test.ts +67 -0
  163. package/src/__tests__/migration-export-http.test.ts +1 -1
  164. package/src/__tests__/migration-import-commit-http.test.ts +1 -1
  165. package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
  166. package/src/__tests__/migration-validate-http.test.ts +1 -1
  167. package/src/__tests__/mtime-cache.test.ts +65 -3
  168. package/src/__tests__/notification-platform-adapter.test.ts +104 -0
  169. package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
  170. package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
  171. package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
  172. package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
  173. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  174. package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
  175. package/src/__tests__/path-policy.test.ts +164 -1
  176. package/src/__tests__/persist-media-references.test.ts +2 -2
  177. package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
  178. package/src/__tests__/persist-unsendable-image.test.ts +23 -23
  179. package/src/__tests__/persona-resolver.test.ts +154 -0
  180. package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
  181. package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
  182. package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
  183. package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
  184. package/src/__tests__/profiler-routes.test.ts +13 -13
  185. package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
  186. package/src/__tests__/proxy-approval-callback.test.ts +3 -3
  187. package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
  188. package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
  189. package/src/__tests__/resolve-trust-class.test.ts +20 -0
  190. package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
  191. package/src/__tests__/run-due-schedules.test.ts +174 -0
  192. package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
  193. package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
  194. package/src/__tests__/schedule-routes.test.ts +730 -8
  195. package/src/__tests__/schedule-store.test.ts +540 -0
  196. package/src/__tests__/schedule-tools.test.ts +103 -0
  197. package/src/__tests__/scheduler-wake.test.ts +5 -0
  198. package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
  199. package/src/__tests__/secure-keys.test.ts +91 -0
  200. package/src/__tests__/server-history-render.test.ts +97 -7
  201. package/src/__tests__/size-guard.test.ts +8 -8
  202. package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
  203. package/src/__tests__/skill-tool-factory.test.ts +247 -0
  204. package/src/__tests__/skills-install-staging.test.ts +15 -6
  205. package/src/__tests__/skills.test.ts +32 -0
  206. package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
  207. package/src/__tests__/slack-inbound-verification.test.ts +4 -1
  208. package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
  209. package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
  210. package/src/__tests__/starter-task-flow.test.ts +5 -4
  211. package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
  212. package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
  213. package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
  214. package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
  215. package/src/__tests__/subagent-disposal.test.ts +10 -0
  216. package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
  217. package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
  218. package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
  219. package/src/__tests__/subagent-manager-notify.test.ts +148 -0
  220. package/src/__tests__/subagent-notify-parent.test.ts +1 -1
  221. package/src/__tests__/subagent-persistence.test.ts +141 -0
  222. package/src/__tests__/subagent-role-registry.test.ts +172 -63
  223. package/src/__tests__/subagent-role-resolution.test.ts +88 -0
  224. package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
  225. package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
  226. package/src/__tests__/subagent-terminal-message.test.ts +88 -1
  227. package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
  228. package/src/__tests__/subagent-tools.test.ts +1901 -108
  229. package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
  230. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
  231. package/src/__tests__/system-prompt.test.ts +82 -7
  232. package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
  233. package/src/__tests__/tool-result-spool.test.ts +74 -18
  234. package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
  235. package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
  236. package/src/__tests__/tools-audio-read.test.ts +10 -10
  237. package/src/__tests__/tools-get-route.test.ts +85 -12
  238. package/src/__tests__/tools-list-cli.test.ts +171 -0
  239. package/src/__tests__/top-level-renderer.test.ts +54 -1
  240. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
  241. package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
  242. package/src/__tests__/turn-tail-chain.test.ts +135 -0
  243. package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
  244. package/src/__tests__/ui-visual-surface.test.ts +5 -4
  245. package/src/__tests__/ui-work-result-surface.test.ts +5 -4
  246. package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
  247. package/src/__tests__/usage-routes.test.ts +62 -0
  248. package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
  249. package/src/__tests__/visible-app-context.test.ts +20 -4
  250. package/src/__tests__/web-search-history.test.ts +96 -0
  251. package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
  252. package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
  253. package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
  254. package/src/acp/failure-error.ts +3 -5
  255. package/src/agent/attachments.ts +26 -24
  256. package/src/agent/history-repair/history-repair.ts +27 -26
  257. package/src/agent/image-optimize.ts +6 -6
  258. package/src/agent/loop.ts +40 -22
  259. package/src/agent/message-types.ts +3 -3
  260. package/src/api/constants/document-tools.ts +55 -0
  261. package/src/api/events/question-answered.ts +53 -0
  262. package/src/api/events/tool-result.ts +7 -0
  263. package/src/api/index.ts +13 -0
  264. package/src/api/responses/conversation-message.ts +11 -0
  265. package/src/api/surfaces-oauth-connect.test.ts +96 -0
  266. package/src/api/surfaces.ts +36 -8
  267. package/src/approvals/guardian-channel-delivery.ts +83 -0
  268. package/src/approvals/guardian-expiry-notifier.ts +13 -11
  269. package/src/approvals/guardian-request-resolvers.ts +222 -135
  270. package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
  271. package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
  272. package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
  273. package/src/calls/call-controller.ts +30 -2
  274. package/src/calls/call-speech-output.ts +12 -4
  275. package/src/calls/call-transport.ts +18 -1
  276. package/src/calls/media-stream-output.ts +53 -5
  277. package/src/calls/media-stream-server.ts +21 -1
  278. package/src/calls/media-stream-stt-session.ts +49 -0
  279. package/src/calls/telephony-synthesis-language.ts +84 -0
  280. package/src/calls/tts-text-sanitizer.ts +12 -5
  281. package/src/calls/voice-session-bridge.ts +86 -17
  282. package/src/calls/voice-triage-escalate.ts +54 -7
  283. package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
  284. package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
  285. package/src/channels/config.ts +20 -2
  286. package/src/channels/gateway-verification-sessions.ts +7 -2
  287. package/src/channels/plugin-channel-declarations.ts +108 -0
  288. package/src/channels/types.ts +37 -1
  289. package/src/cli/AGENTS.md +5 -2
  290. package/src/cli/commands/__tests__/cache.test.ts +7 -4
  291. package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
  292. package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
  293. package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
  294. package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
  295. package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
  296. package/src/cli/commands/__tests__/plugins.test.ts +692 -0
  297. package/src/cli/commands/__tests__/schedules.test.ts +136 -37
  298. package/src/cli/commands/conversations.ts +7 -3
  299. package/src/cli/commands/credentials.help.ts +7 -0
  300. package/src/cli/commands/inference-profiles.ts +9 -3
  301. package/src/cli/commands/inference.help.ts +16 -2
  302. package/src/cli/commands/mcp.help.ts +17 -6
  303. package/src/cli/commands/mcp.ts +9 -0
  304. package/src/cli/commands/memory/memory-retrospective.ts +18 -0
  305. package/src/cli/commands/notifications.ts +3 -6
  306. package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
  307. package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
  308. package/src/cli/commands/oauth/connect.test.ts +26 -1
  309. package/src/cli/commands/oauth/connect.ts +4 -1
  310. package/src/cli/commands/oauth/index.help.ts +3 -1
  311. package/src/cli/commands/oauth/providers.ts +3 -0
  312. package/src/cli/commands/plugins.help.ts +25 -2
  313. package/src/cli/commands/plugins.ts +289 -10
  314. package/src/cli/commands/schedules.help.ts +14 -12
  315. package/src/cli/commands/schedules.ts +19 -5
  316. package/src/cli/commands/stt.help.ts +27 -2
  317. package/src/cli/commands/tools.help.ts +4 -2
  318. package/src/cli/commands/tools.ts +38 -0
  319. package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
  320. package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
  321. package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
  322. package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
  323. package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
  324. package/src/cli/lib/install-from-github.ts +61 -0
  325. package/src/cli/lib/install-from-platform.ts +13 -2
  326. package/src/cli/lib/plugin-surfaces.ts +124 -4
  327. package/src/cli/lib/upgrade-plugin.ts +60 -0
  328. package/src/cli.ts +1 -1
  329. package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
  330. package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
  331. package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
  332. package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
  333. package/src/config/__tests__/profile-tool-support.test.ts +131 -0
  334. package/src/config/bundled-skills/AGENTS.md +1 -1
  335. package/src/config/bundled-skills/acp/SKILL.md +6 -7
  336. package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
  337. package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
  338. package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
  339. package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
  340. package/src/config/bundled-skills/schedule/SKILL.md +5 -1
  341. package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
  342. package/src/config/bundled-skills/settings/TOOLS.json +3 -3
  343. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
  344. package/src/config/bundled-skills/subagent/SKILL.md +53 -22
  345. package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
  346. package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
  347. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
  348. package/src/config/bundled-skills/visualize/SKILL.md +1 -1
  349. package/src/config/call-site-defaults.ts +4 -5
  350. package/src/config/default-profile-catalog.ts +9 -5
  351. package/src/config/feature-flag-registry.json +25 -17
  352. package/src/config/llm-resolver.ts +45 -15
  353. package/src/config/loader.ts +75 -1
  354. package/src/config/profile-tool-support.ts +110 -0
  355. package/src/config/sanitize-for-transfer.ts +11 -0
  356. package/src/config/schema.ts +4 -0
  357. package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
  358. package/src/config/schemas/__tests__/stt.test.ts +20 -5
  359. package/src/config/schemas/call-site-catalog.ts +4 -4
  360. package/src/config/schemas/calls.ts +0 -4
  361. package/src/config/schemas/channels.ts +13 -0
  362. package/src/config/schemas/live-voice.ts +82 -23
  363. package/src/config/schemas/llm.ts +1 -1
  364. package/src/config/schemas/memory-retrospective.ts +25 -0
  365. package/src/config/schemas/plugin-updates.ts +59 -0
  366. package/src/config/schemas/services.ts +8 -4
  367. package/src/config/schemas/stt.ts +18 -5
  368. package/src/config/schemas/timeouts.ts +1 -1
  369. package/src/config/schemas/tts.ts +63 -0
  370. package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
  371. package/src/contacts/guardian-delivery-reader.ts +29 -2
  372. package/src/context/compactor.ts +4 -4
  373. package/src/context/post-turn-tool-result-truncation.ts +32 -16
  374. package/src/context/strip-injections.ts +42 -0
  375. package/src/context/tool-result-spool.ts +32 -18
  376. package/src/credential-execution/ces-runtime.ts +3 -0
  377. package/src/credential-execution/client.ts +22 -1
  378. package/src/credential-execution/process-manager.test.ts +99 -0
  379. package/src/credential-execution/process-manager.ts +62 -16
  380. package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
  381. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
  382. package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
  383. package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
  384. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
  385. package/src/daemon/channel-ui-capability.ts +4 -4
  386. package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
  387. package/src/daemon/conversation-agent-loop.ts +318 -148
  388. package/src/daemon/conversation-attachments.ts +1 -1
  389. package/src/daemon/conversation-error.ts +8 -1
  390. package/src/daemon/conversation-initial-prompt.ts +16 -38
  391. package/src/daemon/conversation-lifecycle.ts +154 -32
  392. package/src/daemon/conversation-messaging.ts +106 -43
  393. package/src/daemon/conversation-process.ts +131 -33
  394. package/src/daemon/conversation-queue-manager.ts +10 -0
  395. package/src/daemon/conversation-runtime-assembly.ts +9 -4
  396. package/src/daemon/conversation-store.ts +52 -3
  397. package/src/daemon/conversation-surface-state.ts +9 -0
  398. package/src/daemon/conversation-surfaces.ts +403 -210
  399. package/src/daemon/conversation-tool-setup.ts +56 -7
  400. package/src/daemon/conversation-turn-finalize.ts +134 -89
  401. package/src/daemon/conversation.ts +111 -27
  402. package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
  403. package/src/daemon/disk-pressure-guard.ts +3 -3
  404. package/src/daemon/doordash-steps.ts +5 -5
  405. package/src/daemon/external-plugins-bootstrap.ts +7 -1
  406. package/src/daemon/handlers/config-channels.ts +48 -5
  407. package/src/daemon/handlers/conversation-history.ts +1 -1
  408. package/src/daemon/handlers/conversations.ts +62 -17
  409. package/src/daemon/handlers/shared.ts +12 -0
  410. package/src/daemon/handlers/skills.ts +11 -1
  411. package/src/daemon/identity-helpers.ts +38 -10
  412. package/src/daemon/install-assistant-command.ts +322 -0
  413. package/src/daemon/lifecycle.ts +14 -3
  414. package/src/daemon/message-types/conversations.ts +6 -7
  415. package/src/daemon/message-types/sync.ts +1 -0
  416. package/src/daemon/persist-media-references.ts +23 -14
  417. package/src/daemon/process-message.ts +6 -6
  418. package/src/daemon/tool-setup-types.ts +19 -122
  419. package/src/daemon/tool-side-effects.ts +17 -94
  420. package/src/daemon/trust-context-types.ts +39 -0
  421. package/src/daemon/trust-context.ts +14 -6
  422. package/src/daemon/turn-tail-chain.ts +73 -0
  423. package/src/daemon/web-search-history.ts +47 -33
  424. package/src/heartbeat/heartbeat-service.ts +6 -6
  425. package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
  426. package/src/inbound/oauth-callback-url.ts +61 -0
  427. package/src/ipc/__tests__/socket-path.test.ts +21 -0
  428. package/src/ipc/assistant-server.ts +17 -10
  429. package/src/ipc/cli-client.ts +10 -1
  430. package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
  431. package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
  432. package/src/ipc/socket-cleanup.ts +9 -7
  433. package/src/ipc/socket-path.ts +4 -96
  434. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
  435. package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
  436. package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
  437. package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
  438. package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
  439. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
  440. package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
  441. package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
  442. package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
  443. package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
  444. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
  445. package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
  446. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
  447. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
  448. package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
  449. package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
  450. package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
  451. package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
  452. package/src/live-voice/__tests__/protocol.test.ts +52 -0
  453. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
  454. package/src/live-voice/live-voice-archive.ts +13 -13
  455. package/src/live-voice/live-voice-connection.ts +7 -1
  456. package/src/live-voice/live-voice-metrics.ts +57 -32
  457. package/src/live-voice/live-voice-photo.ts +168 -0
  458. package/src/live-voice/live-voice-session.ts +1137 -351
  459. package/src/live-voice/live-voice-tts.ts +18 -2
  460. package/src/live-voice/progress-narration.ts +277 -0
  461. package/src/live-voice/progress-phrases.ts +105 -2
  462. package/src/live-voice/protocol.ts +110 -2
  463. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
  464. package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
  465. package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
  466. package/src/mcp/client.ts +14 -7
  467. package/src/mcp/mcp-auth-orchestrator.ts +13 -22
  468. package/src/mcp/mcp-oauth-provider.ts +195 -240
  469. package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
  470. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
  471. package/src/messaging/providers/callback-routing.ts +1 -0
  472. package/src/messaging/providers/discord/api.ts +312 -0
  473. package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
  474. package/src/messaging/providers/discord/render.test.ts +204 -0
  475. package/src/messaging/providers/discord/render.ts +195 -0
  476. package/src/messaging/providers/discord/send.test.ts +179 -0
  477. package/src/messaging/providers/discord/send.ts +212 -0
  478. package/src/messaging/providers/discord/transport.ts +81 -0
  479. package/src/messaging/providers/index.ts +3 -1
  480. package/src/messaging/providers/retry-policy.test.ts +247 -0
  481. package/src/messaging/providers/retry-policy.ts +206 -0
  482. package/src/messaging/providers/slack/mention-source.test.ts +285 -0
  483. package/src/messaging/providers/slack/mention-source.ts +287 -0
  484. package/src/messaging/providers/telegram-bot/api.ts +47 -130
  485. package/src/messaging/providers/whatsapp/api.ts +47 -136
  486. package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
  487. package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
  488. package/src/monitoring/db-integrity-sample.ts +6 -1
  489. package/src/monitoring/plugin-auto-update.ts +463 -0
  490. package/src/monitoring/resource-sampler.ts +21 -5
  491. package/src/monitoring/worker.ts +13 -0
  492. package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
  493. package/src/notifications/__tests__/broadcaster.test.ts +37 -1
  494. package/src/notifications/__tests__/copy-composer.test.ts +165 -0
  495. package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
  496. package/src/notifications/__tests__/notification-utils.test.ts +249 -0
  497. package/src/notifications/adapters/platform.ts +31 -2
  498. package/src/notifications/assistant-reply-producer.ts +79 -32
  499. package/src/notifications/broadcaster.ts +30 -10
  500. package/src/notifications/copy-composer.ts +69 -0
  501. package/src/notifications/emit-signal.ts +82 -15
  502. package/src/notifications/events-store.ts +10 -2
  503. package/src/notifications/home-feed-side-effect.ts +6 -3
  504. package/src/notifications/notification-utils.ts +173 -5
  505. package/src/notifications/signal.ts +13 -0
  506. package/src/oauth/AGENTS.md +4 -2
  507. package/src/oauth/byo-connection.ts +6 -1
  508. package/src/oauth/credential-token-resolver.ts +3 -10
  509. package/src/oauth/manual-token-connection.ts +29 -64
  510. package/src/oauth/manual-token-providers.ts +62 -0
  511. package/src/oauth/provider-serializer.ts +13 -0
  512. package/src/oauth/seed-providers.ts +62 -11
  513. package/src/onboarding/onboarding-events-store.ts +91 -1
  514. package/src/permissions/question-prompter.test.ts +4 -0
  515. package/src/permissions/question-prompter.ts +21 -4
  516. package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
  517. package/src/persistence/attachments-store.ts +15 -15
  518. package/src/persistence/conversation-attention-store.ts +30 -10
  519. package/src/persistence/conversation-bootstrap.ts +17 -0
  520. package/src/persistence/conversation-crud.ts +363 -97
  521. package/src/persistence/conversation-group-migration.ts +1 -1
  522. package/src/persistence/conversation-key-store.ts +38 -10
  523. package/src/persistence/conversation-lineage.ts +226 -0
  524. package/src/persistence/conversation-plugin-facade.ts +28 -3
  525. package/src/persistence/conversation-queries.ts +207 -33
  526. package/src/persistence/conversation-types.ts +107 -1
  527. package/src/persistence/jobs-store.ts +32 -8
  528. package/src/persistence/llm-usage-store.ts +22 -0
  529. package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
  530. package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
  531. package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
  532. package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
  533. package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
  534. package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
  535. package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
  536. package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
  537. package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
  538. package/src/persistence/schema/conversations.ts +32 -0
  539. package/src/persistence/schema/infrastructure.ts +3 -0
  540. package/src/persistence/steps.ts +30 -0
  541. package/src/persistence/subagent-store.ts +148 -6
  542. package/src/platform/managed-speech.test.ts +20 -0
  543. package/src/platform/managed-speech.ts +9 -0
  544. package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
  545. package/src/plugin-api/index.ts +28 -5
  546. package/src/plugin-api/webhook-url.ts +151 -0
  547. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
  548. package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
  549. package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
  550. package/src/plugins/collect-source-versions.ts +4 -27
  551. package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
  552. package/src/plugins/defaults/exploration-drift/package.json +1 -1
  553. package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
  554. package/src/plugins/defaults/image-recovery/recover.ts +67 -57
  555. package/src/plugins/defaults/index.ts +1 -1
  556. package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
  557. package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
  558. package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
  559. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
  560. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
  561. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
  562. package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
  563. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
  564. package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
  565. package/src/plugins/defaults/memory/job-handlers.ts +87 -5
  566. package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
  567. package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
  568. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
  569. package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
  570. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
  571. package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
  572. package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
  573. package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
  574. package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
  575. package/src/plugins/defaults/memory/v2/router.ts +8 -0
  576. package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
  577. package/src/plugins/external-plugin-loader.ts +64 -8
  578. package/src/plugins/installed-plugin-dirs.ts +101 -0
  579. package/src/plugins/mcp-servers.ts +343 -0
  580. package/src/plugins/mtime-cache.ts +34 -22
  581. package/src/plugins/types.ts +69 -8
  582. package/src/prompts/persona-resolver.ts +6 -2
  583. package/src/prompts/system-prompt.ts +72 -17
  584. package/src/providers/__tests__/inference.test.ts +40 -0
  585. package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
  586. package/src/providers/__tests__/retry-callsite.test.ts +72 -2
  587. package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
  588. package/src/providers/anthropic/client.ts +112 -95
  589. package/src/providers/connection-resolution.ts +35 -1
  590. package/src/providers/gemini/client.ts +5 -5
  591. package/src/providers/inference/adapter-factory.ts +16 -10
  592. package/src/providers/inference/auth.ts +16 -0
  593. package/src/providers/inference/connection-availability.ts +164 -12
  594. package/src/providers/inference/connections.ts +25 -1
  595. package/src/providers/media-resolve.ts +21 -11
  596. package/src/providers/model-catalog.ts +15 -0
  597. package/src/providers/openai/chat-completions-provider.ts +4 -4
  598. package/src/providers/openai/responses-provider.ts +12 -25
  599. package/src/providers/registry.ts +9 -4
  600. package/src/providers/retry.ts +25 -0
  601. package/src/providers/server-tool-pairing.ts +134 -0
  602. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
  603. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
  604. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
  605. package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
  606. package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
  607. package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
  608. package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
  609. package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
  610. package/src/providers/speech-to-text/deepgram.ts +1 -0
  611. package/src/providers/speech-to-text/provider-catalog.ts +137 -8
  612. package/src/providers/speech-to-text/resolve.ts +105 -12
  613. package/src/providers/speech-to-text/vellum-managed.ts +7 -1
  614. package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
  615. package/src/providers/types.ts +26 -6
  616. package/src/providers/vellum-model-routing.ts +6 -7
  617. package/src/runtime/AGENTS.md +2 -0
  618. package/src/runtime/__tests__/agent-wake.test.ts +16 -16
  619. package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
  620. package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
  621. package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
  622. package/src/runtime/agent-wake.ts +63 -24
  623. package/src/runtime/assistant-event-hub.ts +110 -12
  624. package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
  625. package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
  626. package/src/runtime/auth/same-actor.ts +56 -8
  627. package/src/runtime/auth/scopes.ts +12 -0
  628. package/src/runtime/auth/types.ts +2 -0
  629. package/src/runtime/background-job-runner.ts +5 -27
  630. package/src/runtime/channel-readiness-service.ts +59 -3
  631. package/src/runtime/channel-readiness-types.ts +18 -0
  632. package/src/runtime/desktop-presence.ts +69 -0
  633. package/src/runtime/http-server.ts +12 -0
  634. package/src/runtime/local-actor-identity.ts +29 -14
  635. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
  636. package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
  637. package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
  638. package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
  639. package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
  640. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
  641. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
  642. package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
  643. package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
  644. package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
  645. package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
  646. package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
  647. package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
  648. package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
  649. package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
  650. package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
  651. package/src/runtime/routes/archive-utils.ts +16 -12
  652. package/src/runtime/routes/attachment-routes.ts +13 -6
  653. package/src/runtime/routes/canned-reply-release.ts +54 -0
  654. package/src/runtime/routes/channel-availability-routes.ts +32 -14
  655. package/src/runtime/routes/channel-verification-routes.ts +2 -27
  656. package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
  657. package/src/runtime/routes/client-routes.ts +114 -2
  658. package/src/runtime/routes/conversation-cli-routes.ts +10 -12
  659. package/src/runtime/routes/conversation-list-routes.ts +59 -10
  660. package/src/runtime/routes/conversation-management-routes.ts +47 -18
  661. package/src/runtime/routes/conversation-query-routes.ts +32 -53
  662. package/src/runtime/routes/conversation-routes.ts +360 -293
  663. package/src/runtime/routes/credential-prompt-routes.ts +20 -2
  664. package/src/runtime/routes/debug-routes.ts +4 -1
  665. package/src/runtime/routes/default-provider-routes.ts +7 -11
  666. package/src/runtime/routes/documents-routes.ts +44 -1
  667. package/src/runtime/routes/events-routes.ts +29 -1
  668. package/src/runtime/routes/host-app-control-routes.ts +2 -0
  669. package/src/runtime/routes/host-bash-routes.ts +2 -0
  670. package/src/runtime/routes/host-browser-routes.ts +2 -0
  671. package/src/runtime/routes/host-cu-routes.ts +2 -0
  672. package/src/runtime/routes/host-file-routes.ts +2 -0
  673. package/src/runtime/routes/host-transfer-routes.ts +4 -0
  674. package/src/runtime/routes/identity-routes.ts +6 -6
  675. package/src/runtime/routes/inbound-message-handler.ts +5 -5
  676. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
  677. package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
  678. package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
  679. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
  680. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
  681. package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
  682. package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
  683. package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
  684. package/src/runtime/routes/inference-profiles-routes.ts +152 -73
  685. package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
  686. package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
  687. package/src/runtime/routes/log-export-routes.ts +1 -1
  688. package/src/runtime/routes/mcp-auth-routes.ts +141 -57
  689. package/src/runtime/routes/notification-routes.ts +0 -14
  690. package/src/runtime/routes/oauth-providers.ts +1 -0
  691. package/src/runtime/routes/plugins-routes.ts +127 -4
  692. package/src/runtime/routes/profiler-routes.ts +4 -2
  693. package/src/runtime/routes/question-routes.ts +28 -10
  694. package/src/runtime/routes/retrospective-routes.ts +9 -3
  695. package/src/runtime/routes/schedule-routes.ts +210 -12
  696. package/src/runtime/routes/secret-routes.ts +40 -23
  697. package/src/runtime/routes/settings-routes.ts +41 -13
  698. package/src/runtime/routes/skills-routes.ts +79 -0
  699. package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
  700. package/src/runtime/routes/stt-routes.ts +31 -25
  701. package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
  702. package/src/runtime/routes/wake-conversation-routes.ts +23 -0
  703. package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
  704. package/src/runtime/services/conversation-serializer.ts +24 -9
  705. package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
  706. package/src/runtime/sync/resource-sync-events.ts +79 -1
  707. package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
  708. package/src/runtime/sync/worker-daemon-notify.ts +44 -6
  709. package/src/runtime/verification-outbound-actions.ts +250 -222
  710. package/src/runtime/verification-templates.ts +41 -30
  711. package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
  712. package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
  713. package/src/schedule/inference-profile.ts +64 -3
  714. package/src/schedule/plugin-schedule-declarations.ts +518 -0
  715. package/src/schedule/plugin-schedule-reconciler.ts +605 -0
  716. package/src/schedule/plugin-schedules-gate.ts +23 -0
  717. package/src/schedule/schedule-store.ts +520 -26
  718. package/src/schedule/scheduler.ts +38 -0
  719. package/src/security/credential-backend.ts +51 -4
  720. package/src/security/encrypted-store.ts +64 -42
  721. package/src/security/oauth2.ts +6 -22
  722. package/src/security/secure-keys.ts +55 -7
  723. package/src/skills/catalog-install.ts +20 -7
  724. package/src/skills/inline-command-runner.ts +2 -7
  725. package/src/skills/skill-history.test.ts +269 -0
  726. package/src/skills/skill-history.ts +250 -0
  727. package/src/skills/skillssh-registry.ts +1 -1
  728. package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
  729. package/src/stt/__tests__/language-metadata.test.ts +85 -0
  730. package/src/stt/__tests__/speech-energy.test.ts +79 -0
  731. package/src/stt/__tests__/types.test.ts +94 -0
  732. package/src/stt/daemon-batch-transcriber.ts +18 -6
  733. package/src/stt/language-metadata.ts +65 -0
  734. package/src/stt/speech-energy.ts +115 -12
  735. package/src/stt/stt-stream-session.ts +8 -4
  736. package/src/stt/types.ts +119 -0
  737. package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
  738. package/src/subagent/consult-prompt.ts +3 -1
  739. package/src/subagent/consult-transcript.ts +1 -1
  740. package/src/subagent/manager.ts +304 -44
  741. package/src/subagent/progress-events.ts +35 -0
  742. package/src/subagent/role-resolution.ts +88 -0
  743. package/src/subagent/types.ts +303 -67
  744. package/src/subagent/validate-allowlists.ts +3 -4
  745. package/src/telegram/__tests__/webhook-health.test.ts +73 -1
  746. package/src/telegram/webhook-health.ts +101 -3
  747. package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
  748. package/src/telemetry/activation-funnel.ts +6 -1
  749. package/src/telemetry/live-voice-funnel.ts +75 -0
  750. package/src/telemetry/telemetry-event-sources.ts +6 -0
  751. package/src/telemetry/telemetry-wire-source.json +1 -1
  752. package/src/telemetry/telemetry-wire.generated.ts +2 -0
  753. package/src/telemetry/types.ts +23 -4
  754. package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
  755. package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
  756. package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
  757. package/src/tools/ask-question/ask-question-tool.ts +56 -2
  758. package/src/tools/browser/browser-execution.ts +1 -1
  759. package/src/tools/calls/call-start.ts +8 -5
  760. package/src/tools/conversation-groups/move_to_group.ts +10 -12
  761. package/src/tools/document/document-tool.ts +11 -4
  762. package/src/tools/execution-timeout.ts +9 -2
  763. package/src/tools/executor.ts +7 -6
  764. package/src/tools/filesystem/edit.ts +1 -1
  765. package/src/tools/filesystem/list.ts +1 -1
  766. package/src/tools/filesystem/read.ts +7 -13
  767. package/src/tools/filesystem/search.ts +23 -21
  768. package/src/tools/filesystem/write.ts +4 -1
  769. package/src/tools/host-filesystem/edit.ts +7 -2
  770. package/src/tools/host-filesystem/read.ts +21 -8
  771. package/src/tools/host-filesystem/write.ts +10 -2
  772. package/src/tools/host-terminal/host-shell.ts +2 -2
  773. package/src/tools/mcp/mcp-tool-factory.ts +25 -1
  774. package/src/tools/permission-checker.ts +8 -3
  775. package/src/tools/schedule/delete.ts +13 -1
  776. package/src/tools/schedule/list.ts +14 -3
  777. package/src/tools/schedule/update.ts +56 -33
  778. package/src/tools/shared/filesystem/audio-read.ts +6 -4
  779. package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
  780. package/src/tools/shared/filesystem/image-read.ts +12 -12
  781. package/src/tools/shared/filesystem/path-policy.ts +70 -5
  782. package/src/tools/shared/filesystem/size-guard.ts +6 -6
  783. package/src/tools/shared/input-misuse.ts +72 -0
  784. package/src/tools/skills/scaffold-managed.ts +82 -0
  785. package/src/tools/skills/skill-tool-factory.ts +14 -1
  786. package/src/tools/subagent/consult-deadline.ts +12 -7
  787. package/src/tools/subagent/message.ts +3 -1
  788. package/src/tools/subagent/read.ts +66 -4
  789. package/src/tools/subagent/spawn.ts +500 -78
  790. package/src/tools/tool-approval-handler.ts +45 -4
  791. package/src/tools/types.ts +22 -3
  792. package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
  793. package/src/tools/ui-surface/visual-validation.ts +2 -0
  794. package/src/tts/__tests__/provider-adapters.test.ts +147 -0
  795. package/src/tts/__tests__/speakable-segments.test.ts +470 -0
  796. package/src/tts/language-voices.ts +23 -0
  797. package/src/tts/providers/deepgram-provider.ts +3 -1
  798. package/src/tts/providers/elevenlabs-provider.ts +73 -1
  799. package/src/tts/providers/xai-provider.ts +28 -2
  800. package/src/tts/speakable-segments.ts +293 -23
  801. package/src/tts/synthesis-stream.ts +7 -0
  802. package/src/tts/types.ts +7 -0
  803. package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
  804. package/src/usage/subagent-attribution.ts +108 -0
  805. package/src/util/__tests__/language-subtag.test.ts +54 -0
  806. package/src/util/__tests__/sqlite-retry.test.ts +112 -0
  807. package/src/util/abort-reasons.ts +16 -0
  808. package/src/util/ansi.ts +44 -0
  809. package/src/util/disk-usage.ts +17 -13
  810. package/src/util/image-conversion.ts +20 -13
  811. package/src/util/language-subtag.ts +43 -0
  812. package/src/util/schedule-source-key.ts +23 -0
  813. package/src/util/unicode.ts +1 -1
  814. package/src/util/worker-process.ts +7 -1
  815. package/src/workspace/git-service.ts +33 -9
  816. package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
  817. package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
  818. package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
  819. package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
  820. package/src/workspace/migrations/registry.ts +6 -0
  821. package/src/workspace/top-level-renderer.ts +15 -2
  822. package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
  823. package/src/daemon/install-symlink.ts +0 -207
  824. package/src/live-voice/__tests__/front-decision.test.ts +0 -525
  825. package/src/live-voice/front-decision.ts +0 -429
  826. package/src/notifications/deferred-emit.ts +0 -147
@@ -24,8 +24,11 @@ mock.module("../../security/secure-keys.js", () => ({
24
24
  getProviderKeyAsync: async () => "test-api-key",
25
25
  }));
26
26
 
27
- const { LiveVoiceTtsError, streamLiveVoiceTtsAudio } =
28
- await import("../live-voice-tts.js");
27
+ const {
28
+ LiveVoiceTtsError,
29
+ resolveLanguageVoiceOverride,
30
+ streamLiveVoiceTtsAudio,
31
+ } = await import("../live-voice-tts.js");
29
32
 
30
33
  beforeEach(() => {
31
34
  config = makeConfig();
@@ -494,6 +497,76 @@ describe("streamLiveVoiceTtsAudio", () => {
494
497
  }
495
498
  });
496
499
 
500
+ test("applies the configured per-language voice for a language-known turn", async () => {
501
+ config = makeConfig({
502
+ provider: "elevenlabs",
503
+ elevenlabsLanguageVoices: { hi: "voice-hindi", ja: "voice-japanese" },
504
+ });
505
+ const { requests, probeRequests } = installCapturingElevenLabsStub();
506
+
507
+ await streamLiveVoiceTtsAudio({
508
+ config,
509
+ text: "namaste",
510
+ language: "hi",
511
+ onAudioChunk: () => {},
512
+ });
513
+
514
+ expect(requests[0]?.voiceId).toBe("voice-hindi");
515
+ expect(probeRequests[0]?.voiceId).toBe("voice-hindi");
516
+ });
517
+
518
+ test("skips the override when the map has no entry for the turn language", async () => {
519
+ config = makeConfig({
520
+ provider: "elevenlabs",
521
+ elevenlabsLanguageVoices: { hi: "voice-hindi" },
522
+ });
523
+ const { requests } = installCapturingElevenLabsStub();
524
+
525
+ await streamLiveVoiceTtsAudio({
526
+ config,
527
+ text: "hola",
528
+ language: "es",
529
+ onAudioChunk: () => {},
530
+ });
531
+
532
+ expect(requests[0]?.voiceId).toBeUndefined();
533
+ });
534
+
535
+ test("an explicit request voiceId outranks the per-language map", async () => {
536
+ config = makeConfig({
537
+ provider: "elevenlabs",
538
+ elevenlabsLanguageVoices: { hi: "voice-hindi" },
539
+ });
540
+ const { requests, probeRequests } = installCapturingElevenLabsStub();
541
+
542
+ await streamLiveVoiceTtsAudio({
543
+ config,
544
+ text: "namaste",
545
+ language: "hi",
546
+ voiceId: "voice-explicit",
547
+ onAudioChunk: () => {},
548
+ });
549
+
550
+ expect(requests[0]?.voiceId).toBe("voice-explicit");
551
+ expect(probeRequests[0]?.voiceId).toBe("voice-explicit");
552
+ });
553
+
554
+ test("performs no lookup when the turn has no language", async () => {
555
+ config = makeConfig({
556
+ provider: "elevenlabs",
557
+ elevenlabsLanguageVoices: { hi: "voice-hindi" },
558
+ });
559
+ const { requests } = installCapturingElevenLabsStub();
560
+
561
+ await streamLiveVoiceTtsAudio({
562
+ config,
563
+ text: "hello",
564
+ onAudioChunk: () => {},
565
+ });
566
+
567
+ expect(requests[0]?.voiceId).toBeUndefined();
568
+ });
569
+
497
570
  test("keeps live voice TTS behind the registry instead of direct provider SDKs", () => {
498
571
  const source = readFileSync(
499
572
  new URL("../live-voice-tts.ts", import.meta.url),
@@ -508,11 +581,83 @@ describe("streamLiveVoiceTtsAudio", () => {
508
581
  });
509
582
  });
510
583
 
584
+ describe("resolveLanguageVoiceOverride", () => {
585
+ test("keys the lookup by the language's lowercase base subtag", () => {
586
+ const map = { hi: "voice-hindi" };
587
+ expect(resolveLanguageVoiceOverride(map, "hi")).toBe("voice-hindi");
588
+ expect(resolveLanguageVoiceOverride(map, "hi-IN")).toBe("voice-hindi");
589
+ expect(resolveLanguageVoiceOverride(map, "HI")).toBe("voice-hindi");
590
+ });
591
+
592
+ test("returns undefined without a language", () => {
593
+ expect(
594
+ resolveLanguageVoiceOverride({ hi: "voice-hindi" }, undefined),
595
+ ).toBeUndefined();
596
+ expect(
597
+ resolveLanguageVoiceOverride({ hi: "voice-hindi" }, ""),
598
+ ).toBeUndefined();
599
+ });
600
+
601
+ test("returns undefined for a missing map, a missing entry, or a blank voice", () => {
602
+ expect(resolveLanguageVoiceOverride(undefined, "hi")).toBeUndefined();
603
+ expect(
604
+ resolveLanguageVoiceOverride({ ja: "voice-japanese" }, "hi"),
605
+ ).toBeUndefined();
606
+ expect(resolveLanguageVoiceOverride({ hi: " " }, "hi")).toBeUndefined();
607
+ expect(resolveLanguageVoiceOverride({}, "constructor")).toBeUndefined();
608
+ });
609
+
610
+ test("trims the resolved voice identifier", () => {
611
+ expect(resolveLanguageVoiceOverride({ hi: " voice-hindi " }, "hi")).toBe(
612
+ "voice-hindi",
613
+ );
614
+ });
615
+ });
616
+
617
+ /**
618
+ * Install an ElevenLabs-id streaming stub that records the sample-rate probe
619
+ * and synthesis requests.
620
+ */
621
+ function installCapturingElevenLabsStub(): {
622
+ requests: TtsSynthesisRequest[];
623
+ probeRequests: TtsSynthesisRequest[];
624
+ } {
625
+ const requests: TtsSynthesisRequest[] = [];
626
+ const probeRequests: TtsSynthesisRequest[] = [];
627
+ _setTtsProviderForTests({
628
+ id: "elevenlabs",
629
+ capabilities: {
630
+ supportsStreaming: true,
631
+ supportedFormats: ["mp3", "pcm"],
632
+ },
633
+ resolveOutputSampleRateHz: (request) => {
634
+ probeRequests.push(request);
635
+ return 24_000;
636
+ },
637
+ async synthesize(): Promise<TtsSynthesisResult> {
638
+ throw new Error("buffered synthesis should not be used");
639
+ },
640
+ async synthesizeStream(
641
+ request: TtsSynthesisRequest,
642
+ onChunk: (chunk: Uint8Array) => void,
643
+ ): Promise<TtsSynthesisResult> {
644
+ requests.push(request);
645
+ onChunk(Buffer.from("pcm-one!"));
646
+ return {
647
+ audio: Buffer.from("pcm-one!"),
648
+ contentType: "audio/pcm",
649
+ };
650
+ },
651
+ });
652
+ return { requests, probeRequests };
653
+ }
654
+
511
655
  function makeConfig(
512
656
  overrides: {
513
657
  provider?: string;
514
658
  format?: "mp3" | "wav" | "opus";
515
659
  sampleRate?: number;
660
+ elevenlabsLanguageVoices?: Record<string, string>;
516
661
  } = {},
517
662
  ): LiveVoiceTtsConfig {
518
663
  return {
@@ -535,6 +680,7 @@ function makeConfig(
535
680
  stability: 0.5,
536
681
  similarityBoost: 0.75,
537
682
  conversationTimeoutSeconds: 30,
683
+ languageVoices: overrides.elevenlabsLanguageVoices,
538
684
  },
539
685
  deepgram: {
540
686
  model: "aura-asteria-en",
@@ -19,7 +19,6 @@ import type {
19
19
  } from "../../stt/types.js";
20
20
  import { __resetRegistryForTesting } from "../../tools/registry.js";
21
21
  import { getWorkspaceSkillsDir } from "../../util/platform.js";
22
- import type { VoiceFrontDecider } from "../front-decision.js";
23
22
  import type { LiveVoiceAudioArchiveResult } from "../live-voice-archive.js";
24
23
  import {
25
24
  CONTINUATION_DELIVERY_CONTENT,
@@ -83,6 +82,21 @@ function pcm(amplitude: number, sampleCount = 240): Uint8Array {
83
82
  return new Uint8Array(buffer);
84
83
  }
85
84
 
85
+ function tonePcm(
86
+ amplitude: number,
87
+ frequencyHz: number,
88
+ sampleCount = 240,
89
+ ): Uint8Array {
90
+ const buffer = Buffer.alloc(sampleCount * 2);
91
+ for (let index = 0; index < sampleCount; index += 1) {
92
+ const sample = Math.round(
93
+ amplitude * Math.sin((2 * Math.PI * frequencyHz * index) / SAMPLE_RATE),
94
+ );
95
+ buffer.writeInt16LE(sample, index * 2);
96
+ }
97
+ return new Uint8Array(buffer);
98
+ }
99
+
86
100
  // 10 ms of speech at 24 kHz.
87
101
  const LOUD_CHUNK = pcm(8_000);
88
102
  // 300 ms of speech at 24 kHz — comfortably exceeds the default sustained-speech
@@ -103,9 +117,13 @@ class MockStreamingTranscriber implements StreamingTranscriber {
103
117
  readonly received: Buffer[] = [];
104
118
  stopped = false;
105
119
  private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
120
+ private flushed = false;
121
+ // Claimed on the first audio chunk, so the scripted transcript is spent in
122
+ // the order cycles hear speech (see takeScriptedFinal).
123
+ private scriptedFinal: string | null = null;
106
124
 
107
125
  constructor(
108
- private readonly stopEvents: SttStreamServerEvent[],
126
+ private readonly takeScriptedFinal: () => string,
109
127
  private readonly holdStopEvents = false,
110
128
  ) {}
111
129
 
@@ -114,6 +132,7 @@ class MockStreamingTranscriber implements StreamingTranscriber {
114
132
  }
115
133
 
116
134
  sendAudio(chunk: Buffer): void {
135
+ this.scriptedFinal ??= this.takeScriptedFinal();
117
136
  this.received.push(Buffer.from(chunk));
118
137
  }
119
138
 
@@ -128,19 +147,20 @@ class MockStreamingTranscriber implements StreamingTranscriber {
128
147
  }
129
148
 
130
149
  flushStopEvents(): void {
150
+ if (this.flushed) {
151
+ return;
152
+ }
153
+ this.flushed = true;
131
154
  // A stream that was never sent audio has nothing to transcribe, so a real
132
- // provider closes without a final. Scripting one anyway let a cycle that
133
- // armed and tore down in the same breath — a client interrupt racing the
134
- // post-cancel re-arm — hand the session a transcript it never heard, which
135
- // launched a turn nothing would ever finalize and wedged the
155
+ // provider closes without a final. Scripting one anyway lets a cycle that
156
+ // arms and tears down in the same breath (a client interrupt racing the
157
+ // post-cancel re-arm) hand the session a transcript it never heard, which
158
+ // launches a turn nothing will ever finalize and wedges the
136
159
  // one-turn-at-a-time gate for every later utterance.
137
- const events =
138
- this.received.length > 0
139
- ? this.stopEvents
140
- : this.stopEvents.filter((event) => event.type !== "final");
141
- for (const event of events) {
142
- this.onEvent?.(event);
160
+ if (this.scriptedFinal !== null) {
161
+ this.onEvent?.({ type: "final", text: this.scriptedFinal });
143
162
  }
163
+ this.onEvent?.({ type: "closed" });
144
164
  }
145
165
 
146
166
  // Provider-initiated event (e.g. an idle-timeout close), no stop() needed.
@@ -158,7 +178,9 @@ function createHarness(options: {
158
178
  turnDetectorConfig?: TurnDetectorConfig;
159
179
  speechEnergyThreshold?: number;
160
180
  bargeInMinSpeechMs?: number;
161
- frontDecider?: VoiceFrontDecider | null;
181
+ echoBargeInMargin?: number;
182
+ echoEmaHalfLifeMs?: number;
183
+ echoDrainSlackMs?: number;
162
184
  frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
163
185
  emitMetrics?: boolean;
164
186
  metricsClock?: () => number;
@@ -192,12 +214,23 @@ function createHarness(options: {
192
214
  };
193
215
 
194
216
  const finals = options.finals ?? ["hello world"];
217
+ // Scripted finals are handed out in the order cycles first receive audio,
218
+ // not in the order they arm. Server VAD arms and discards audio-free cycles
219
+ // on wall-clock timers (the trailing-silence timer, the post-cancel re-arm),
220
+ // so how many cycles have armed by a given point depends on how fast the
221
+ // event loop is, while the order the test feeds audio in does not. Keying
222
+ // off audio keeps `finals` reading as "what the user says, in order".
223
+ let scriptedFinalIndex = 0;
224
+ const takeScriptedFinal = (): string => {
225
+ const text =
226
+ finals[scriptedFinalIndex] ?? `utterance ${scriptedFinalIndex + 1}`;
227
+ scriptedFinalIndex += 1;
228
+ return text;
229
+ };
195
230
  const transcribers: MockStreamingTranscriber[] = [];
196
231
  const resolveTranscriber = mock(async () => {
197
- const text =
198
- finals[transcribers.length] ?? `utterance ${transcribers.length + 1}`;
199
232
  const transcriber = new MockStreamingTranscriber(
200
- [{ type: "final", text }, { type: "closed" }],
233
+ takeScriptedFinal,
201
234
  options.holdStopEventsFor?.includes(transcribers.length) ?? false,
202
235
  );
203
236
  transcribers.push(transcriber);
@@ -225,9 +258,10 @@ function createHarness(options: {
225
258
  (options.viaFactory ? undefined : { silenceThresholdMs: 40 }),
226
259
  speechEnergyThreshold: options.speechEnergyThreshold,
227
260
  bargeInMinSpeechMs: options.bargeInMinSpeechMs,
228
- ...(options.frontDecider !== undefined
229
- ? { frontDecider: options.frontDecider }
230
- : {}),
261
+ echoBargeInMargin:
262
+ options.echoBargeInMargin ?? (options.viaFactory ? undefined : 1),
263
+ echoEmaHalfLifeMs: options.echoEmaHalfLifeMs,
264
+ echoDrainSlackMs: options.echoDrainSlackMs,
231
265
  ...(options.frontModelConfig
232
266
  ? { frontModelConfig: options.frontModelConfig }
233
267
  : {}),
@@ -274,6 +308,15 @@ function makeTtsChunk(text: string): LiveVoiceTtsAudioChunk {
274
308
  };
275
309
  }
276
310
 
311
+ function makePcmTtsChunk(audio: Uint8Array): LiveVoiceTtsAudioChunk {
312
+ return {
313
+ type: "tts_audio",
314
+ contentType: "audio/pcm",
315
+ sampleRate: SAMPLE_RATE,
316
+ dataBase64: Buffer.from(audio).toString("base64"),
317
+ };
318
+ }
319
+
277
320
  function makeTtsResult(text: string): LiveVoiceTtsResult {
278
321
  return {
279
322
  provider: "fish-audio",
@@ -2929,8 +2972,11 @@ describe("LiveVoiceSession server VAD", () => {
2929
2972
 
2930
2973
  test("an idle transcriber close before speech re-arms capture for the next utterance", async () => {
2931
2974
  const { startVoiceTurn, calls } = makeAutoCompletingTurnStarter(["Hi."]);
2975
+ // The first transcriber closes before it is sent any audio, so it never
2976
+ // claims a scripted final: the one entry belongs to the cycle that speech
2977
+ // actually reaches.
2932
2978
  const { frames, session, transcribers } = createHarness({
2933
- finals: ["never spoken", "hello after close"],
2979
+ finals: ["hello after close"],
2934
2980
  startVoiceTurn,
2935
2981
  });
2936
2982
 
@@ -3189,6 +3235,9 @@ describe("LiveVoiceSession VAD threshold configuration", () => {
3189
3235
  silenceThresholdMs: 1200,
3190
3236
  maxTurnDurationMs: 30_000,
3191
3237
  bargeInMinSpeechMs: 250,
3238
+ echoBargeInMargin: 1.5,
3239
+ echoEmaHalfLifeMs: 400,
3240
+ echoDrainSlackMs: 300,
3192
3241
  });
3193
3242
 
3194
3243
  const { frames, session } = createHarness({ viaFactory: true });
@@ -3302,6 +3351,10 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
3302
3351
  // detector timers stay out of the guard's audio-duration accounting.
3303
3352
  function createSpeakingTurnHarness(options: {
3304
3353
  bargeInMinSpeechMs: number;
3354
+ echoEmaHalfLifeMs?: number;
3355
+ echoBargeInMargin?: number;
3356
+ echoDrainSlackMs?: number;
3357
+ ttsAudio?: Uint8Array;
3305
3358
  finals?: string[];
3306
3359
  startFrame?: LiveVoiceClientStartFrame;
3307
3360
  }) {
@@ -3312,7 +3365,11 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
3312
3365
  return { turnId: "bridge-turn", abort };
3313
3366
  });
3314
3367
  const streamTtsAudio = mock(async (ttsOptions: LiveVoiceTtsOptions) => {
3315
- ttsOptions.onAudioChunk(makeTtsChunk("assistant audio"));
3368
+ ttsOptions.onAudioChunk(
3369
+ options.ttsAudio
3370
+ ? makePcmTtsChunk(options.ttsAudio)
3371
+ : makeTtsChunk("assistant audio"),
3372
+ );
3316
3373
  return makeTtsResult("assistant audio");
3317
3374
  });
3318
3375
  const harness = createHarness({
@@ -3320,6 +3377,9 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
3320
3377
  startVoiceTurn,
3321
3378
  streamTtsAudio,
3322
3379
  bargeInMinSpeechMs: options.bargeInMinSpeechMs,
3380
+ echoEmaHalfLifeMs: options.echoEmaHalfLifeMs ?? 4,
3381
+ echoBargeInMargin: options.echoBargeInMargin ?? 1,
3382
+ echoDrainSlackMs: options.echoDrainSlackMs ?? 60_000,
3323
3383
  turnDetectorConfig: { silenceThresholdMs: 5_000 },
3324
3384
  ...(options.startFrame ? { startFrame: options.startFrame } : {}),
3325
3385
  });
@@ -3633,6 +3693,238 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
3633
3693
  ).toMatchObject({ type: "turn_cancelled", turnId: "live-turn-1" });
3634
3694
  await waitFor(() => abort.mock.calls.length === 1);
3635
3695
  });
3696
+
3697
+ describe("echo-adaptive barge-in", () => {
3698
+ const playbackEchoChunk = tonePcm(4_700, 200);
3699
+ const bargeInSpeechChunk = tonePcm(9_400, 530);
3700
+ const playbackReference = tonePcm(4_700, 200, SAMPLE_RATE * 2);
3701
+
3702
+ test("steady loud playback echo does not interrupt the turn", async () => {
3703
+ const { frames, session, abort, speakFirstReply } =
3704
+ createSpeakingTurnHarness({
3705
+ bargeInMinSpeechMs: 60,
3706
+ echoBargeInMargin: 1.5,
3707
+ echoEmaHalfLifeMs: 40,
3708
+ ttsAudio: playbackReference,
3709
+ });
3710
+ await speakFirstReply();
3711
+ const speechStartedBaseline = countType(frames, "speech_started");
3712
+
3713
+ for (let index = 0; index < 40; index += 1) {
3714
+ await session.handleBinaryAudio(playbackEchoChunk);
3715
+ }
3716
+ await flushAsyncCallbacks();
3717
+
3718
+ expect(countType(frames, "speech_started")).toBe(speechStartedBaseline);
3719
+ expect(countType(frames, "turn_cancelled")).toBe(0);
3720
+ expect(abort).not.toHaveBeenCalled();
3721
+ });
3722
+
3723
+ test("speech above the learned echo margin still interrupts", async () => {
3724
+ const { frames, session, abort, speakFirstReply } =
3725
+ createSpeakingTurnHarness({
3726
+ bargeInMinSpeechMs: 60,
3727
+ echoBargeInMargin: 1.5,
3728
+ echoEmaHalfLifeMs: 400,
3729
+ ttsAudio: playbackReference,
3730
+ });
3731
+ await speakFirstReply();
3732
+
3733
+ for (let index = 0; index < 25; index += 1) {
3734
+ await session.handleBinaryAudio(playbackEchoChunk);
3735
+ }
3736
+ for (let index = 0; index < 8; index += 1) {
3737
+ await session.handleBinaryAudio(bargeInSpeechChunk);
3738
+ }
3739
+
3740
+ await waitFor(() => countType(frames, "turn_cancelled") === 1);
3741
+ await waitFor(() => abort.mock.calls.length === 1);
3742
+ });
3743
+
3744
+ test("classified echo resets a partial guard run immediately", async () => {
3745
+ const { frames, session, abort, speakFirstReply } =
3746
+ createSpeakingTurnHarness({
3747
+ bargeInMinSpeechMs: 60,
3748
+ echoBargeInMargin: 1.5,
3749
+ echoEmaHalfLifeMs: 400,
3750
+ ttsAudio: playbackReference,
3751
+ });
3752
+ await speakFirstReply();
3753
+
3754
+ for (let index = 0; index < 25; index += 1) {
3755
+ await session.handleBinaryAudio(playbackEchoChunk);
3756
+ }
3757
+ for (let index = 0; index < 5; index += 1) {
3758
+ await session.handleBinaryAudio(bargeInSpeechChunk);
3759
+ }
3760
+ await session.handleBinaryAudio(playbackEchoChunk);
3761
+ await session.handleBinaryAudio(bargeInSpeechChunk);
3762
+ await flushAsyncCallbacks();
3763
+
3764
+ expect(countType(frames, "turn_cancelled")).toBe(0);
3765
+ expect(abort).not.toHaveBeenCalled();
3766
+
3767
+ for (let index = 0; index < 5; index += 1) {
3768
+ await session.handleBinaryAudio(bargeInSpeechChunk);
3769
+ }
3770
+ await waitFor(() => countType(frames, "turn_cancelled") === 1);
3771
+ });
3772
+
3773
+ test("quiet playback keeps fixed-threshold barge-in sensitivity", async () => {
3774
+ const { frames, session, abort, speakFirstReply } =
3775
+ createSpeakingTurnHarness({
3776
+ bargeInMinSpeechMs: 60,
3777
+ echoBargeInMargin: 1.5,
3778
+ echoEmaHalfLifeMs: 40,
3779
+ ttsAudio: playbackReference,
3780
+ });
3781
+ await speakFirstReply();
3782
+
3783
+ for (let index = 0; index < 31; index += 1) {
3784
+ await session.handleBinaryAudio(pcm(200));
3785
+ }
3786
+ for (let index = 0; index < 7; index += 1) {
3787
+ await session.handleBinaryAudio(bargeInSpeechChunk);
3788
+ }
3789
+
3790
+ await waitFor(() => countType(frames, "turn_cancelled") === 1);
3791
+ await waitFor(() => abort.mock.calls.length === 1);
3792
+ });
3793
+
3794
+ test("playback echo is not forwarded as transcription pre-roll", async () => {
3795
+ const { frames, session, transcribers, speakFirstReply } =
3796
+ createSpeakingTurnHarness({
3797
+ bargeInMinSpeechMs: 60,
3798
+ echoBargeInMargin: 1.5,
3799
+ echoEmaHalfLifeMs: 40,
3800
+ ttsAudio: playbackReference,
3801
+ });
3802
+ await speakFirstReply();
3803
+
3804
+ const echoChunk = playbackEchoChunk;
3805
+ for (let index = 0; index < 5; index += 1) {
3806
+ await session.handleBinaryAudio(echoChunk);
3807
+ }
3808
+ for (let index = 0; index < 7; index += 1) {
3809
+ await session.handleBinaryAudio(bargeInSpeechChunk);
3810
+ }
3811
+ await waitFor(() => countType(frames, "turn_cancelled") === 1);
3812
+
3813
+ const echoBuffer = Buffer.from(echoChunk);
3814
+ expect(
3815
+ transcribers.some((transcriber) =>
3816
+ transcriber.received.some((buffer) => buffer.equals(echoBuffer)),
3817
+ ),
3818
+ ).toBe(false);
3819
+ });
3820
+
3821
+ test("instant barge-in remains protected from onset echo", async () => {
3822
+ const { frames, session, abort, speakFirstReply } =
3823
+ createSpeakingTurnHarness({
3824
+ bargeInMinSpeechMs: 0,
3825
+ echoBargeInMargin: 1.5,
3826
+ echoEmaHalfLifeMs: 40,
3827
+ ttsAudio: playbackReference,
3828
+ });
3829
+ await speakFirstReply();
3830
+
3831
+ for (let index = 0; index < 30; index += 1) {
3832
+ await session.handleBinaryAudio(playbackEchoChunk);
3833
+ }
3834
+ await flushAsyncCallbacks();
3835
+ expect(countType(frames, "turn_cancelled")).toBe(0);
3836
+
3837
+ await session.handleBinaryAudio(bargeInSpeechChunk);
3838
+ await waitFor(() => countType(frames, "turn_cancelled") === 1);
3839
+ await waitFor(() => abort.mock.calls.length === 1);
3840
+ });
3841
+
3842
+ test("echo suppression covers the client playback tail", async () => {
3843
+ const { frames, session, abort, speakFirstReply, completeFirstReply } =
3844
+ createSpeakingTurnHarness({
3845
+ bargeInMinSpeechMs: 60,
3846
+ echoBargeInMargin: 1.5,
3847
+ echoEmaHalfLifeMs: 40,
3848
+ ttsAudio: playbackReference,
3849
+ });
3850
+ await speakFirstReply();
3851
+ completeFirstReply();
3852
+ await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
3853
+ const speechStartedBaseline = countType(frames, "speech_started");
3854
+
3855
+ for (let index = 0; index < 40; index += 1) {
3856
+ await session.handleBinaryAudio(playbackEchoChunk);
3857
+ }
3858
+ await flushAsyncCallbacks();
3859
+
3860
+ expect(countType(frames, "speech_started")).toBe(speechStartedBaseline);
3861
+ expect(countType(frames, "turn_cancelled")).toBe(0);
3862
+ expect(abort).not.toHaveBeenCalled();
3863
+ });
3864
+
3865
+ test("speech at playback onset cannot seed its own echo threshold", async () => {
3866
+ const { frames, session, abort, speakFirstReply, transcribers } =
3867
+ createSpeakingTurnHarness({
3868
+ bargeInMinSpeechMs: 250,
3869
+ echoBargeInMargin: 1.5,
3870
+ echoEmaHalfLifeMs: 400,
3871
+ ttsAudio: playbackReference,
3872
+ });
3873
+ await speakFirstReply();
3874
+
3875
+ const onsetSpeech = tonePcm(9_400, 530, 7_200);
3876
+ await session.handleBinaryAudio(onsetSpeech);
3877
+
3878
+ await waitFor(() => countType(frames, "turn_cancelled") === 1);
3879
+ await waitFor(() => abort.mock.calls.length === 1);
3880
+ expect(
3881
+ transcribers.some((transcriber) =>
3882
+ transcriber.received.some((buffer) =>
3883
+ buffer.equals(Buffer.from(onsetSpeech)),
3884
+ ),
3885
+ ),
3886
+ ).toBe(true);
3887
+ });
3888
+
3889
+ test("speech already in progress bypasses playback warm-up", async () => {
3890
+ let callbacks: VoiceTurnCallbacks | undefined;
3891
+ const abort = mock();
3892
+ const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
3893
+ callbacks ??= options.callbacks;
3894
+ return { turnId: "bridge-turn", abort };
3895
+ });
3896
+ const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
3897
+ options.onAudioChunk(makeTtsChunk("assistant audio"));
3898
+ return makeTtsResult("assistant audio");
3899
+ });
3900
+ const { frames, session } = createHarness({
3901
+ finals: ["what's the weather", "actually never mind"],
3902
+ startVoiceTurn,
3903
+ streamTtsAudio,
3904
+ bargeInMinSpeechMs: 60,
3905
+ echoBargeInMargin: 1.5,
3906
+ echoEmaHalfLifeMs: 400,
3907
+ echoDrainSlackMs: 60_000,
3908
+ turnDetectorConfig: { silenceThresholdMs: 5_000 },
3909
+ });
3910
+
3911
+ await session.start();
3912
+ await session.handleBinaryAudio(LOUD_CHUNK);
3913
+ await session.handleClientFrame({ type: "ptt_release" });
3914
+ await waitFor(() => frames.some((frame) => frame.type === "thinking"));
3915
+ for (let index = 0; index < 3; index += 1) {
3916
+ await session.handleBinaryAudio(pcm(3_000));
3917
+ }
3918
+ callbacks?.assistant_text_delta?.(makeTextDelta("It is sunny today."));
3919
+ await waitFor(() => frames.some((frame) => frame.type === "tts_audio"));
3920
+ for (let index = 0; index < 3; index += 1) {
3921
+ await session.handleBinaryAudio(pcm(3_000));
3922
+ }
3923
+
3924
+ await waitFor(() => countType(frames, "turn_cancelled") === 1);
3925
+ await waitFor(() => abort.mock.calls.length === 1);
3926
+ });
3927
+ });
3636
3928
  });
3637
3929
 
3638
3930
  describe("LiveVoiceSession unified front-door endpointing", () => {
@@ -3670,15 +3962,6 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
3670
3962
  return { startVoiceTurn, calls, discard };
3671
3963
  }
3672
3964
 
3673
- // A decider that phrases nothing: the front-door leg IS the endpoint
3674
- // decision, so these tests only need the decider seam to exist.
3675
- function makeSilentDecider(): VoiceFrontDecider {
3676
- return {
3677
- generateAckText: async () => null,
3678
- generateProgressText: async () => null,
3679
- };
3680
- }
3681
-
3682
3965
  async function startWithPartial(
3683
3966
  session: LiveVoiceSession,
3684
3967
  transcribers: MockStreamingTranscriber[],
@@ -3691,12 +3974,10 @@ describe("LiveVoiceSession unified front-door endpointing", () => {
3691
3974
  }
3692
3975
 
3693
3976
  test("a chatty answer commits: verdict leg replaces the decider, frames follow commit order", async () => {
3694
- const decider = makeSilentDecider();
3695
3977
  const starter = makeVerdictTurnStarter([["Hey! Not much."]]);
3696
3978
  const { frames, session, transcribers } = createHarness({
3697
3979
  finals: ["hello world"],
3698
3980
  startVoiceTurn: starter.startVoiceTurn,
3699
- frontDecider: decider,
3700
3981
  });
3701
3982
 
3702
3983
  await startWithPartial(session, transcribers);