@vellumai/assistant 0.11.2 → 0.11.3-dev.202608080427.92f668a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (826) hide show
  1. package/ARCHITECTURE.md +69 -9
  2. package/docs/flux-turn-detection-spike.md +243 -0
  3. package/docs/stt-provider-onboarding.md +3 -1
  4. package/docs/trusted-contact-access.md +9 -5
  5. package/docs/vellum-doctor.md +195 -0
  6. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  7. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  8. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  9. package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
  10. package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
  11. package/node_modules/@vellumai/environments/package.json +2 -1
  12. package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
  13. package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
  14. package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
  15. package/node_modules/@vellumai/environments/src/index.ts +17 -6
  16. package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
  17. package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
  18. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  19. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  20. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  21. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
  22. package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
  23. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
  24. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
  25. package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
  26. package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
  27. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
  28. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
  29. package/node_modules/@vellumai/service-contracts/src/channels.ts +50 -0
  30. package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  31. package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  32. package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
  33. package/openapi.yaml +619 -47
  34. package/package.json +1 -1
  35. package/scripts/voice-ttft-spike.ts +3 -3
  36. package/src/__tests__/agent-image-optimize.test.ts +4 -4
  37. package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
  38. package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
  39. package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
  40. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
  41. package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
  42. package/src/__tests__/answered-question-persistence.test.ts +240 -0
  43. package/src/__tests__/anthropic-provider.test.ts +165 -30
  44. package/src/__tests__/app-control-flow.test.ts +9 -8
  45. package/src/__tests__/assistant-event-hub.test.ts +48 -0
  46. package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
  47. package/src/__tests__/attachments-store.test.ts +130 -68
  48. package/src/__tests__/attachments.test.ts +36 -36
  49. package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
  50. package/src/__tests__/call-controller.test.ts +120 -0
  51. package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
  52. package/src/__tests__/canned-reply-release.test.ts +86 -0
  53. package/src/__tests__/channel-availability-routes.test.ts +14 -1
  54. package/src/__tests__/channel-guardian.test.ts +59 -29
  55. package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
  56. package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
  57. package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
  58. package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
  59. package/src/__tests__/config-loader-backfill.test.ts +19 -0
  60. package/src/__tests__/config-schema.test.ts +150 -7
  61. package/src/__tests__/consult-deadline.test.ts +65 -0
  62. package/src/__tests__/conversation-agent-loop.test.ts +438 -2
  63. package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
  64. package/src/__tests__/conversation-disk-view.test.ts +4 -4
  65. package/src/__tests__/conversation-error.test.ts +22 -3
  66. package/src/__tests__/conversation-fork-crud.test.ts +5 -1
  67. package/src/__tests__/conversation-fork-referential.test.ts +352 -0
  68. package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
  69. package/src/__tests__/conversation-group-tools.test.ts +32 -2
  70. package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
  71. package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
  72. package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
  73. package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
  74. package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
  75. package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
  76. package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
  77. package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
  78. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
  79. package/src/__tests__/conversation-queue.test.ts +433 -1
  80. package/src/__tests__/conversation-retry-route.test.ts +3 -1
  81. package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
  82. package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
  83. package/src/__tests__/conversation-store.test.ts +26 -10
  84. package/src/__tests__/conversation-summarize-route.test.ts +1 -1
  85. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
  86. package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
  87. package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
  88. package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
  89. package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
  90. package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
  91. package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
  92. package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
  93. package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
  94. package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
  95. package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
  96. package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
  97. package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
  98. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
  99. package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
  100. package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
  101. package/src/__tests__/credential-prompt-route.test.ts +54 -7
  102. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  103. package/src/__tests__/cu-unified-flow.test.ts +89 -28
  104. package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
  105. package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
  106. package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
  107. package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
  108. package/src/__tests__/disk-pressure-guard.test.ts +55 -55
  109. package/src/__tests__/disk-pressure-routes.test.ts +10 -10
  110. package/src/__tests__/disk-pressure-tools.test.ts +2 -2
  111. package/src/__tests__/disk-usage.test.ts +35 -20
  112. package/src/__tests__/document-sync-tags.test.ts +379 -0
  113. package/src/__tests__/document-update-default-surface.test.ts +28 -0
  114. package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
  115. package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
  116. package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
  117. package/src/__tests__/events-dev-bypass-actor.test.ts +294 -1
  118. package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
  119. package/src/__tests__/file-list-tool.test.ts +12 -10
  120. package/src/__tests__/file-ops-service.test.ts +151 -44
  121. package/src/__tests__/file-read-container-boundary.test.ts +137 -0
  122. package/src/__tests__/file-read-tool.test.ts +21 -9
  123. package/src/__tests__/file-url-path-guard.test.ts +83 -0
  124. package/src/__tests__/filesystem-tools.test.ts +82 -66
  125. package/src/__tests__/guardian-outbound-http.test.ts +8 -8
  126. package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
  127. package/src/__tests__/helpers/hub-clients.ts +42 -0
  128. package/src/__tests__/helpers/mock-conversation.ts +46 -0
  129. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
  130. package/src/__tests__/history-repair.test.ts +112 -12
  131. package/src/__tests__/host-bash-proxy.test.ts +10 -5
  132. package/src/__tests__/host-bash-routes.test.ts +67 -0
  133. package/src/__tests__/host-browser-proxy.test.ts +2 -2
  134. package/src/__tests__/host-cu-proxy.test.ts +3 -7
  135. package/src/__tests__/host-file-proxy.test.ts +3 -7
  136. package/src/__tests__/host-file-read-tool.test.ts +5 -1
  137. package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
  138. package/src/__tests__/http-conversation-lineage.test.ts +2 -5
  139. package/src/__tests__/http-user-message-parity.test.ts +44 -0
  140. package/src/__tests__/identity-routes.test.ts +13 -13
  141. package/src/__tests__/image-conversion.test.ts +29 -26
  142. package/src/__tests__/image-source-path-reinject.test.ts +2 -2
  143. package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
  144. package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
  145. package/src/__tests__/introduction-card-resolver.test.ts +6 -1
  146. package/src/__tests__/list-messages-attachments.test.ts +56 -32
  147. package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
  148. package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
  149. package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
  150. package/src/__tests__/list-messages-page-latest.test.ts +61 -61
  151. package/src/__tests__/list-messages-provider-error.test.ts +6 -6
  152. package/src/__tests__/list-messages-queued.test.ts +49 -6
  153. package/src/__tests__/list-messages-system-card.test.ts +4 -4
  154. package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
  155. package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
  156. package/src/__tests__/llm-usage-store.test.ts +55 -0
  157. package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
  158. package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
  159. package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
  160. package/src/__tests__/media-stream-output.test.ts +175 -0
  161. package/src/__tests__/media-stream-server-integration.test.ts +36 -0
  162. package/src/__tests__/media-stream-stt-session.test.ts +67 -0
  163. package/src/__tests__/migration-export-http.test.ts +1 -1
  164. package/src/__tests__/migration-import-commit-http.test.ts +1 -1
  165. package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
  166. package/src/__tests__/migration-validate-http.test.ts +1 -1
  167. package/src/__tests__/mtime-cache.test.ts +65 -3
  168. package/src/__tests__/notification-platform-adapter.test.ts +104 -0
  169. package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
  170. package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
  171. package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
  172. package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
  173. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  174. package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
  175. package/src/__tests__/path-policy.test.ts +164 -1
  176. package/src/__tests__/persist-media-references.test.ts +2 -2
  177. package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
  178. package/src/__tests__/persist-unsendable-image.test.ts +23 -23
  179. package/src/__tests__/persona-resolver.test.ts +154 -0
  180. package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
  181. package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
  182. package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
  183. package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
  184. package/src/__tests__/profiler-routes.test.ts +13 -13
  185. package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
  186. package/src/__tests__/proxy-approval-callback.test.ts +3 -3
  187. package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
  188. package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
  189. package/src/__tests__/resolve-trust-class.test.ts +20 -0
  190. package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
  191. package/src/__tests__/run-due-schedules.test.ts +174 -0
  192. package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
  193. package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
  194. package/src/__tests__/schedule-routes.test.ts +730 -8
  195. package/src/__tests__/schedule-store.test.ts +540 -0
  196. package/src/__tests__/schedule-tools.test.ts +103 -0
  197. package/src/__tests__/scheduler-wake.test.ts +5 -0
  198. package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
  199. package/src/__tests__/secure-keys.test.ts +91 -0
  200. package/src/__tests__/server-history-render.test.ts +97 -7
  201. package/src/__tests__/size-guard.test.ts +8 -8
  202. package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
  203. package/src/__tests__/skill-tool-factory.test.ts +247 -0
  204. package/src/__tests__/skills-install-staging.test.ts +15 -6
  205. package/src/__tests__/skills.test.ts +32 -0
  206. package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
  207. package/src/__tests__/slack-inbound-verification.test.ts +4 -1
  208. package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
  209. package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
  210. package/src/__tests__/starter-task-flow.test.ts +5 -4
  211. package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
  212. package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
  213. package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
  214. package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
  215. package/src/__tests__/subagent-disposal.test.ts +10 -0
  216. package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
  217. package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
  218. package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
  219. package/src/__tests__/subagent-manager-notify.test.ts +148 -0
  220. package/src/__tests__/subagent-notify-parent.test.ts +1 -1
  221. package/src/__tests__/subagent-persistence.test.ts +141 -0
  222. package/src/__tests__/subagent-role-registry.test.ts +172 -63
  223. package/src/__tests__/subagent-role-resolution.test.ts +88 -0
  224. package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
  225. package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
  226. package/src/__tests__/subagent-terminal-message.test.ts +88 -1
  227. package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
  228. package/src/__tests__/subagent-tools.test.ts +1901 -108
  229. package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
  230. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
  231. package/src/__tests__/system-prompt.test.ts +82 -7
  232. package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
  233. package/src/__tests__/tool-result-spool.test.ts +74 -18
  234. package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
  235. package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
  236. package/src/__tests__/tools-audio-read.test.ts +10 -10
  237. package/src/__tests__/tools-get-route.test.ts +85 -12
  238. package/src/__tests__/tools-list-cli.test.ts +171 -0
  239. package/src/__tests__/top-level-renderer.test.ts +54 -1
  240. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
  241. package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
  242. package/src/__tests__/turn-tail-chain.test.ts +135 -0
  243. package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
  244. package/src/__tests__/ui-visual-surface.test.ts +5 -4
  245. package/src/__tests__/ui-work-result-surface.test.ts +5 -4
  246. package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
  247. package/src/__tests__/usage-routes.test.ts +62 -0
  248. package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
  249. package/src/__tests__/visible-app-context.test.ts +20 -4
  250. package/src/__tests__/web-search-history.test.ts +96 -0
  251. package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
  252. package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
  253. package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +150 -0
  254. package/src/acp/failure-error.ts +3 -5
  255. package/src/agent/attachments.ts +26 -24
  256. package/src/agent/history-repair/history-repair.ts +27 -26
  257. package/src/agent/image-optimize.ts +6 -6
  258. package/src/agent/loop.ts +40 -22
  259. package/src/agent/message-types.ts +3 -3
  260. package/src/api/constants/document-tools.ts +55 -0
  261. package/src/api/events/question-answered.ts +53 -0
  262. package/src/api/events/tool-result.ts +7 -0
  263. package/src/api/index.ts +13 -0
  264. package/src/api/responses/conversation-message.ts +11 -0
  265. package/src/api/surfaces-oauth-connect.test.ts +96 -0
  266. package/src/api/surfaces.ts +36 -8
  267. package/src/approvals/guardian-channel-delivery.ts +83 -0
  268. package/src/approvals/guardian-expiry-notifier.ts +13 -11
  269. package/src/approvals/guardian-request-resolvers.ts +222 -135
  270. package/src/calls/__tests__/tts-text-sanitizer.test.ts +13 -0
  271. package/src/calls/__tests__/voice-session-bridge.test.ts +131 -12
  272. package/src/calls/__tests__/voice-triage-escalate.test.ts +106 -0
  273. package/src/calls/call-controller.ts +30 -2
  274. package/src/calls/call-speech-output.ts +12 -4
  275. package/src/calls/call-transport.ts +18 -1
  276. package/src/calls/media-stream-output.ts +53 -5
  277. package/src/calls/media-stream-server.ts +21 -1
  278. package/src/calls/media-stream-stt-session.ts +49 -0
  279. package/src/calls/telephony-synthesis-language.ts +84 -0
  280. package/src/calls/tts-text-sanitizer.ts +12 -5
  281. package/src/calls/voice-session-bridge.ts +86 -17
  282. package/src/calls/voice-triage-escalate.ts +54 -7
  283. package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
  284. package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
  285. package/src/channels/config.ts +20 -2
  286. package/src/channels/gateway-verification-sessions.ts +7 -2
  287. package/src/channels/plugin-channel-declarations.ts +108 -0
  288. package/src/channels/types.ts +37 -1
  289. package/src/cli/AGENTS.md +5 -2
  290. package/src/cli/commands/__tests__/cache.test.ts +7 -4
  291. package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
  292. package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
  293. package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
  294. package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
  295. package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
  296. package/src/cli/commands/__tests__/plugins.test.ts +692 -0
  297. package/src/cli/commands/__tests__/schedules.test.ts +136 -37
  298. package/src/cli/commands/conversations.ts +7 -3
  299. package/src/cli/commands/credentials.help.ts +7 -0
  300. package/src/cli/commands/inference-profiles.ts +9 -3
  301. package/src/cli/commands/inference.help.ts +16 -2
  302. package/src/cli/commands/mcp.help.ts +17 -6
  303. package/src/cli/commands/mcp.ts +9 -0
  304. package/src/cli/commands/memory/memory-retrospective.ts +18 -0
  305. package/src/cli/commands/notifications.ts +3 -6
  306. package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
  307. package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
  308. package/src/cli/commands/oauth/connect.test.ts +26 -1
  309. package/src/cli/commands/oauth/connect.ts +4 -1
  310. package/src/cli/commands/oauth/index.help.ts +3 -1
  311. package/src/cli/commands/oauth/providers.ts +3 -0
  312. package/src/cli/commands/plugins.help.ts +25 -2
  313. package/src/cli/commands/plugins.ts +289 -10
  314. package/src/cli/commands/schedules.help.ts +14 -12
  315. package/src/cli/commands/schedules.ts +19 -5
  316. package/src/cli/commands/stt.help.ts +27 -2
  317. package/src/cli/commands/tools.help.ts +4 -2
  318. package/src/cli/commands/tools.ts +38 -0
  319. package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
  320. package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
  321. package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
  322. package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
  323. package/src/cli/lib/__tests__/upgrade-plugin.test.ts +161 -0
  324. package/src/cli/lib/install-from-github.ts +61 -0
  325. package/src/cli/lib/install-from-platform.ts +13 -2
  326. package/src/cli/lib/plugin-surfaces.ts +124 -4
  327. package/src/cli/lib/upgrade-plugin.ts +60 -0
  328. package/src/cli.ts +1 -1
  329. package/src/config/__tests__/default-profile-catalog.test.ts +11 -1
  330. package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
  331. package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
  332. package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
  333. package/src/config/__tests__/profile-tool-support.test.ts +131 -0
  334. package/src/config/bundled-skills/AGENTS.md +1 -1
  335. package/src/config/bundled-skills/acp/SKILL.md +6 -7
  336. package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
  337. package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
  338. package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
  339. package/src/config/bundled-skills/phone-calls/references/CONFIG.md +15 -15
  340. package/src/config/bundled-skills/schedule/SKILL.md +5 -1
  341. package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
  342. package/src/config/bundled-skills/settings/TOOLS.json +3 -3
  343. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
  344. package/src/config/bundled-skills/subagent/SKILL.md +53 -22
  345. package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
  346. package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
  347. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
  348. package/src/config/bundled-skills/visualize/SKILL.md +1 -1
  349. package/src/config/call-site-defaults.ts +4 -5
  350. package/src/config/default-profile-catalog.ts +9 -5
  351. package/src/config/feature-flag-registry.json +25 -17
  352. package/src/config/llm-resolver.ts +45 -15
  353. package/src/config/loader.ts +75 -1
  354. package/src/config/profile-tool-support.ts +110 -0
  355. package/src/config/sanitize-for-transfer.ts +11 -0
  356. package/src/config/schema.ts +4 -0
  357. package/src/config/schemas/__tests__/live-voice.test.ts +142 -4
  358. package/src/config/schemas/__tests__/stt.test.ts +20 -5
  359. package/src/config/schemas/call-site-catalog.ts +4 -4
  360. package/src/config/schemas/calls.ts +0 -4
  361. package/src/config/schemas/channels.ts +13 -0
  362. package/src/config/schemas/live-voice.ts +82 -23
  363. package/src/config/schemas/llm.ts +1 -1
  364. package/src/config/schemas/memory-retrospective.ts +25 -0
  365. package/src/config/schemas/plugin-updates.ts +59 -0
  366. package/src/config/schemas/services.ts +8 -4
  367. package/src/config/schemas/stt.ts +18 -5
  368. package/src/config/schemas/timeouts.ts +1 -1
  369. package/src/config/schemas/tts.ts +63 -0
  370. package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
  371. package/src/contacts/guardian-delivery-reader.ts +29 -2
  372. package/src/context/compactor.ts +4 -4
  373. package/src/context/post-turn-tool-result-truncation.ts +32 -16
  374. package/src/context/strip-injections.ts +42 -0
  375. package/src/context/tool-result-spool.ts +32 -18
  376. package/src/credential-execution/ces-runtime.ts +3 -0
  377. package/src/credential-execution/client.ts +22 -1
  378. package/src/credential-execution/process-manager.test.ts +99 -0
  379. package/src/credential-execution/process-manager.ts +62 -16
  380. package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
  381. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
  382. package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
  383. package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
  384. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
  385. package/src/daemon/channel-ui-capability.ts +4 -4
  386. package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
  387. package/src/daemon/conversation-agent-loop.ts +318 -148
  388. package/src/daemon/conversation-attachments.ts +1 -1
  389. package/src/daemon/conversation-error.ts +8 -1
  390. package/src/daemon/conversation-initial-prompt.ts +16 -38
  391. package/src/daemon/conversation-lifecycle.ts +154 -32
  392. package/src/daemon/conversation-messaging.ts +106 -43
  393. package/src/daemon/conversation-process.ts +131 -33
  394. package/src/daemon/conversation-queue-manager.ts +10 -0
  395. package/src/daemon/conversation-runtime-assembly.ts +9 -4
  396. package/src/daemon/conversation-store.ts +52 -3
  397. package/src/daemon/conversation-surface-state.ts +9 -0
  398. package/src/daemon/conversation-surfaces.ts +403 -210
  399. package/src/daemon/conversation-tool-setup.ts +56 -7
  400. package/src/daemon/conversation-turn-finalize.ts +134 -89
  401. package/src/daemon/conversation.ts +111 -27
  402. package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
  403. package/src/daemon/disk-pressure-guard.ts +3 -3
  404. package/src/daemon/doordash-steps.ts +5 -5
  405. package/src/daemon/external-plugins-bootstrap.ts +7 -1
  406. package/src/daemon/handlers/config-channels.ts +48 -5
  407. package/src/daemon/handlers/conversation-history.ts +1 -1
  408. package/src/daemon/handlers/conversations.ts +62 -17
  409. package/src/daemon/handlers/shared.ts +12 -0
  410. package/src/daemon/handlers/skills.ts +11 -1
  411. package/src/daemon/identity-helpers.ts +38 -10
  412. package/src/daemon/install-assistant-command.ts +322 -0
  413. package/src/daemon/lifecycle.ts +14 -3
  414. package/src/daemon/message-types/conversations.ts +6 -7
  415. package/src/daemon/message-types/sync.ts +1 -0
  416. package/src/daemon/persist-media-references.ts +23 -14
  417. package/src/daemon/process-message.ts +6 -6
  418. package/src/daemon/tool-setup-types.ts +19 -122
  419. package/src/daemon/tool-side-effects.ts +17 -94
  420. package/src/daemon/trust-context-types.ts +39 -0
  421. package/src/daemon/trust-context.ts +14 -6
  422. package/src/daemon/turn-tail-chain.ts +73 -0
  423. package/src/daemon/web-search-history.ts +47 -33
  424. package/src/heartbeat/heartbeat-service.ts +6 -6
  425. package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
  426. package/src/inbound/oauth-callback-url.ts +61 -0
  427. package/src/ipc/__tests__/socket-path.test.ts +21 -0
  428. package/src/ipc/assistant-server.ts +17 -10
  429. package/src/ipc/cli-client.ts +10 -1
  430. package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
  431. package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
  432. package/src/ipc/socket-cleanup.ts +9 -7
  433. package/src/ipc/socket-path.ts +4 -96
  434. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
  435. package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
  436. package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
  437. package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
  438. package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
  439. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +932 -0
  440. package/src/live-voice/__tests__/live-voice-integration.test.ts +12 -5
  441. package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
  442. package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
  443. package/src/live-voice/__tests__/live-voice-progress.test.ts +179 -146
  444. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
  445. package/src/live-voice/__tests__/live-voice-stt.test.ts +324 -2
  446. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +131 -2
  447. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
  448. package/src/live-voice/__tests__/live-voice-tts.test.ts +148 -2
  449. package/src/live-voice/__tests__/live-voice-vad.test.ts +313 -32
  450. package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
  451. package/src/live-voice/__tests__/progress-phrases.test.ts +167 -0
  452. package/src/live-voice/__tests__/protocol.test.ts +52 -0
  453. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
  454. package/src/live-voice/live-voice-archive.ts +13 -13
  455. package/src/live-voice/live-voice-connection.ts +7 -1
  456. package/src/live-voice/live-voice-metrics.ts +57 -32
  457. package/src/live-voice/live-voice-photo.ts +168 -0
  458. package/src/live-voice/live-voice-session.ts +1137 -351
  459. package/src/live-voice/live-voice-tts.ts +18 -2
  460. package/src/live-voice/progress-narration.ts +277 -0
  461. package/src/live-voice/progress-phrases.ts +105 -2
  462. package/src/live-voice/protocol.ts +110 -2
  463. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
  464. package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +196 -0
  465. package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
  466. package/src/mcp/client.ts +14 -7
  467. package/src/mcp/mcp-auth-orchestrator.ts +13 -22
  468. package/src/mcp/mcp-oauth-provider.ts +195 -240
  469. package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
  470. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
  471. package/src/messaging/providers/callback-routing.ts +1 -0
  472. package/src/messaging/providers/discord/api.ts +312 -0
  473. package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
  474. package/src/messaging/providers/discord/render.test.ts +204 -0
  475. package/src/messaging/providers/discord/render.ts +195 -0
  476. package/src/messaging/providers/discord/send.test.ts +179 -0
  477. package/src/messaging/providers/discord/send.ts +212 -0
  478. package/src/messaging/providers/discord/transport.ts +81 -0
  479. package/src/messaging/providers/index.ts +3 -1
  480. package/src/messaging/providers/retry-policy.test.ts +247 -0
  481. package/src/messaging/providers/retry-policy.ts +206 -0
  482. package/src/messaging/providers/slack/mention-source.test.ts +285 -0
  483. package/src/messaging/providers/slack/mention-source.ts +287 -0
  484. package/src/messaging/providers/telegram-bot/api.ts +47 -130
  485. package/src/messaging/providers/whatsapp/api.ts +47 -136
  486. package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
  487. package/src/monitoring/__tests__/plugin-auto-update.test.ts +495 -0
  488. package/src/monitoring/db-integrity-sample.ts +6 -1
  489. package/src/monitoring/plugin-auto-update.ts +463 -0
  490. package/src/monitoring/resource-sampler.ts +21 -5
  491. package/src/monitoring/worker.ts +13 -0
  492. package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
  493. package/src/notifications/__tests__/broadcaster.test.ts +37 -1
  494. package/src/notifications/__tests__/copy-composer.test.ts +165 -0
  495. package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
  496. package/src/notifications/__tests__/notification-utils.test.ts +249 -0
  497. package/src/notifications/adapters/platform.ts +31 -2
  498. package/src/notifications/assistant-reply-producer.ts +79 -32
  499. package/src/notifications/broadcaster.ts +30 -10
  500. package/src/notifications/copy-composer.ts +69 -0
  501. package/src/notifications/emit-signal.ts +82 -15
  502. package/src/notifications/events-store.ts +10 -2
  503. package/src/notifications/home-feed-side-effect.ts +6 -3
  504. package/src/notifications/notification-utils.ts +173 -5
  505. package/src/notifications/signal.ts +13 -0
  506. package/src/oauth/AGENTS.md +4 -2
  507. package/src/oauth/byo-connection.ts +6 -1
  508. package/src/oauth/credential-token-resolver.ts +3 -10
  509. package/src/oauth/manual-token-connection.ts +29 -64
  510. package/src/oauth/manual-token-providers.ts +62 -0
  511. package/src/oauth/provider-serializer.ts +13 -0
  512. package/src/oauth/seed-providers.ts +62 -11
  513. package/src/onboarding/onboarding-events-store.ts +91 -1
  514. package/src/permissions/question-prompter.test.ts +4 -0
  515. package/src/permissions/question-prompter.ts +21 -4
  516. package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
  517. package/src/persistence/attachments-store.ts +15 -15
  518. package/src/persistence/conversation-attention-store.ts +30 -10
  519. package/src/persistence/conversation-bootstrap.ts +17 -0
  520. package/src/persistence/conversation-crud.ts +363 -97
  521. package/src/persistence/conversation-group-migration.ts +1 -1
  522. package/src/persistence/conversation-key-store.ts +38 -10
  523. package/src/persistence/conversation-lineage.ts +226 -0
  524. package/src/persistence/conversation-plugin-facade.ts +28 -3
  525. package/src/persistence/conversation-queries.ts +207 -33
  526. package/src/persistence/conversation-types.ts +107 -1
  527. package/src/persistence/jobs-store.ts +32 -8
  528. package/src/persistence/llm-usage-store.ts +22 -0
  529. package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
  530. package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
  531. package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
  532. package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
  533. package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
  534. package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
  535. package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
  536. package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
  537. package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
  538. package/src/persistence/schema/conversations.ts +32 -0
  539. package/src/persistence/schema/infrastructure.ts +3 -0
  540. package/src/persistence/steps.ts +30 -0
  541. package/src/persistence/subagent-store.ts +148 -6
  542. package/src/platform/managed-speech.test.ts +20 -0
  543. package/src/platform/managed-speech.ts +9 -0
  544. package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
  545. package/src/plugin-api/index.ts +28 -5
  546. package/src/plugin-api/webhook-url.ts +151 -0
  547. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
  548. package/src/plugins/__tests__/mcp-servers.test.ts +370 -0
  549. package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
  550. package/src/plugins/collect-source-versions.ts +4 -27
  551. package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
  552. package/src/plugins/defaults/exploration-drift/package.json +1 -1
  553. package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
  554. package/src/plugins/defaults/image-recovery/recover.ts +67 -57
  555. package/src/plugins/defaults/index.ts +1 -1
  556. package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
  557. package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
  558. package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
  559. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
  560. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
  561. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
  562. package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
  563. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
  564. package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
  565. package/src/plugins/defaults/memory/job-handlers.ts +87 -5
  566. package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
  567. package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
  568. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
  569. package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
  570. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
  571. package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
  572. package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
  573. package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
  574. package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
  575. package/src/plugins/defaults/memory/v2/router.ts +8 -0
  576. package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
  577. package/src/plugins/external-plugin-loader.ts +64 -8
  578. package/src/plugins/installed-plugin-dirs.ts +101 -0
  579. package/src/plugins/mcp-servers.ts +343 -0
  580. package/src/plugins/mtime-cache.ts +34 -22
  581. package/src/plugins/types.ts +69 -8
  582. package/src/prompts/persona-resolver.ts +6 -2
  583. package/src/prompts/system-prompt.ts +72 -17
  584. package/src/providers/__tests__/inference.test.ts +40 -0
  585. package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
  586. package/src/providers/__tests__/retry-callsite.test.ts +72 -2
  587. package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
  588. package/src/providers/anthropic/client.ts +112 -95
  589. package/src/providers/connection-resolution.ts +35 -1
  590. package/src/providers/gemini/client.ts +5 -5
  591. package/src/providers/inference/adapter-factory.ts +16 -10
  592. package/src/providers/inference/auth.ts +16 -0
  593. package/src/providers/inference/connection-availability.ts +164 -12
  594. package/src/providers/inference/connections.ts +25 -1
  595. package/src/providers/media-resolve.ts +21 -11
  596. package/src/providers/model-catalog.ts +15 -0
  597. package/src/providers/openai/chat-completions-provider.ts +4 -4
  598. package/src/providers/openai/responses-provider.ts +12 -25
  599. package/src/providers/registry.ts +9 -4
  600. package/src/providers/retry.ts +25 -0
  601. package/src/providers/server-tool-pairing.ts +134 -0
  602. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
  603. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +620 -0
  604. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
  605. package/src/providers/speech-to-text/__tests__/resolve.test.ts +472 -12
  606. package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
  607. package/src/providers/speech-to-text/deepgram-flux-realtime.ts +719 -0
  608. package/src/providers/speech-to-text/deepgram-realtime.test.ts +283 -1
  609. package/src/providers/speech-to-text/deepgram-realtime.ts +117 -3
  610. package/src/providers/speech-to-text/deepgram.ts +1 -0
  611. package/src/providers/speech-to-text/provider-catalog.ts +137 -8
  612. package/src/providers/speech-to-text/resolve.ts +105 -12
  613. package/src/providers/speech-to-text/vellum-managed.ts +7 -1
  614. package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
  615. package/src/providers/types.ts +26 -6
  616. package/src/providers/vellum-model-routing.ts +6 -7
  617. package/src/runtime/AGENTS.md +2 -0
  618. package/src/runtime/__tests__/agent-wake.test.ts +16 -16
  619. package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
  620. package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
  621. package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts +104 -0
  622. package/src/runtime/agent-wake.ts +63 -24
  623. package/src/runtime/assistant-event-hub.ts +110 -12
  624. package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
  625. package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
  626. package/src/runtime/auth/same-actor.ts +56 -8
  627. package/src/runtime/auth/scopes.ts +12 -0
  628. package/src/runtime/auth/types.ts +2 -0
  629. package/src/runtime/background-job-runner.ts +5 -27
  630. package/src/runtime/channel-readiness-service.ts +59 -3
  631. package/src/runtime/channel-readiness-types.ts +18 -0
  632. package/src/runtime/desktop-presence.ts +69 -0
  633. package/src/runtime/http-server.ts +12 -0
  634. package/src/runtime/local-actor-identity.ts +29 -14
  635. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
  636. package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
  637. package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
  638. package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
  639. package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
  640. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
  641. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
  642. package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
  643. package/src/runtime/routes/__tests__/plugins-routes.test.ts +217 -1
  644. package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
  645. package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
  646. package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
  647. package/src/runtime/routes/__tests__/sse-actor-principal-heal.test.ts +165 -0
  648. package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
  649. package/src/runtime/routes/__tests__/surface-action-routes.test.ts +2 -0
  650. package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
  651. package/src/runtime/routes/archive-utils.ts +16 -12
  652. package/src/runtime/routes/attachment-routes.ts +13 -6
  653. package/src/runtime/routes/canned-reply-release.ts +54 -0
  654. package/src/runtime/routes/channel-availability-routes.ts +32 -14
  655. package/src/runtime/routes/channel-verification-routes.ts +2 -27
  656. package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
  657. package/src/runtime/routes/client-routes.ts +114 -2
  658. package/src/runtime/routes/conversation-cli-routes.ts +10 -12
  659. package/src/runtime/routes/conversation-list-routes.ts +59 -10
  660. package/src/runtime/routes/conversation-management-routes.ts +47 -18
  661. package/src/runtime/routes/conversation-query-routes.ts +32 -53
  662. package/src/runtime/routes/conversation-routes.ts +360 -293
  663. package/src/runtime/routes/credential-prompt-routes.ts +20 -2
  664. package/src/runtime/routes/debug-routes.ts +4 -1
  665. package/src/runtime/routes/default-provider-routes.ts +7 -11
  666. package/src/runtime/routes/documents-routes.ts +44 -1
  667. package/src/runtime/routes/events-routes.ts +29 -1
  668. package/src/runtime/routes/host-app-control-routes.ts +2 -0
  669. package/src/runtime/routes/host-bash-routes.ts +2 -0
  670. package/src/runtime/routes/host-browser-routes.ts +2 -0
  671. package/src/runtime/routes/host-cu-routes.ts +2 -0
  672. package/src/runtime/routes/host-file-routes.ts +2 -0
  673. package/src/runtime/routes/host-transfer-routes.ts +4 -0
  674. package/src/runtime/routes/identity-routes.ts +6 -6
  675. package/src/runtime/routes/inbound-message-handler.ts +5 -5
  676. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
  677. package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
  678. package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
  679. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
  680. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
  681. package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
  682. package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
  683. package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
  684. package/src/runtime/routes/inference-profiles-routes.ts +152 -73
  685. package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
  686. package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
  687. package/src/runtime/routes/log-export-routes.ts +1 -1
  688. package/src/runtime/routes/mcp-auth-routes.ts +141 -57
  689. package/src/runtime/routes/notification-routes.ts +0 -14
  690. package/src/runtime/routes/oauth-providers.ts +1 -0
  691. package/src/runtime/routes/plugins-routes.ts +127 -4
  692. package/src/runtime/routes/profiler-routes.ts +4 -2
  693. package/src/runtime/routes/question-routes.ts +28 -10
  694. package/src/runtime/routes/retrospective-routes.ts +9 -3
  695. package/src/runtime/routes/schedule-routes.ts +210 -12
  696. package/src/runtime/routes/secret-routes.ts +40 -23
  697. package/src/runtime/routes/settings-routes.ts +41 -13
  698. package/src/runtime/routes/skills-routes.ts +79 -0
  699. package/src/runtime/routes/sse-actor-principal-heal.ts +113 -0
  700. package/src/runtime/routes/stt-routes.ts +31 -25
  701. package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
  702. package/src/runtime/routes/wake-conversation-routes.ts +23 -0
  703. package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
  704. package/src/runtime/services/conversation-serializer.ts +24 -9
  705. package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
  706. package/src/runtime/sync/resource-sync-events.ts +79 -1
  707. package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
  708. package/src/runtime/sync/worker-daemon-notify.ts +44 -6
  709. package/src/runtime/verification-outbound-actions.ts +250 -222
  710. package/src/runtime/verification-templates.ts +41 -30
  711. package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
  712. package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
  713. package/src/schedule/inference-profile.ts +64 -3
  714. package/src/schedule/plugin-schedule-declarations.ts +518 -0
  715. package/src/schedule/plugin-schedule-reconciler.ts +605 -0
  716. package/src/schedule/plugin-schedules-gate.ts +23 -0
  717. package/src/schedule/schedule-store.ts +520 -26
  718. package/src/schedule/scheduler.ts +38 -0
  719. package/src/security/credential-backend.ts +51 -4
  720. package/src/security/encrypted-store.ts +64 -42
  721. package/src/security/oauth2.ts +6 -22
  722. package/src/security/secure-keys.ts +55 -7
  723. package/src/skills/catalog-install.ts +20 -7
  724. package/src/skills/inline-command-runner.ts +2 -7
  725. package/src/skills/skill-history.test.ts +269 -0
  726. package/src/skills/skill-history.ts +250 -0
  727. package/src/skills/skillssh-registry.ts +1 -1
  728. package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
  729. package/src/stt/__tests__/language-metadata.test.ts +85 -0
  730. package/src/stt/__tests__/speech-energy.test.ts +79 -0
  731. package/src/stt/__tests__/types.test.ts +94 -0
  732. package/src/stt/daemon-batch-transcriber.ts +18 -6
  733. package/src/stt/language-metadata.ts +65 -0
  734. package/src/stt/speech-energy.ts +115 -12
  735. package/src/stt/stt-stream-session.ts +8 -4
  736. package/src/stt/types.ts +119 -0
  737. package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
  738. package/src/subagent/consult-prompt.ts +3 -1
  739. package/src/subagent/consult-transcript.ts +1 -1
  740. package/src/subagent/manager.ts +304 -44
  741. package/src/subagent/progress-events.ts +35 -0
  742. package/src/subagent/role-resolution.ts +88 -0
  743. package/src/subagent/types.ts +303 -67
  744. package/src/subagent/validate-allowlists.ts +3 -4
  745. package/src/telegram/__tests__/webhook-health.test.ts +73 -1
  746. package/src/telegram/webhook-health.ts +101 -3
  747. package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
  748. package/src/telemetry/activation-funnel.ts +6 -1
  749. package/src/telemetry/live-voice-funnel.ts +75 -0
  750. package/src/telemetry/telemetry-event-sources.ts +6 -0
  751. package/src/telemetry/telemetry-wire-source.json +1 -1
  752. package/src/telemetry/telemetry-wire.generated.ts +2 -0
  753. package/src/telemetry/types.ts +23 -4
  754. package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
  755. package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
  756. package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
  757. package/src/tools/ask-question/ask-question-tool.ts +56 -2
  758. package/src/tools/browser/browser-execution.ts +1 -1
  759. package/src/tools/calls/call-start.ts +8 -5
  760. package/src/tools/conversation-groups/move_to_group.ts +10 -12
  761. package/src/tools/document/document-tool.ts +11 -4
  762. package/src/tools/execution-timeout.ts +9 -2
  763. package/src/tools/executor.ts +7 -6
  764. package/src/tools/filesystem/edit.ts +1 -1
  765. package/src/tools/filesystem/list.ts +1 -1
  766. package/src/tools/filesystem/read.ts +7 -13
  767. package/src/tools/filesystem/search.ts +23 -21
  768. package/src/tools/filesystem/write.ts +4 -1
  769. package/src/tools/host-filesystem/edit.ts +7 -2
  770. package/src/tools/host-filesystem/read.ts +21 -8
  771. package/src/tools/host-filesystem/write.ts +10 -2
  772. package/src/tools/host-terminal/host-shell.ts +2 -2
  773. package/src/tools/mcp/mcp-tool-factory.ts +25 -1
  774. package/src/tools/permission-checker.ts +8 -3
  775. package/src/tools/schedule/delete.ts +13 -1
  776. package/src/tools/schedule/list.ts +14 -3
  777. package/src/tools/schedule/update.ts +56 -33
  778. package/src/tools/shared/filesystem/audio-read.ts +6 -4
  779. package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
  780. package/src/tools/shared/filesystem/image-read.ts +12 -12
  781. package/src/tools/shared/filesystem/path-policy.ts +70 -5
  782. package/src/tools/shared/filesystem/size-guard.ts +6 -6
  783. package/src/tools/shared/input-misuse.ts +72 -0
  784. package/src/tools/skills/scaffold-managed.ts +82 -0
  785. package/src/tools/skills/skill-tool-factory.ts +14 -1
  786. package/src/tools/subagent/consult-deadline.ts +12 -7
  787. package/src/tools/subagent/message.ts +3 -1
  788. package/src/tools/subagent/read.ts +66 -4
  789. package/src/tools/subagent/spawn.ts +500 -78
  790. package/src/tools/tool-approval-handler.ts +45 -4
  791. package/src/tools/types.ts +22 -3
  792. package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
  793. package/src/tools/ui-surface/visual-validation.ts +2 -0
  794. package/src/tts/__tests__/provider-adapters.test.ts +147 -0
  795. package/src/tts/__tests__/speakable-segments.test.ts +470 -0
  796. package/src/tts/language-voices.ts +23 -0
  797. package/src/tts/providers/deepgram-provider.ts +3 -1
  798. package/src/tts/providers/elevenlabs-provider.ts +73 -1
  799. package/src/tts/providers/xai-provider.ts +28 -2
  800. package/src/tts/speakable-segments.ts +293 -23
  801. package/src/tts/synthesis-stream.ts +7 -0
  802. package/src/tts/types.ts +7 -0
  803. package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
  804. package/src/usage/subagent-attribution.ts +108 -0
  805. package/src/util/__tests__/language-subtag.test.ts +54 -0
  806. package/src/util/__tests__/sqlite-retry.test.ts +112 -0
  807. package/src/util/abort-reasons.ts +16 -0
  808. package/src/util/ansi.ts +44 -0
  809. package/src/util/disk-usage.ts +17 -13
  810. package/src/util/image-conversion.ts +20 -13
  811. package/src/util/language-subtag.ts +43 -0
  812. package/src/util/schedule-source-key.ts +23 -0
  813. package/src/util/unicode.ts +1 -1
  814. package/src/util/worker-process.ts +7 -1
  815. package/src/workspace/git-service.ts +33 -9
  816. package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
  817. package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
  818. package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
  819. package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +116 -0
  820. package/src/workspace/migrations/registry.ts +6 -0
  821. package/src/workspace/top-level-renderer.ts +15 -2
  822. package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
  823. package/src/daemon/install-symlink.ts +0 -207
  824. package/src/live-voice/__tests__/front-decision.test.ts +0 -525
  825. package/src/live-voice/front-decision.ts +0 -429
  826. package/src/notifications/deferred-emit.ts +0 -147
@@ -3,6 +3,8 @@ import { describe, expect, test } from "bun:test";
3
3
  import {
4
4
  DEFAULT_SPEECH_ENERGY_THRESHOLD,
5
5
  detectPcm16SpeechActivity,
6
+ pcm16MaxNormalizedCorrelation,
7
+ pcm16MeanAmplitude,
6
8
  } from "../speech-energy.js";
7
9
 
8
10
  /** Build a PCM16LE buffer from an array of sample values. */
@@ -66,3 +68,80 @@ describe("detectPcm16SpeechActivity", () => {
66
68
  expect(detectPcm16SpeechActivity(quiet, 500)).toBe(false);
67
69
  });
68
70
  });
71
+
72
+ describe("pcm16MeanAmplitude", () => {
73
+ test("returns 0 for an empty buffer", () => {
74
+ expect(pcm16MeanAmplitude(Buffer.alloc(0))).toBe(0);
75
+ });
76
+
77
+ test("returns the exact mean absolute amplitude", () => {
78
+ expect(pcm16MeanAmplitude(pcm16([1_000, -2_000, 3_000, -4_000]))).toBe(
79
+ 2_500,
80
+ );
81
+ });
82
+
83
+ test("ignores a trailing odd byte", () => {
84
+ const samples = pcm16([3_000, -3_000]);
85
+ expect(
86
+ pcm16MeanAmplitude(Buffer.concat([samples, Buffer.from([0x01])])),
87
+ ).toBe(3_000);
88
+ });
89
+
90
+ test("matches the detector's threshold comparison", () => {
91
+ const chunks = [
92
+ Buffer.alloc(0),
93
+ pcm16([0, 0]),
94
+ pcm16([500, -500]),
95
+ pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD]),
96
+ pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD + 1]),
97
+ ];
98
+ for (const chunk of chunks) {
99
+ expect(detectPcm16SpeechActivity(chunk)).toBe(
100
+ pcm16MeanAmplitude(chunk) > DEFAULT_SPEECH_ENERGY_THRESHOLD,
101
+ );
102
+ }
103
+ });
104
+ });
105
+
106
+ describe("pcm16MaxNormalizedCorrelation", () => {
107
+ const wave = (frequency: number, count: number): Buffer =>
108
+ pcm16(
109
+ Array.from({ length: count }, (_, index) =>
110
+ Math.round(
111
+ 8_000 * Math.sin((2 * Math.PI * frequency * index) / 16_000),
112
+ ),
113
+ ),
114
+ );
115
+
116
+ test("finds a matching waveform at an arbitrary reference offset", () => {
117
+ const input = wave(240, 800);
118
+ const reference = Buffer.concat([wave(410, 400), input, wave(610, 400)]);
119
+
120
+ expect(pcm16MaxNormalizedCorrelation(input, reference)).toBeGreaterThan(
121
+ 0.99,
122
+ );
123
+ });
124
+
125
+ test("is invariant to gain and polarity", () => {
126
+ const inputSamples = Array.from({ length: 800 }, (_, index) =>
127
+ Math.round(5_000 * Math.sin((2 * Math.PI * index) / 83)),
128
+ );
129
+ const inverted = pcm16(inputSamples.map((sample) => -2 * sample));
130
+
131
+ expect(
132
+ pcm16MaxNormalizedCorrelation(pcm16(inputSamples), inverted),
133
+ ).toBeGreaterThan(0.99);
134
+ });
135
+
136
+ test("rejects an unrelated waveform and flat power", () => {
137
+ expect(
138
+ pcm16MaxNormalizedCorrelation(wave(240, 800), wave(610, 1_600)),
139
+ ).toBeLessThan(0.3);
140
+ expect(
141
+ pcm16MaxNormalizedCorrelation(
142
+ pcm16(new Array(800).fill(3_000)),
143
+ pcm16(new Array(1_600).fill(3_000)),
144
+ ),
145
+ ).toBe(0);
146
+ });
147
+ });
@@ -2,10 +2,14 @@ import { describe, expect, test } from "bun:test";
2
2
 
3
3
  import type {
4
4
  SttStreamServerClosedEvent,
5
+ SttStreamServerEagerTurnEndEvent,
5
6
  SttStreamServerErrorEvent,
6
7
  SttStreamServerEvent,
7
8
  SttStreamServerFinalEvent,
8
9
  SttStreamServerPartialEvent,
10
+ SttStreamServerTurnEndEvent,
11
+ SttStreamServerTurnResumedEvent,
12
+ SttStreamServerTurnStartEvent,
9
13
  } from "../types.js";
10
14
 
11
15
  // ---------------------------------------------------------------------------
@@ -87,3 +91,93 @@ describe("SttStreamServerEvent types", () => {
87
91
  expect(_label).toBeUndefined();
88
92
  });
89
93
  });
94
+
95
+ describe("turn-detection server events", () => {
96
+ test("turn-start event compiles and round-trips", () => {
97
+ const event: SttStreamServerTurnStartEvent = { type: "turn-start" };
98
+ const asUnion: SttStreamServerEvent = event;
99
+ expect(asUnion.type).toBe("turn-start");
100
+ });
101
+
102
+ test("eager-turn-end event carries the speculated transcript", () => {
103
+ const event: SttStreamServerEagerTurnEndEvent = {
104
+ type: "eager-turn-end",
105
+ text: "is this the end",
106
+ };
107
+ const asUnion: SttStreamServerEvent = event;
108
+ expect(asUnion.type).toBe("eager-turn-end");
109
+ expect(event.text).toBe("is this the end");
110
+ });
111
+
112
+ test("turn-resumed event compiles and round-trips", () => {
113
+ const event: SttStreamServerTurnResumedEvent = { type: "turn-resumed" };
114
+ const asUnion: SttStreamServerEvent = event;
115
+ expect(asUnion.type).toBe("turn-resumed");
116
+ });
117
+
118
+ test("turn-end event carries the committed transcript and optional confidence", () => {
119
+ const withoutConfidence: SttStreamServerTurnEndEvent = {
120
+ type: "turn-end",
121
+ text: "done speaking",
122
+ };
123
+ expect(withoutConfidence.confidence).toBeUndefined();
124
+
125
+ const event: SttStreamServerTurnEndEvent = {
126
+ type: "turn-end",
127
+ text: "done speaking",
128
+ confidence: 0.82,
129
+ };
130
+ const asUnion: SttStreamServerEvent = event;
131
+ expect(asUnion.type).toBe("turn-end");
132
+ expect(event.text).toBe("done speaking");
133
+ expect(event.confidence).toBe(0.82);
134
+ });
135
+
136
+ test("a consumer switch over every variant is exhaustive", () => {
137
+ // Fails to compile if a variant is added to SttStreamServerEvent
138
+ // without a case here, which is what forces consumers to be revisited.
139
+ const label = (event: SttStreamServerEvent): string => {
140
+ switch (event.type) {
141
+ case "partial":
142
+ case "final":
143
+ case "finalized":
144
+ case "error":
145
+ case "closed":
146
+ return "transcript";
147
+ case "turn-start":
148
+ case "eager-turn-end":
149
+ case "turn-resumed":
150
+ case "turn-end":
151
+ return "turn";
152
+ default: {
153
+ const _exhaustive: never = event;
154
+ return _exhaustive;
155
+ }
156
+ }
157
+ };
158
+
159
+ const events: SttStreamServerEvent[] = [
160
+ { type: "partial", text: "a" },
161
+ { type: "final", text: "a" },
162
+ { type: "finalized" },
163
+ { type: "error", category: "provider-error", message: "boom" },
164
+ { type: "closed" },
165
+ { type: "turn-start" },
166
+ { type: "eager-turn-end", text: "a" },
167
+ { type: "turn-resumed" },
168
+ { type: "turn-end", text: "a" },
169
+ ];
170
+
171
+ expect(events.map(label)).toEqual([
172
+ "transcript",
173
+ "transcript",
174
+ "transcript",
175
+ "transcript",
176
+ "transcript",
177
+ "turn",
178
+ "turn",
179
+ "turn",
180
+ "turn",
181
+ ]);
182
+ });
183
+ });
@@ -13,6 +13,7 @@
13
13
  * - xAI (`xai`)
14
14
  */
15
15
 
16
+ import { batchBoundaryGapReason } from "../providers/speech-to-text/provider-catalog.js";
16
17
  import type {
17
18
  BatchTranscriber,
18
19
  SttProviderId,
@@ -214,6 +215,8 @@ class VellumManagedBatchTranscriber implements BatchTranscriber {
214
215
  readonly providerId = "vellum" as const;
215
216
  readonly boundaryId = "daemon-batch" as const;
216
217
 
218
+ constructor(private readonly language: string | undefined) {}
219
+
217
220
  async transcribe(
218
221
  request: SttTranscribeRequest,
219
222
  ): Promise<SttTranscribeResult> {
@@ -223,6 +226,7 @@ class VellumManagedBatchTranscriber implements BatchTranscriber {
223
226
  request.audio,
224
227
  request.mimeType,
225
228
  request.signal,
229
+ this.language,
226
230
  );
227
231
  }
228
232
  }
@@ -237,12 +241,14 @@ class VellumManagedBatchTranscriber implements BatchTranscriber {
237
241
  * is unavailable.
238
242
  *
239
243
  * `language` is the spoken language, forwarded to providers whose batch API
240
- * accepts one: Deepgram (where `"multi"` also pins nova-3) and xAI (where
244
+ * accepts one: Deepgram (where `"multi"` also pins nova-3), xAI (where
241
245
  * `"multi"` is dropped because it is a Deepgram-specific value, not a
242
- * language code). Whisper and Gemini auto-detect natively and take no
243
- * language parameter, so it is silently ignored for them, as is the vellum
244
- * managed path (the platform speech proxy accepts no language parameter;
245
- * see the deferral note in `resolveBatchTranscriber`).
246
+ * language code), and the vellum managed path, whose platform proxy passes
247
+ * it to Deepgram server-side. Whisper and Gemini auto-detect natively and
248
+ * take no language parameter, so it is silently ignored for them.
249
+ *
250
+ * Throws an {@link SttError} for a provider with no batch endpoint at all,
251
+ * which no key can fix and `null` would understate.
246
252
  */
247
253
  export function createDaemonBatchTranscriber(
248
254
  apiKey: string | null | undefined,
@@ -251,7 +257,7 @@ export function createDaemonBatchTranscriber(
251
257
  ): BatchTranscriber | null {
252
258
  // vellum authenticates via the platform connection, not an API key.
253
259
  if (providerId === "vellum") {
254
- return new VellumManagedBatchTranscriber();
260
+ return new VellumManagedBatchTranscriber(language);
255
261
  }
256
262
  if (!apiKey) {
257
263
  return null;
@@ -266,6 +272,12 @@ export function createDaemonBatchTranscriber(
266
272
  return new GoogleGeminiBatchTranscriber(apiKey);
267
273
  case "xai":
268
274
  return new XAIBatchTranscriber(apiKey, language);
275
+ case "deepgram-flux":
276
+ // Same copy the resolver raises, so a direct factory caller and a
277
+ // config-driven one report the mismatch identically.
278
+ throw new SttError("provider-error", batchBoundaryGapReason(providerId), {
279
+ userFacing: true,
280
+ });
269
281
  default: {
270
282
  // Exhaustive check — compile error if a new SttProviderId is added
271
283
  // without a corresponding case here.
@@ -0,0 +1,65 @@
1
+ /**
2
+ * Pure helpers for tallying and ranking detected-language tags emitted
3
+ * by streaming STT providers (e.g. Deepgram nova-3 `multi` per-word tags).
4
+ *
5
+ * Tag normalization is `baseLanguageSubtag` (util/language-subtag.ts);
6
+ * these helpers layer vote-counting on top of it so provider adapters and
7
+ * event consumers share one notion of a dominant language.
8
+ */
9
+
10
+ import { baseLanguageSubtag } from "../util/language-subtag.js";
11
+
12
+ /**
13
+ * Cast one vote into a language tally for a transcript chunk's dominant
14
+ * detected language. `languages` is dominance-ranked, so only the first
15
+ * entry votes: secondary tags at full weight would let a minority
16
+ * language outvote the dominant one. Blank and absent tags cast no vote;
17
+ * regional variants count toward their base subtag.
18
+ */
19
+ export function voteDominantLanguage(
20
+ tally: Map<string, number>,
21
+ languages: readonly string[] | undefined,
22
+ ): void {
23
+ const dominant = baseLanguageSubtag(languages?.[0]);
24
+ if (dominant !== undefined) {
25
+ tally.set(dominant, (tally.get(dominant) ?? 0) + 1);
26
+ }
27
+ }
28
+
29
+ /**
30
+ * The dominant tag in a language tally: most votes wins, ties break by
31
+ * first insertion (Map iteration is insertion order). Undefined for an
32
+ * empty tally.
33
+ */
34
+ export function dominantLanguageTag(
35
+ tally: ReadonlyMap<string, number>,
36
+ ): string | undefined {
37
+ let dominant: string | undefined;
38
+ let dominantCount = 0;
39
+ for (const [tag, count] of tally) {
40
+ if (count > dominantCount) {
41
+ dominant = tag;
42
+ dominantCount = count;
43
+ }
44
+ }
45
+ return dominant;
46
+ }
47
+
48
+ /**
49
+ * Tally normalized language tags and return them most-frequent-first.
50
+ * Ties are broken by first appearance in the input. Blank tags are
51
+ * skipped; regional variants count toward their base subtag.
52
+ */
53
+ export function rankLanguages(tags: Iterable<string>): string[] {
54
+ // Map iteration order is insertion order, so a stable sort by count
55
+ // leaves tied tags in first-appearance order.
56
+ const counts = new Map<string, number>();
57
+ for (const tag of tags) {
58
+ const normalized = baseLanguageSubtag(tag);
59
+ if (normalized === undefined) {
60
+ continue;
61
+ }
62
+ counts.set(normalized, (counts.get(normalized) ?? 0) + 1);
63
+ }
64
+ return [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([tag]) => tag);
65
+ }
@@ -18,6 +18,120 @@
18
18
  */
19
19
  export const DEFAULT_SPEECH_ENERGY_THRESHOLD = 800;
20
20
 
21
+ /**
22
+ * Return the mean absolute amplitude of little-endian signed PCM16 audio.
23
+ * Empty buffers return 0, and a trailing odd byte is ignored.
24
+ */
25
+ export function pcm16MeanAmplitude(chunk: Buffer): number {
26
+ const sampleCount = Math.floor(chunk.length / 2);
27
+ if (sampleCount === 0) {
28
+ return 0;
29
+ }
30
+
31
+ let totalAmplitude = 0;
32
+ for (let i = 0; i < sampleCount; i += 1) {
33
+ totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
34
+ }
35
+ return totalAmplitude / sampleCount;
36
+ }
37
+
38
+ /**
39
+ * Find the strongest normalized correlation between a PCM16 input window and
40
+ * any same-length window in a PCM16 reference. The samples are block-averaged
41
+ * before matching so this stays cheap enough for live audio and remains
42
+ * tolerant of the low-pass filtering introduced by speakers and microphones.
43
+ *
44
+ * The absolute coefficient makes polarity inversion harmless. A flat input or
45
+ * reference has no identifying waveform and returns 0 instead of being treated
46
+ * as a match based on level alone.
47
+ */
48
+ export function pcm16MaxNormalizedCorrelation(
49
+ input: Buffer,
50
+ reference: Buffer,
51
+ downsampleFactor = 8,
52
+ ): number {
53
+ if (!Number.isInteger(downsampleFactor) || downsampleFactor <= 0) {
54
+ return 0;
55
+ }
56
+
57
+ let inputSamples = blockAveragePcm16(input, downsampleFactor);
58
+ const referenceSamples = blockAveragePcm16(reference, downsampleFactor);
59
+ if (inputSamples.length > referenceSamples.length) {
60
+ inputSamples = inputSamples.subarray(0, referenceSamples.length);
61
+ }
62
+ if (inputSamples.length < 2) {
63
+ return 0;
64
+ }
65
+
66
+ let inputSum = 0;
67
+ for (let index = 0; index < inputSamples.length; index += 1) {
68
+ inputSum += inputSamples[index]!;
69
+ }
70
+ const inputMean = inputSum / inputSamples.length;
71
+ const centeredInput = new Float64Array(inputSamples.length);
72
+ let inputEnergy = 0;
73
+ for (let index = 0; index < inputSamples.length; index += 1) {
74
+ const centered = inputSamples[index]! - inputMean;
75
+ centeredInput[index] = centered;
76
+ inputEnergy += centered * centered;
77
+ }
78
+ if (inputEnergy === 0) {
79
+ return 0;
80
+ }
81
+
82
+ const prefixSum = new Float64Array(referenceSamples.length + 1);
83
+ const prefixSquareSum = new Float64Array(referenceSamples.length + 1);
84
+ for (let index = 0; index < referenceSamples.length; index += 1) {
85
+ const sample = referenceSamples[index]!;
86
+ prefixSum[index + 1] = prefixSum[index]! + sample;
87
+ prefixSquareSum[index + 1] = prefixSquareSum[index]! + sample * sample;
88
+ }
89
+
90
+ let best = 0;
91
+ const windowLength = inputSamples.length;
92
+ for (
93
+ let offset = 0;
94
+ offset + windowLength <= referenceSamples.length;
95
+ offset += 1
96
+ ) {
97
+ const referenceSum = prefixSum[offset + windowLength]! - prefixSum[offset]!;
98
+ const referenceSquareSum =
99
+ prefixSquareSum[offset + windowLength]! - prefixSquareSum[offset]!;
100
+ const referenceEnergy =
101
+ referenceSquareSum - (referenceSum * referenceSum) / windowLength;
102
+ if (referenceEnergy <= 0) {
103
+ continue;
104
+ }
105
+
106
+ let dotProduct = 0;
107
+ for (let index = 0; index < windowLength; index += 1) {
108
+ dotProduct += centeredInput[index]! * referenceSamples[offset + index]!;
109
+ }
110
+ const correlation =
111
+ Math.abs(dotProduct) / Math.sqrt(inputEnergy * referenceEnergy);
112
+ best = Math.max(best, Math.min(correlation, 1));
113
+ }
114
+ return best;
115
+ }
116
+
117
+ function blockAveragePcm16(
118
+ chunk: Buffer,
119
+ downsampleFactor: number,
120
+ ): Float64Array {
121
+ const sampleCount = Math.floor(chunk.length / 2);
122
+ const blockCount = Math.floor(sampleCount / downsampleFactor);
123
+ const result = new Float64Array(blockCount);
124
+ for (let block = 0; block < blockCount; block += 1) {
125
+ let sum = 0;
126
+ const firstSample = block * downsampleFactor;
127
+ for (let offset = 0; offset < downsampleFactor; offset += 1) {
128
+ sum += chunk.readInt16LE((firstSample + offset) * 2);
129
+ }
130
+ result[block] = sum / downsampleFactor;
131
+ }
132
+ return result;
133
+ }
134
+
21
135
  /**
22
136
  * Detect speech activity in a chunk of little-endian signed 16-bit mono
23
137
  * PCM samples.
@@ -34,16 +148,5 @@ export function detectPcm16SpeechActivity(
34
148
  chunk: Buffer,
35
149
  threshold = DEFAULT_SPEECH_ENERGY_THRESHOLD,
36
150
  ): boolean {
37
- const sampleCount = Math.floor(chunk.length / 2);
38
- if (sampleCount === 0) {
39
- return false;
40
- }
41
-
42
- let totalAmplitude = 0;
43
- for (let i = 0; i < sampleCount; i++) {
44
- totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
45
- }
46
- const avgAmplitude = totalAmplitude / sampleCount;
47
-
48
- return avgAmplitude > threshold;
151
+ return pcm16MeanAmplitude(chunk) > threshold;
49
152
  }
@@ -5,8 +5,11 @@
5
5
  * a client WebSocket connection and a provider-specific streaming adapter.
6
6
  * The orchestrator accepts client events (`audio`, `stop`), routes audio
7
7
  * frames to the resolved {@link StreamingTranscriber}, and emits normalized
8
- * server events (`partial`, `final`, `error`, `closed`) back over the same
9
- * WebSocket connection with per-session ordering guarantees.
8
+ * server events back over the same WebSocket connection with per-session
9
+ * ordering guarantees: transcripts (`partial`, `final`, `finalized`),
10
+ * turn boundaries from providers that detect them (`turn-start`,
11
+ * `eager-turn-end`, `turn-resumed`, `turn-end`), and termination
12
+ * (`error`, `closed`).
10
13
  *
11
14
  * Session lifecycle:
12
15
  * 1. Client opens a WebSocket to `/v1/stt/stream` with required `mimeType`
@@ -17,8 +20,9 @@
17
20
  * `resolveStreamingTranscriber()` and starts the provider session.
18
21
  * 3. The client sends `audio` frames (binary or base64-encoded JSON) and
19
22
  * a `stop` event when recording is complete.
20
- * 4. The provider emits `partial` and `final` transcript events which the
21
- * orchestrator forwards to the client as JSON frames.
23
+ * 4. The provider emits transcript events, plus turn-boundary events when it
24
+ * detects turns, and the orchestrator forwards each to the client as a
25
+ * JSON frame.
22
26
  * 5. The session closes deterministically on client disconnect, `stop`
23
27
  * event, idle timeout, or runtime shutdown.
24
28
  *
package/src/stt/types.ts CHANGED
@@ -23,6 +23,11 @@
23
23
  export type SttProviderId =
24
24
  | "openai-whisper"
25
25
  | "deepgram"
26
+ /**
27
+ * Deepgram's Flux conversational speech model. Shares the `deepgram`
28
+ * credential and is streaming-only: Flux has no batch endpoint.
29
+ */
30
+ | "deepgram-flux"
26
31
  | "google-gemini"
27
32
  | "xai"
28
33
  | "vellum";
@@ -65,6 +70,26 @@ export type ConversationStreamingMode =
65
70
  | "incremental-batch"
66
71
  | "none";
67
72
 
73
+ /**
74
+ * Turn-detection support mode.
75
+ *
76
+ * Describes whether a provider decides end-of-turn itself, in-band with the
77
+ * audio it is already transcribing, rather than leaving the boundary to the
78
+ * session's local silence timer and front-door hold verdict.
79
+ *
80
+ * - `"provider"`: the provider emits turn lifecycle events (`turn-start`,
81
+ * `turn-end`) on its transcript stream, and a live-voice session may let
82
+ * those events commit the turn.
83
+ * - `"none"`: the provider transcribes only; the local silence boundary owns
84
+ * end-of-turn.
85
+ *
86
+ * A provider declaring `"provider"` should number its turns: the staleness
87
+ * check prefers the turn index carried on the event, and degrades to the
88
+ * session's local VAD generation counter when one is absent, which understates
89
+ * the recorded latency for a turn the caller resumed and stopped again.
90
+ */
91
+ export type SttTurnDetectionMode = "provider" | "none";
92
+
68
93
  // ---------------------------------------------------------------------------
69
94
  // Boundary identifier
70
95
  // ---------------------------------------------------------------------------
@@ -245,6 +270,10 @@ export type SttStreamServerEvent =
245
270
  | SttStreamServerPartialEvent
246
271
  | SttStreamServerFinalEvent
247
272
  | SttStreamServerFinalizedEvent
273
+ | SttStreamServerTurnStartEvent
274
+ | SttStreamServerEagerTurnEndEvent
275
+ | SttStreamServerTurnResumedEvent
276
+ | SttStreamServerTurnEndEvent
248
277
  | SttStreamServerErrorEvent
249
278
  | SttStreamServerClosedEvent;
250
279
 
@@ -263,6 +292,14 @@ export interface SttStreamServerPartialEvent {
263
292
  * provider does not surface confidence on interim chunks.
264
293
  */
265
294
  readonly confidence?: number;
295
+ /**
296
+ * All detected languages of this chunk in dominance order (first entry
297
+ * dominant), already normalized to lowercase base subtags (e.g. "en",
298
+ * "hi") by the emitting adapter; consumers must not re-normalize. Only
299
+ * providers with code-switching metadata (currently Deepgram nova-3
300
+ * `multi`) populate this. Absence means "unknown", never English.
301
+ */
302
+ readonly languages?: readonly string[];
266
303
  }
267
304
 
268
305
  /**
@@ -279,6 +316,14 @@ export interface SttStreamServerFinalEvent {
279
316
  * provider does not surface confidence on this chunk.
280
317
  */
281
318
  readonly confidence?: number;
319
+ /**
320
+ * All detected languages of this chunk in dominance order (first entry
321
+ * dominant), already normalized to lowercase base subtags (e.g. "en",
322
+ * "hi") by the emitting adapter; consumers must not re-normalize. Only
323
+ * providers with code-switching metadata (currently Deepgram nova-3
324
+ * `multi`) populate this. Absence means "unknown", never English.
325
+ */
326
+ readonly languages?: readonly string[];
282
327
  /**
283
328
  * True when this final is the flush response to
284
329
  * {@link StreamingTranscriber.finalizeUtterance} — i.e. it commits audio
@@ -298,6 +343,80 @@ export interface SttStreamServerFinalizedEvent {
298
343
  readonly type: "finalized";
299
344
  }
300
345
 
346
+ // ---------------------------------------------------------------------------
347
+ // Turn-detection events
348
+ //
349
+ // Consumers that ignore these keep committing transcripts from `final`
350
+ // exactly as they do without a turn-detecting provider.
351
+ // ---------------------------------------------------------------------------
352
+
353
+ /**
354
+ * The provider's turn model detected the start of a user turn.
355
+ *
356
+ * Emitted only by providers with model-integrated turn detection; providers
357
+ * without it never emit these.
358
+ */
359
+ export interface SttStreamServerTurnStartEvent {
360
+ readonly type: "turn-start";
361
+ /**
362
+ * Provider's index for the turn being opened, monotonically increasing
363
+ * within a stream. Undefined when the provider does not number its turns.
364
+ */
365
+ readonly turnIndex?: number;
366
+ }
367
+
368
+ /**
369
+ * The provider's turn model speculates the user has finished, ahead of its
370
+ * committed end-of-turn decision. A {@link SttStreamServerTurnResumedEvent}
371
+ * retracts the speculation; a {@link SttStreamServerTurnEndEvent} confirms it.
372
+ *
373
+ * Emitted only by providers with model-integrated turn detection; providers
374
+ * without it never emit these.
375
+ */
376
+ export interface SttStreamServerEagerTurnEndEvent {
377
+ readonly type: "eager-turn-end";
378
+ /** Transcript text as of the speculated turn end. */
379
+ readonly text: string;
380
+ }
381
+
382
+ /**
383
+ * The user kept speaking after an {@link SttStreamServerEagerTurnEndEvent},
384
+ * retracting the speculated turn end.
385
+ *
386
+ * Emitted only by providers with model-integrated turn detection; providers
387
+ * without it never emit these.
388
+ */
389
+ export interface SttStreamServerTurnResumedEvent {
390
+ readonly type: "turn-resumed";
391
+ }
392
+
393
+ /**
394
+ * The provider's turn model committed an end of turn: the user has finished
395
+ * speaking and a reply can be dispatched.
396
+ *
397
+ * Emitted only by providers with model-integrated turn detection; providers
398
+ * without it never emit these.
399
+ */
400
+ export interface SttStreamServerTurnEndEvent {
401
+ readonly type: "turn-end";
402
+ /** Committed transcript text for the completed turn. */
403
+ readonly text: string;
404
+ /**
405
+ * Provider-emitted end-of-turn confidence in [0, 1]. Undefined when the
406
+ * provider does not surface a score.
407
+ */
408
+ readonly confidence?: number;
409
+ /**
410
+ * Provider's index for the turn being closed, matching the
411
+ * {@link SttStreamServerTurnStartEvent} that opened it. A consumer can
412
+ * compare it against the newest turn it has seen opened to tell an
413
+ * end-of-turn for the turn still in progress from one the provider has
414
+ * already superseded. Undefined when the provider does not number its
415
+ * turns.
416
+ */
417
+ readonly turnIndex?: number;
418
+ }
419
+
301
420
  /** An error occurred during streaming transcription. */
302
421
  export interface SttStreamServerErrorEvent {
303
422
  readonly type: "error";
@@ -20,6 +20,31 @@ describe("buildAdvisorSystem", () => {
20
20
  expect(prompt).not.toContain("<agent_system_prompt>");
21
21
  });
22
22
 
23
+ test("tells the advisor it has read-only tools for verifying decisive facts", () => {
24
+ const prompt = buildAdvisorSystem(null);
25
+ expect(prompt).toContain("read-only tools");
26
+ expect(prompt).toContain("read files");
27
+ expect(prompt).toContain("verification, not exploration");
28
+ expect(prompt).toContain("You cannot change anything");
29
+ });
30
+
31
+ test("does not offer the advisor a memory or conversation search", () => {
32
+ // The advisor's read tools stop at the workspace. Naming `recall` here
33
+ // would advertise a search the role allowlist does not grant, and would
34
+ // contradict the scope the consult framing promises the user.
35
+ const prompt = buildAdvisorSystem(null);
36
+ expect(prompt).not.toContain("recall");
37
+ expect(prompt).toContain("you cannot see other conversations");
38
+ });
39
+
40
+ test("does not claim the advisor is tool-less", () => {
41
+ // The advisor can open a file to check a fact; a prompt that says otherwise
42
+ // suppresses the read it was given tools for.
43
+ const prompt = buildAdvisorSystem(null);
44
+ expect(prompt).not.toContain("You have no tools");
45
+ expect(prompt).not.toContain("cannot search, read files, or run commands");
46
+ });
47
+
23
48
  test("keeps the situational context pack out of the system prompt", () => {
24
49
  // System Prompt Minimalism: the pack rides in the request turn instead.
25
50
  expect(buildAdvisorSystem("parent")).not.toContain("<agent_environment>");