@vellumai/assistant 0.11.6 → 0.11.7-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (478) hide show
  1. package/ARCHITECTURE.md +31 -19
  2. package/Dockerfile +1 -0
  3. package/docs/architecture/security.md +3 -3
  4. package/docs/flux-turn-detection-spike.md +8 -7
  5. package/docs/runbook-trusted-contacts.md +14 -0
  6. package/docs/stt-provider-onboarding.md +8 -4
  7. package/knip.json +1 -0
  8. package/node_modules/@vellumai/avatar-manifest/package.json +18 -0
  9. package/node_modules/@vellumai/avatar-manifest/src/__tests__/manifest.test.ts +173 -0
  10. package/node_modules/@vellumai/avatar-manifest/src/__tests__/read.test.ts +114 -0
  11. package/node_modules/@vellumai/avatar-manifest/src/index.ts +29 -0
  12. package/node_modules/@vellumai/avatar-manifest/src/layout.ts +18 -0
  13. package/node_modules/@vellumai/avatar-manifest/src/manifest.ts +156 -0
  14. package/node_modules/@vellumai/avatar-manifest/src/read.ts +109 -0
  15. package/node_modules/@vellumai/avatar-manifest/tsconfig.json +20 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
  19. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
  20. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
  21. package/node_modules/@vellumai/ces-client/src/http-credentials.ts +161 -0
  22. package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
  23. package/node_modules/@vellumai/environments/package.json +2 -1
  24. package/node_modules/@vellumai/environments/src/shell.test.ts +56 -0
  25. package/node_modules/@vellumai/environments/src/shell.ts +65 -0
  26. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
  27. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
  28. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
  29. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
  30. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
  31. package/node_modules/@vellumai/gateway-client/src/__tests__/contact-read-contracts.test.ts +32 -0
  32. package/node_modules/@vellumai/gateway-client/src/__tests__/gateway-client.test.ts +1 -0
  33. package/node_modules/@vellumai/gateway-client/src/__tests__/inbound-event-kind.test.ts +82 -0
  34. package/node_modules/@vellumai/gateway-client/src/__tests__/plugin-admission-denied-contract.test.ts +56 -0
  35. package/node_modules/@vellumai/gateway-client/src/__tests__/trust-verdict-contract.test.ts +18 -0
  36. package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +5 -11
  37. package/node_modules/@vellumai/gateway-client/src/channel-permission-contract.ts +3 -1
  38. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +9 -0
  39. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +4 -0
  40. package/node_modules/@vellumai/gateway-client/src/inbound-event-kind.ts +75 -0
  41. package/node_modules/@vellumai/gateway-client/src/index.ts +17 -0
  42. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +8 -0
  43. package/node_modules/@vellumai/gateway-client/src/plugin-admission-denied-contract.ts +47 -0
  44. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +11 -0
  45. package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
  46. package/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
  47. package/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
  48. package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
  49. package/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
  50. package/openapi.yaml +419 -55
  51. package/package.json +4 -1
  52. package/scripts/sync-llm-catalog.ts +6 -0
  53. package/src/__tests__/approval-card-edit-inbound-handoff.test.ts +199 -0
  54. package/src/__tests__/approval-card-edit.test.ts +153 -0
  55. package/src/__tests__/attachments-store-heic-normalize.test.ts +1 -1
  56. package/src/__tests__/available-skills.test.ts +54 -0
  57. package/src/__tests__/avatar-identity-sync.test.ts +5 -1
  58. package/src/__tests__/catalog-install-normalize.test.ts +9 -1
  59. package/src/__tests__/channel-policy.test.ts +1 -3
  60. package/src/__tests__/channel-readiness-discord.test.ts +9 -70
  61. package/src/__tests__/compactor-call-site-logging.test.ts +61 -0
  62. package/src/__tests__/config-loader-backfill.test.ts +69 -0
  63. package/src/__tests__/config-schema.test.ts +2 -3
  64. package/src/__tests__/conversation-agent-loop.test.ts +238 -0
  65. package/src/__tests__/conversation-attachments.test.ts +0 -1
  66. package/src/__tests__/conversation-error.test.ts +44 -0
  67. package/src/__tests__/conversation-evictor.test.ts +19 -0
  68. package/src/__tests__/conversation-fork-referential.test.ts +73 -63
  69. package/src/__tests__/conversation-fork-retrospective.test.ts +99 -74
  70. package/src/__tests__/conversation-runtime-assembly.test.ts +23 -0
  71. package/src/__tests__/conversation-slash.test.ts +9 -0
  72. package/src/__tests__/conversation-title-service.test.ts +97 -0
  73. package/src/__tests__/conversation-tool-setup-attribution.test.ts +16 -0
  74. package/src/__tests__/credential-broker-server-use.test.ts +0 -218
  75. package/src/__tests__/credential-broker.test.ts +43 -420
  76. package/src/__tests__/credential-record-write-through.test.ts +78 -0
  77. package/src/__tests__/credential-routes.test.ts +17 -1
  78. package/src/__tests__/credential-security-invariants.test.ts +8 -9
  79. package/src/__tests__/default-profile-catalog-fallback.test.ts +289 -0
  80. package/src/__tests__/delete-propagation.test.ts +92 -2
  81. package/src/__tests__/discord-callback-round-trip.test.ts +93 -0
  82. package/src/__tests__/discord-channel-config.test.ts +237 -0
  83. package/src/__tests__/edit-propagation.test.ts +42 -2
  84. package/src/__tests__/evict-conversations-for-reload.test.ts +51 -1
  85. package/src/__tests__/fallback-breaker.test.ts +1335 -0
  86. package/src/__tests__/gateway-threshold-reader-mock.ts +6 -0
  87. package/src/__tests__/guardian-card-withdrawal.test.ts +30 -0
  88. package/src/__tests__/handlers-user-message-approval-consumption.test.ts +0 -2
  89. package/src/__tests__/image-conversion.test.ts +38 -4
  90. package/src/__tests__/install-skill-routing.test.ts +25 -0
  91. package/src/__tests__/llm-catalog-parity.test.ts +4 -0
  92. package/src/__tests__/llm-resolver.test.ts +74 -0
  93. package/src/__tests__/llm-schema-fallback-profile.test.ts +359 -0
  94. package/src/__tests__/managed-fallback-dispatch.test.ts +430 -0
  95. package/src/__tests__/managed-profile-guard.test.ts +28 -0
  96. package/src/__tests__/managed-speech-defaults.test.ts +196 -1
  97. package/src/__tests__/mcp-cli.test.ts +23 -1
  98. package/src/__tests__/mcp-tool-annotations-risk.test.ts +43 -34
  99. package/src/__tests__/media-stream-stt-session.test.ts +2 -0
  100. package/src/__tests__/notification-decision-fallback.test.ts +106 -0
  101. package/src/__tests__/notification-discord-adapter.test.ts +162 -0
  102. package/src/__tests__/oauth-connect-orchestrator.test.ts +95 -0
  103. package/src/__tests__/persist-unsendable-image-downscale.test.ts +6 -8
  104. package/src/__tests__/persist-unsendable-image.test.ts +3 -3
  105. package/src/__tests__/platform.test.ts +19 -1
  106. package/src/__tests__/plugin-import-boundary-guard.test.ts +5 -1
  107. package/src/__tests__/provider-catalog-visibility.test.ts +17 -0
  108. package/src/__tests__/provider-platform-proxy-integration.test.ts +8 -1
  109. package/src/__tests__/provider-usage-tracking.test.ts +56 -0
  110. package/src/__tests__/require-fresh-approval.test.ts +5 -2
  111. package/src/__tests__/retry-fallback-escalation.test.ts +1093 -0
  112. package/src/__tests__/schedule-retry.test.ts +183 -4
  113. package/src/__tests__/scheduler-reuse-conversation.test.ts +6 -6
  114. package/src/__tests__/secret-routes-platform-proxy.test.ts +55 -0
  115. package/src/__tests__/skill-load-tool.test.ts +34 -0
  116. package/src/__tests__/skill-tool-factory.test.ts +6 -1
  117. package/src/__tests__/skill-tool-manifest.test.ts +20 -0
  118. package/src/__tests__/skills.test.ts +25 -0
  119. package/src/__tests__/subagent-manager-notify.test.ts +38 -0
  120. package/src/__tests__/terminal-tools.test.ts +2 -0
  121. package/src/__tests__/tool-approval-handler.test.ts +146 -2
  122. package/src/__tests__/tool-approval-seed-content-blocks.test.ts +108 -2
  123. package/src/__tests__/tool-audit.test.ts +111 -40
  124. package/src/__tests__/tool-executor-lifecycle-events.test.ts +228 -18
  125. package/src/__tests__/ui-shape-teaching.test.ts +1 -1
  126. package/src/__tests__/verification-control-plane-policy.test.ts +6 -2
  127. package/src/__tests__/voice-config-update.test.ts +43 -0
  128. package/src/__tests__/workspace-migration-147-rename-colliding-backup-profile-names.test.ts +480 -0
  129. package/src/__tests__/workspace-migration-148-strip-unsupported-fallback-profiles.test.ts +167 -0
  130. package/src/__tests__/workspace-migration-149-repoint-backup-profile-selections.test.ts +144 -0
  131. package/src/agent/loop.ts +31 -2
  132. package/src/approvals/guardian-card-withdrawal.ts +22 -0
  133. package/src/avatar/__tests__/ensure-raster.test.ts +291 -0
  134. package/src/avatar/avatar-manifest.ts +48 -100
  135. package/src/avatar/avatar-store.ts +7 -7
  136. package/src/avatar/ensure-raster.ts +146 -0
  137. package/src/avatar/resvg-lazy.test.ts +0 -1
  138. package/src/avatar/traits-png-sync.ts +9 -10
  139. package/src/background-wake/wake-intent-hooks.test.ts +0 -1
  140. package/src/backup/__tests__/paths.test.ts +6 -2
  141. package/src/backup/paths.ts +6 -4
  142. package/src/bundler/app-compiler.ts +2 -2
  143. package/src/bundler/compiler-tools.test.ts +36 -0
  144. package/src/bundler/compiler-tools.ts +54 -18
  145. package/src/calls/__tests__/telephony-synthesis-language.test.ts +81 -0
  146. package/src/calls/__tests__/voice-session-bridge.test.ts +57 -0
  147. package/src/calls/media-stream-stt-session.ts +2 -1
  148. package/src/calls/telephony-synthesis-language.ts +13 -5
  149. package/src/calls/voice-session-bridge.ts +19 -4
  150. package/src/channels/config.ts +23 -12
  151. package/src/cli/AGENTS.md +3 -1
  152. package/src/cli/commands/backup.help.ts +10 -13
  153. package/src/cli/commands/contacts.help.ts +16 -4
  154. package/src/cli/commands/contacts.ts +5 -1
  155. package/src/cli/commands/credentials.help.ts +1 -1
  156. package/src/cli/commands/inference-providers.ts +18 -53
  157. package/src/cli/commands/mcp.help.ts +6 -5
  158. package/src/cli/commands/mcp.ts +1 -1
  159. package/src/cli/commands/memory/memory-retrospective.ts +1 -0
  160. package/src/cli/commands/stt.help.ts +2 -2
  161. package/src/config/__tests__/backup-schema.test.ts +1 -1
  162. package/src/config/__tests__/default-profile-catalog.test.ts +83 -2
  163. package/src/config/__tests__/loader-sparse-array-cleanup.test.ts +1 -1
  164. package/src/config/__tests__/memory-retrospective-schema.test.ts +12 -15
  165. package/src/config/bundled-skills/computer-use/SKILL.md +6 -6
  166. package/src/config/bundled-skills/computer-use/TOOLS.json +9 -6
  167. package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +3 -1
  168. package/src/config/bundled-skills/media-processing/services/preprocess.ts +1 -1
  169. package/src/config/bundled-skills/phone-calls/SKILL.md +1 -1
  170. package/src/config/bundled-skills/phone-calls/references/CONFIG.md +16 -15
  171. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +2 -1
  172. package/src/config/bundled-skills/settings/TOOLS.json +11 -5
  173. package/src/config/bundled-skills/settings/tools/open-system-settings.test.ts +85 -0
  174. package/src/config/bundled-skills/settings/tools/open-system-settings.ts +33 -8
  175. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +54 -3
  176. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +1 -1
  177. package/src/config/default-profile-catalog.ts +195 -10
  178. package/src/config/default-profile-names.ts +41 -0
  179. package/src/config/feature-flag-registry.json +34 -2
  180. package/src/config/inference-profile-validation.ts +11 -2
  181. package/src/config/llm-resolver.ts +25 -0
  182. package/src/config/loader.ts +3 -0
  183. package/src/config/managed-speech-defaults.ts +160 -10
  184. package/src/config/schema.ts +2 -0
  185. package/src/config/schemas/__tests__/live-voice.test.ts +7 -6
  186. package/src/config/schemas/__tests__/stt.test.ts +94 -0
  187. package/src/config/schemas/backup.ts +1 -1
  188. package/src/config/schemas/channels.ts +32 -0
  189. package/src/config/schemas/live-voice.ts +13 -4
  190. package/src/config/schemas/llm.ts +263 -15
  191. package/src/config/schemas/mcp.ts +10 -2
  192. package/src/config/schemas/memory-retrospective.ts +1 -19
  193. package/src/config/schemas/services.ts +24 -0
  194. package/src/config/schemas/stt.ts +128 -5
  195. package/src/config/seed-inference-profiles.ts +32 -3
  196. package/src/config/skill-state.ts +5 -0
  197. package/src/config/skills.ts +21 -0
  198. package/src/contacts/notify-contacts-changed.ts +5 -2
  199. package/src/context/compactor.ts +74 -1
  200. package/src/daemon/__tests__/conversation-tool-setup.test.ts +138 -1
  201. package/src/daemon/config-watcher.ts +2 -1
  202. package/src/daemon/conversation-agent-loop-handlers.ts +27 -0
  203. package/src/daemon/conversation-agent-loop.ts +86 -10
  204. package/src/daemon/conversation-error.ts +25 -0
  205. package/src/daemon/conversation-evictor.ts +2 -4
  206. package/src/daemon/conversation-runtime-assembly.ts +19 -7
  207. package/src/daemon/conversation-skill-tools.ts +5 -1
  208. package/src/daemon/conversation-slash.ts +2 -2
  209. package/src/daemon/conversation-store.ts +8 -1
  210. package/src/daemon/conversation-tool-setup.ts +37 -0
  211. package/src/daemon/conversation.ts +0 -4
  212. package/src/daemon/handlers/channel-config-result.ts +30 -0
  213. package/src/daemon/handlers/config-discord-channel.ts +301 -0
  214. package/src/daemon/handlers/config-ingress.ts +9 -0
  215. package/src/daemon/handlers/config-slack-channel.ts +13 -15
  216. package/src/daemon/handlers/config-telegram.ts +2 -5
  217. package/src/daemon/handlers/conversations.ts +0 -2
  218. package/src/daemon/handlers/skills.ts +25 -7
  219. package/src/daemon/lifecycle.ts +3 -1
  220. package/src/daemon/message-types/messages.ts +0 -2
  221. package/src/daemon/tool-setup-types.ts +5 -3
  222. package/src/daemon/trust-context-types.test.ts +33 -0
  223. package/src/heartbeat/__tests__/heartbeat-service.test.ts +0 -1
  224. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +58 -4
  225. package/src/live-voice/__tests__/live-voice-managed-speech-fallback.test.ts +8 -3
  226. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +73 -11
  227. package/src/live-voice/__tests__/live-voice-session-preflight.test.ts +148 -2
  228. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +64 -0
  229. package/src/live-voice/__tests__/live-voice-stt.test.ts +3 -0
  230. package/src/live-voice/__tests__/live-voice-text-turn.test.ts +391 -0
  231. package/src/live-voice/__tests__/protocol.test.ts +224 -0
  232. package/src/live-voice/live-voice-credential-preflight.ts +5 -1
  233. package/src/live-voice/live-voice-session-manager.ts +39 -2
  234. package/src/live-voice/live-voice-session.ts +224 -9
  235. package/src/live-voice/protocol.ts +187 -1
  236. package/src/messaging/providers/discord/api.ts +2 -1
  237. package/src/messaging/providers/discord/withdraw.ts +62 -0
  238. package/src/messaging/read-provider-metadata.ts +32 -0
  239. package/src/monitoring/__tests__/file-descriptors.test.ts +32 -0
  240. package/src/monitoring/__tests__/process-memory.test.ts +34 -1
  241. package/src/monitoring/db-integrity-sample.ts +1 -0
  242. package/src/monitoring/file-descriptors.ts +39 -6
  243. package/src/monitoring/process-memory.ts +37 -5
  244. package/src/monitoring/resource-sampler.ts +25 -8
  245. package/src/notifications/__tests__/activity-failed-dedupe.test.ts +161 -0
  246. package/src/notifications/__tests__/background-failure-signal.test.ts +107 -0
  247. package/src/notifications/__tests__/connected-channels.test.ts +12 -0
  248. package/src/notifications/__tests__/copy-composer.test.ts +83 -0
  249. package/src/notifications/__tests__/destination-resolver.test.ts +38 -0
  250. package/src/notifications/activity-failed-dedupe.ts +95 -0
  251. package/src/notifications/adapters/discord.ts +132 -0
  252. package/src/notifications/adapters/shared.ts +16 -0
  253. package/src/notifications/adapters/slack.ts +6 -1
  254. package/src/notifications/adapters/telegram.ts +5 -8
  255. package/src/notifications/approval-card-data.ts +94 -1
  256. package/src/notifications/background-failure-signal.ts +116 -0
  257. package/src/notifications/broadcaster.ts +3 -9
  258. package/src/notifications/copy-composer.ts +41 -8
  259. package/src/notifications/decision-engine.ts +55 -0
  260. package/src/notifications/destination-resolver.ts +45 -5
  261. package/src/notifications/emit-signal.ts +19 -5
  262. package/src/notifications/guardian-question-mode.ts +57 -0
  263. package/src/notifications/types.ts +2 -8
  264. package/src/oauth/__tests__/seed-providers-managed.test.ts +18 -0
  265. package/src/oauth/connect-orchestrator.ts +16 -0
  266. package/src/oauth/seed-providers.ts +262 -0
  267. package/src/permissions/checker.test.ts +21 -0
  268. package/src/permissions/checker.ts +6 -0
  269. package/src/permissions/gateway-threshold-reader.test.ts +184 -0
  270. package/src/permissions/gateway-threshold-reader.ts +144 -10
  271. package/src/permissions/prompter.ts +0 -6
  272. package/src/permissions/types.ts +5 -4
  273. package/src/persistence/__tests__/bulk-write-gate.test.ts +0 -38
  274. package/src/persistence/bulk-write-gate.ts +9 -10
  275. package/src/persistence/conversation-crud.ts +88 -134
  276. package/src/persistence/conversation-title-service.ts +43 -29
  277. package/src/persistence/embeddings/embedding-local.ts +14 -85
  278. package/src/persistence/embeddings/embedding-runtime-manager.ts +1 -0
  279. package/src/persistence/lifecycle-events-store.test.ts +50 -0
  280. package/src/persistence/lifecycle-events-store.ts +75 -5
  281. package/src/platform/platform-patch-queue.test.ts +368 -0
  282. package/src/platform/platform-patch-queue.ts +190 -0
  283. package/src/platform/sync-avatar.test.ts +527 -0
  284. package/src/platform/sync-avatar.ts +209 -0
  285. package/src/platform/sync-identity.test.ts +75 -0
  286. package/src/platform/sync-identity.ts +17 -110
  287. package/src/plugin-api/model-profiles.test.ts +2 -2
  288. package/src/plugin-api/model-profiles.ts +7 -7
  289. package/src/plugin-api/transcription-session.ts +1 -1
  290. package/src/plugins/defaults/compaction/window-manager.ts +4 -0
  291. package/src/plugins/defaults/image-recovery/recover.ts +2 -2
  292. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +133 -34
  293. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +21 -31
  294. package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +13 -9
  295. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +31 -4
  296. package/src/plugins/defaults/memory/memory-retrospective-constants.ts +0 -11
  297. package/src/plugins/defaults/memory/memory-retrospective-fork-boundary.ts +27 -13
  298. package/src/plugins/defaults/memory/memory-retrospective-job.ts +183 -113
  299. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +13 -13
  300. package/src/plugins/defaults/memory/src/memory-v2-routes.ts +2 -2
  301. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +1 -1
  302. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +4 -2
  303. package/src/plugins/defaults/memory/substrate/sweep-job.ts +9 -42
  304. package/src/plugins/defaults/memory/v2/rerank-local.ts +1 -0
  305. package/src/plugins/defaults/memory/worker.ts +7 -0
  306. package/src/providers/__tests__/provider-secret-catalog.test.ts +1 -0
  307. package/src/providers/__tests__/retry-callsite.test.ts +55 -0
  308. package/src/providers/__tests__/vellum-connection-routing.test.ts +21 -3
  309. package/src/providers/connection-resolution.ts +2 -17
  310. package/src/providers/fallback-breaker.ts +389 -0
  311. package/src/providers/inference/__tests__/adapter-factory-opencode.test.ts +92 -0
  312. package/src/providers/inference/__tests__/base-url-route-validation.test.ts +16 -0
  313. package/src/providers/inference/__tests__/base-url-security.test.ts +12 -0
  314. package/src/providers/inference/adapter-factory.ts +246 -3
  315. package/src/providers/inference/auth.ts +15 -14
  316. package/src/providers/inference/chatgpt-subscription-auth.ts +70 -0
  317. package/src/providers/inference/chatgpt-subscription-credentials.ts +84 -0
  318. package/src/providers/inference/resolve-auth.ts +1 -1
  319. package/src/providers/media-resolve.ts +2 -3
  320. package/src/providers/model-catalog.ts +100 -0
  321. package/src/providers/openai/__tests__/api-error-normalization.test.ts +22 -0
  322. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +198 -0
  323. package/src/providers/openai/api-error-normalization.ts +13 -0
  324. package/src/providers/openai/chat-completions-provider.ts +84 -0
  325. package/src/providers/opencode/client.test.ts +150 -0
  326. package/src/providers/opencode/client.ts +61 -0
  327. package/src/providers/platform-proxy/constants.ts +5 -0
  328. package/src/providers/provider-resolvability.ts +20 -0
  329. package/src/providers/provider-secret-catalog.ts +3 -2
  330. package/src/providers/retry.ts +956 -68
  331. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +42 -0
  332. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +136 -0
  333. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +120 -0
  334. package/src/providers/speech-to-text/__tests__/resolve.test.ts +316 -12
  335. package/src/providers/speech-to-text/deepgram-flux-frames.ts +83 -3
  336. package/src/providers/speech-to-text/deepgram-flux-realtime.ts +132 -4
  337. package/src/providers/speech-to-text/provider-catalog.ts +191 -3
  338. package/src/providers/speech-to-text/resolve.ts +132 -24
  339. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +296 -0
  340. package/src/providers/types.ts +39 -0
  341. package/src/providers/usage-tracking.ts +12 -1
  342. package/src/providers/vellum/client.ts +29 -0
  343. package/src/providers/vellum-model-routing.test.ts +2 -0
  344. package/src/providers/vellum-model-routing.ts +3 -3
  345. package/src/runtime/AGENTS.md +1 -2
  346. package/src/runtime/__tests__/agent-wake.test.ts +105 -3
  347. package/src/runtime/__tests__/background-job-runner.test.ts +89 -3
  348. package/src/runtime/agent-wake.ts +44 -5
  349. package/src/runtime/background-job-runner.ts +68 -50
  350. package/src/runtime/channel-readiness-service.ts +0 -39
  351. package/src/runtime/http-server.ts +12 -4
  352. package/src/runtime/routes/__tests__/avatar-state-routes.test.ts +18 -18
  353. package/src/runtime/routes/__tests__/backup-routes.test.ts +9 -5
  354. package/src/runtime/routes/__tests__/chatgpt-subscription-auth-routes.test.ts +372 -0
  355. package/src/runtime/routes/__tests__/contact-routes.test.ts +7 -0
  356. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +96 -0
  357. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +37 -0
  358. package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +2 -1
  359. package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +87 -0
  360. package/src/runtime/routes/acp-claude-auth-routes.ts +25 -54
  361. package/src/runtime/routes/approval-routes.ts +0 -12
  362. package/src/runtime/routes/avatar-routes.ts +17 -30
  363. package/src/runtime/routes/backup-routes.ts +2 -2
  364. package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +296 -113
  365. package/src/runtime/routes/contact-prompt-routes.ts +65 -6
  366. package/src/runtime/routes/contact-routes.ts +20 -15
  367. package/src/runtime/routes/conversation-query-routes.ts +98 -10
  368. package/src/runtime/routes/conversation-routes.ts +2 -2
  369. package/src/runtime/routes/credential-routes.ts +2 -1
  370. package/src/runtime/routes/guardian-approval-interception.ts +12 -9
  371. package/src/runtime/routes/inbound-message-handler.ts +47 -48
  372. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +18 -13
  373. package/src/runtime/routes/inbound-stages/admission-policy.ts +2 -2
  374. package/src/runtime/routes/inbound-stages/edit-intercept.ts +20 -2
  375. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +9 -8
  376. package/src/runtime/routes/inbound-stages/transcribe-audio.ts +1 -1
  377. package/src/runtime/routes/index.ts +2 -0
  378. package/src/runtime/routes/inference-profile-session-handler.ts +2 -2
  379. package/src/runtime/routes/inference-profiles-routes.ts +12 -7
  380. package/src/runtime/routes/inference-provider-connection-routes.ts +2 -2
  381. package/src/runtime/routes/inference-send-routes.ts +2 -2
  382. package/src/runtime/routes/ingress-status-routes.ts +2 -2
  383. package/src/runtime/routes/integrations/discord.ts +119 -0
  384. package/src/runtime/routes/llm-call-sites-routes.ts +2 -2
  385. package/src/runtime/routes/mcp-auth-routes.ts +12 -6
  386. package/src/runtime/routes/migration-routes.ts +3 -1
  387. package/src/runtime/routes/oauth-pending-flows.ts +136 -0
  388. package/src/runtime/routes/secret-routes.ts +7 -0
  389. package/src/runtime/routes/stt-routes.ts +38 -6
  390. package/src/runtime/routes/wake-conversation-routes.ts +8 -1
  391. package/src/runtime/routes/watch-routes.ts +9 -5
  392. package/src/runtime/sync/resource-sync-events.ts +2 -0
  393. package/src/runtime/trust-verdict-consumer.ts +3 -3
  394. package/src/schedule/__tests__/worker-feature-flags.test.ts +58 -0
  395. package/src/schedule/retry-policy.ts +8 -8
  396. package/src/schedule/run-script.ts +14 -23
  397. package/src/schedule/schedule-recovery.ts +0 -1
  398. package/src/schedule/scheduler.ts +89 -51
  399. package/src/schedule/worker.ts +57 -0
  400. package/src/security/__tests__/openai-device-auth.test.ts +355 -0
  401. package/src/security/ces-rpc-record-backend.ts +123 -0
  402. package/src/security/oauth2.ts +7 -0
  403. package/src/security/openai-device-auth.ts +364 -0
  404. package/src/security/secure-keys.ts +15 -0
  405. package/src/skills/available-skills.ts +10 -1
  406. package/src/skills/catalog-install.ts +19 -1
  407. package/src/skills/inline-command-runner.ts +6 -2
  408. package/src/skills/platform-compatibility.test.ts +49 -0
  409. package/src/skills/platform-compatibility.ts +65 -0
  410. package/src/skills/tool-manifest.ts +20 -0
  411. package/src/stt/__tests__/roles.test.ts +152 -0
  412. package/src/stt/daemon-batch-transcriber.ts +4 -1
  413. package/src/stt/roles.ts +180 -0
  414. package/src/stt/stt-stream-session.ts +3 -2
  415. package/src/stt/types.ts +33 -3
  416. package/src/subagent/manager.ts +12 -0
  417. package/src/telemetry/__tests__/live-voice-funnel.test.ts +34 -0
  418. package/src/telemetry/live-voice-funnel.ts +12 -2
  419. package/src/telemetry/telemetry-wire.generated.ts +6 -0
  420. package/src/telemetry/tool-audit.ts +83 -30
  421. package/src/telemetry/tool-executed-events-store.test.ts +0 -1
  422. package/src/telemetry/turn-outcome.ts +75 -4
  423. package/src/tools/client-os.ts +115 -0
  424. package/src/tools/computer-use/definitions.ts +11 -7
  425. package/src/tools/computer-use/skill-proxy-bridge.ts +2 -2
  426. package/src/tools/credentials/broker-types.ts +0 -71
  427. package/src/tools/credentials/broker.ts +6 -201
  428. package/src/tools/credentials/metadata-store.ts +65 -16
  429. package/src/tools/credentials/store.ts +2 -0
  430. package/src/tools/executor.ts +2 -8
  431. package/src/tools/host-shell.test.ts +75 -0
  432. package/src/tools/host-shell.ts +24 -0
  433. package/src/tools/host-terminal/host-shell.ts +15 -44
  434. package/src/tools/mcp/mcp-tool-factory.ts +44 -9
  435. package/src/tools/permission-checker.ts +73 -21
  436. package/src/tools/policy-context.ts +1 -0
  437. package/src/tools/shared/filesystem/image-read.ts +1 -1
  438. package/src/tools/skills/find-similar-skills.ts +10 -2
  439. package/src/tools/skills/load.ts +41 -6
  440. package/src/tools/skills/sandbox-runner.ts +8 -16
  441. package/src/tools/skills/skill-tool-factory.ts +1 -0
  442. package/src/tools/terminal/__tests__/safe-env.test.ts +64 -1
  443. package/src/tools/terminal/safe-env.ts +39 -9
  444. package/src/tools/terminal/shell.ts +12 -31
  445. package/src/tools/tool-approval-handler.ts +64 -10
  446. package/src/tools/tool-types.ts +10 -0
  447. package/src/tools/types.ts +19 -3
  448. package/src/tools/ui-surface/surface-shape-docs.ts +4 -3
  449. package/src/tools/workflows/manage-workflows.ts +2 -2
  450. package/src/tools/workflows/run-workflow.test.ts +2 -1
  451. package/src/util/__tests__/file-use.test.ts +61 -0
  452. package/src/util/__tests__/process-table.test.ts +66 -0
  453. package/src/util/__tests__/process-tree.test.ts +46 -0
  454. package/src/util/errors.ts +1 -0
  455. package/src/util/file-use.ts +91 -0
  456. package/src/util/host-process.test.ts +17 -0
  457. package/src/util/host-process.ts +75 -0
  458. package/src/util/image-conversion.ts +39 -7
  459. package/src/util/platform.ts +32 -13
  460. package/src/util/process-table.ts +249 -0
  461. package/src/util/process-tree.ts +85 -129
  462. package/src/util/provider-error-patterns.ts +20 -0
  463. package/src/util/spawn.ts +9 -2
  464. package/src/watch/__tests__/watch-retro.test.ts +48 -1
  465. package/src/watch/watch-retro.ts +46 -4
  466. package/src/workspace/git-service.ts +4 -14
  467. package/src/workspace/migrations/147-rename-colliding-backup-profile-names.ts +360 -0
  468. package/src/workspace/migrations/148-strip-unsupported-fallback-profiles.ts +117 -0
  469. package/src/workspace/migrations/149-repoint-backup-profile-selections.ts +204 -0
  470. package/src/workspace/migrations/150-stt-flux-provider-to-model-family.ts +139 -0
  471. package/src/workspace/migrations/__tests__/150-stt-flux-provider-to-model-family.test.ts +160 -0
  472. package/src/workspace/migrations/registry.ts +8 -0
  473. package/src/__tests__/permission-types.test.ts +0 -17
  474. package/src/channels/gateway-discord-admission.ts +0 -32
  475. package/src/persistence/fork-message-copy.ts +0 -239
  476. package/src/plugins/defaults/memory/__tests__/fork-message-copy.test.ts +0 -221
  477. package/src/security/__tests__/oauth2-device-code.test.ts +0 -431
  478. package/src/security/oauth2-device-code.ts +0 -312
@@ -105,6 +105,22 @@ mock.module("../vellum-managed-realtime.js", () => ({
105
105
  },
106
106
  }));
107
107
 
108
+ /** Constructor calls for the managed Flux transcriber (`vellum-flux`). */
109
+ const vellumFluxStreamCtorCalls: Array<{
110
+ connection: unknown;
111
+ options: unknown;
112
+ }> = [];
113
+
114
+ mock.module("../vellum-managed-flux-realtime.js", () => ({
115
+ VellumManagedFluxRealtimeTranscriber: class {
116
+ readonly providerId = "vellum-flux";
117
+ readonly boundaryId = "daemon-streaming";
118
+ constructor(connection: unknown, options: unknown) {
119
+ vellumFluxStreamCtorCalls.push({ connection, options });
120
+ }
121
+ },
122
+ }));
123
+
108
124
  // -- Streaming adapter mocks ----------------------------------------------
109
125
 
110
126
  /**
@@ -206,6 +222,7 @@ const {
206
222
  function applyConfig(overrides: {
207
223
  provider?: string;
208
224
  language?: string;
225
+ roles?: Record<string, { provider: string; model?: string }>;
209
226
  }): void {
210
227
  const provider = overrides.provider ?? "openai-whisper";
211
228
  // Seed a schema-valid base so the loader caches a fresh config object, then
@@ -230,6 +247,13 @@ function applyConfig(overrides: {
230
247
  (getConfig().services.stt as { language?: string }).language =
231
248
  overrides.language;
232
249
  }
250
+ if (overrides.roles !== undefined) {
251
+ (
252
+ getConfig().services.stt as {
253
+ roles?: Record<string, { provider: string; model?: string }>;
254
+ }
255
+ ).roles = overrides.roles;
256
+ }
233
257
  }
234
258
 
235
259
  // ---------------------------------------------------------------------------
@@ -486,6 +510,58 @@ describe("resolveTelephonySttCapability", () => {
486
510
  expect(result.credentialProvider).toBe("gemini");
487
511
  }
488
512
  });
513
+
514
+ test("answers about the telephony role, not the global provider", async () => {
515
+ // The role's provider is the one a call dials, so preflight must judge
516
+ // that one: a credentialed role behind an uncredentialed global would
517
+ // otherwise be turned away from a call it can serve.
518
+ mockProviderKeys = { deepgram: "dg-key" };
519
+ applyConfig({
520
+ provider: "openai-whisper",
521
+ roles: { telephony: { provider: "deepgram" } },
522
+ });
523
+
524
+ const result = await resolveTelephonySttCapability();
525
+
526
+ expect(result.status).toBe("supported");
527
+ if (result.status === "supported") {
528
+ expect(result.providerId).toBe("deepgram");
529
+ expect(result.telephonyMode).toBe("realtime-ws");
530
+ }
531
+ });
532
+
533
+ test("reports the telephony role's missing credential, not the global's", async () => {
534
+ // The inverse: a credentialed global must not vouch for a role that
535
+ // cannot dial, which passes preflight and then fails at the dial.
536
+ mockProviderKeys = { openai: "oai-key" };
537
+ applyConfig({
538
+ provider: "openai-whisper",
539
+ roles: { telephony: { provider: "deepgram" } },
540
+ });
541
+
542
+ const result = await resolveTelephonySttCapability();
543
+
544
+ expect(result.status).toBe("missing-credentials");
545
+ if (result.status === "missing-credentials") {
546
+ expect(result.providerId).toBe("deepgram");
547
+ expect(result.credentialProvider).toBe("deepgram");
548
+ }
549
+ });
550
+
551
+ test("an unset telephony role still reads the global provider", async () => {
552
+ mockProviderKeys = { openai: "oai-key" };
553
+ applyConfig({
554
+ provider: "openai-whisper",
555
+ roles: { liveVoice: { provider: "deepgram" } },
556
+ });
557
+
558
+ const result = await resolveTelephonySttCapability();
559
+
560
+ expect(result.status).toBe("supported");
561
+ if (result.status === "supported") {
562
+ expect(result.providerId).toBe("openai-whisper");
563
+ }
564
+ });
489
565
  });
490
566
 
491
567
  // ---------------------------------------------------------------------------
@@ -511,11 +587,14 @@ describe("telephony capability catalog alignment", () => {
511
587
  /**
512
588
  * Providers that deliberately sit out telephony (`telephonyMode: "none"`).
513
589
  * `deepgram-flux` is a streaming-only spike, and telephony is out of its
514
- * scope. Nothing reroutes a call to another provider, so opting out here
515
- * means a Flux-configured assistant does not transcribe calls.
590
+ * scope. `vellum-flux` sits out for a concrete reason: the relay maps
591
+ * `Finalize` onto Flux's `CloseStream`, and utterance-boundary finals are a
592
+ * nova-3 concept. Nothing reroutes a call to another provider, so opting
593
+ * out here means a Flux-configured assistant does not transcribe calls.
516
594
  */
517
595
  const TELEPHONY_OPT_OUT: ReadonlySet<SttProviderId> = new Set([
518
596
  "deepgram-flux",
597
+ "vellum-flux",
519
598
  ]);
520
599
 
521
600
  test("deepgram catalog entry has realtime-ws telephonyMode", () => {
@@ -1046,6 +1125,90 @@ describe("resolveStreamingTranscriber diarize preference", () => {
1046
1125
  });
1047
1126
  });
1048
1127
 
1128
+ describe("vellum-flux managed resolution", () => {
1129
+ beforeEach(() => {
1130
+ mockVellumAvailable = false;
1131
+ mockProviderKeys = {};
1132
+ mockVelayConnection = {
1133
+ wsBaseUrl: "ws://gateway.test",
1134
+ httpBaseUrl: "http://gateway.test",
1135
+ mintServiceToken: () => "vk-test",
1136
+ };
1137
+ vellumFluxStreamCtorCalls.length = 0;
1138
+ });
1139
+
1140
+ test("streaming resolver constructs the managed Flux adapter over the same relay connection", async () => {
1141
+ mockVellumAvailable = true;
1142
+ applyConfig({ provider: "vellum-flux", language: "es" });
1143
+
1144
+ const transcriber = await resolveStreamingTranscriber({
1145
+ sampleRate: 16000,
1146
+ });
1147
+
1148
+ expect(transcriber?.providerId).toBe("vellum-flux");
1149
+ expect(vellumFluxStreamCtorCalls).toEqual([
1150
+ {
1151
+ connection: mockVelayConnection,
1152
+ // The relay picks flux-general-en vs flux-general-multi from the
1153
+ // language, so forwarding it is the only lever over the model.
1154
+ options: { sampleRate: 16000, language: "es" },
1155
+ },
1156
+ ]);
1157
+ });
1158
+
1159
+ test("an unset language reaches the relay as code-switching, not English", async () => {
1160
+ // The relay resolves an absent language to flux-general-en, so unset
1161
+ // would be a silent English pin on non-English speech.
1162
+ mockVellumAvailable = true;
1163
+ applyConfig({ provider: "vellum-flux" });
1164
+
1165
+ await resolveStreamingTranscriber({ sampleRate: 16000 });
1166
+
1167
+ expect(
1168
+ (vellumFluxStreamCtorCalls[0]?.options as { language?: string })
1169
+ ?.language,
1170
+ ).toBe("multi");
1171
+ });
1172
+
1173
+ test("resolves null without a platform connection", async () => {
1174
+ // The credential is the platform connection, not a stored API key.
1175
+ mockVellumAvailable = false;
1176
+ applyConfig({ provider: "vellum-flux" });
1177
+
1178
+ expect(await resolveStreamingTranscriber({ sampleRate: 16000 })).toBeNull();
1179
+ expect(vellumFluxStreamCtorCalls).toHaveLength(0);
1180
+ });
1181
+
1182
+ test("has no batch boundary, like BYOK Flux", async () => {
1183
+ mockVellumAvailable = true;
1184
+ applyConfig({ provider: "vellum-flux" });
1185
+
1186
+ await expect(resolveBatchTranscriber()).rejects.toThrow(SttError);
1187
+ });
1188
+
1189
+ test("refuses a language Flux has no model for, like BYOK Flux", async () => {
1190
+ // The relay rejects the dial for a language outside Flux's roster, which
1191
+ // reaches the caller as an unexplained param error. Refusing here names
1192
+ // the reason where it can be read.
1193
+ mockVellumAvailable = true;
1194
+ applyConfig({ provider: "vellum-flux", language: "ko" });
1195
+
1196
+ expect(await resolveStreamingTranscriber({ sampleRate: 16000 })).toBeNull();
1197
+ expect(vellumFluxStreamCtorCalls).toHaveLength(0);
1198
+ });
1199
+
1200
+ test("serves a language that is on the Flux roster", async () => {
1201
+ mockVellumAvailable = true;
1202
+ applyConfig({ provider: "vellum-flux", language: "hi" });
1203
+
1204
+ const transcriber = await resolveStreamingTranscriber({
1205
+ sampleRate: 16000,
1206
+ });
1207
+
1208
+ expect(transcriber?.providerId).toBe("vellum-flux");
1209
+ });
1210
+ });
1211
+
1049
1212
  describe("vellum managed resolution", () => {
1050
1213
  beforeEach(() => {
1051
1214
  mockVellumAvailable = false;
@@ -1452,6 +1615,8 @@ describe("streaming capability matches the streaming factory", () => {
1452
1615
  xai: "xai",
1453
1616
  // vellum's credential is the platform connection, mocked below.
1454
1617
  vellum: "vellum",
1618
+ // Managed Flux rides the same platform connection.
1619
+ "vellum-flux": "vellum",
1455
1620
  };
1456
1621
 
1457
1622
  /** The exact list both user-facing error strings advertise. */
@@ -1464,16 +1629,18 @@ describe("streaming capability matches the streaming factory", () => {
1464
1629
  function seedCredentialsFor(id: SttProviderId): void {
1465
1630
  const credential = STREAMING_CREDENTIAL[id];
1466
1631
  expect(credential).toBeDefined();
1467
- mockVellumAvailable = id === "vellum";
1468
- mockVelayConnection =
1469
- id === "vellum"
1470
- ? {
1471
- wsBaseUrl: "ws://gateway.test",
1472
- httpBaseUrl: "http://gateway.test",
1473
- mintServiceToken: () => "vk-test",
1474
- }
1475
- : null;
1476
- mockProviderKeys = id === "vellum" ? {} : { [credential!]: `key-${id}` };
1632
+ // Both managed entries authenticate with the platform connection rather
1633
+ // than a stored API key, and both dial through the same relay.
1634
+ const managed = id === "vellum" || id === "vellum-flux";
1635
+ mockVellumAvailable = managed;
1636
+ mockVelayConnection = managed
1637
+ ? {
1638
+ wsBaseUrl: "ws://gateway.test",
1639
+ httpBaseUrl: "http://gateway.test",
1640
+ mintServiceToken: () => "vk-test",
1641
+ }
1642
+ : null;
1643
+ mockProviderKeys = managed ? {} : { [credential!]: `key-${id}` };
1477
1644
  applyConfig({ provider: id });
1478
1645
  }
1479
1646
 
@@ -1714,3 +1881,140 @@ describe("the multilingual default reaches configs with no stt block", () => {
1714
1881
  expect(getConfig().services.stt.language).toBe("ta");
1715
1882
  });
1716
1883
  });
1884
+
1885
+ // ---------------------------------------------------------------------------
1886
+ // Tests: per-consumer role selection
1887
+ // ---------------------------------------------------------------------------
1888
+
1889
+ describe("services.stt.roles selection", () => {
1890
+ beforeEach(() => {
1891
+ mockVellumAvailable = false;
1892
+ mockVelayConnection = null;
1893
+ mockProviderKeys = { deepgram: "dg-key", openai: "oai-key" };
1894
+ deepgramCtorCalls.length = 0;
1895
+ whisperCtorCalls.length = 0;
1896
+ });
1897
+
1898
+ test("a role override wins over the global provider", async () => {
1899
+ applyConfig({
1900
+ provider: "openai-whisper",
1901
+ roles: { dictation: { provider: "deepgram" } },
1902
+ });
1903
+
1904
+ const transcriber = await resolveStreamingTranscriber({
1905
+ role: "dictation",
1906
+ sampleRate: 16000,
1907
+ });
1908
+
1909
+ expect(transcriber?.providerId).toBe("deepgram");
1910
+ expect(whisperCtorCalls).toHaveLength(0);
1911
+ });
1912
+
1913
+ test("a role with no override falls back to the global provider", async () => {
1914
+ // The invariant that keeps an install with no role set on one provider.
1915
+ applyConfig({
1916
+ provider: "openai-whisper",
1917
+ roles: { dictation: { provider: "deepgram" } },
1918
+ });
1919
+
1920
+ const transcriber = await resolveStreamingTranscriber({
1921
+ role: "liveVoice",
1922
+ sampleRate: 16000,
1923
+ });
1924
+
1925
+ expect(transcriber?.providerId).toBe("openai-whisper");
1926
+ });
1927
+
1928
+ test("a roleless caller still reads the global provider", async () => {
1929
+ applyConfig({
1930
+ provider: "openai-whisper",
1931
+ roles: { dictation: { provider: "deepgram" } },
1932
+ });
1933
+
1934
+ const transcriber = await resolveStreamingTranscriber({
1935
+ sampleRate: 16000,
1936
+ });
1937
+
1938
+ expect(transcriber?.providerId).toBe("openai-whisper");
1939
+ });
1940
+
1941
+ test("an explicit providerId beats the role, since that caller already resolved", async () => {
1942
+ applyConfig({
1943
+ provider: "openai-whisper",
1944
+ roles: { liveVoice: { provider: "openai-whisper" } },
1945
+ });
1946
+
1947
+ const transcriber = await resolveStreamingTranscriber({
1948
+ role: "liveVoice",
1949
+ providerId: "deepgram",
1950
+ sampleRate: 16000,
1951
+ });
1952
+
1953
+ expect(transcriber?.providerId).toBe("deepgram");
1954
+ });
1955
+
1956
+ test("the batch resolver honours its own role", async () => {
1957
+ applyConfig({
1958
+ provider: "openai-whisper",
1959
+ roles: { batch: { provider: "deepgram" } },
1960
+ });
1961
+
1962
+ const transcriber = await resolveBatchTranscriber({ role: "batch" });
1963
+
1964
+ expect(transcriber?.providerId).toBe("deepgram");
1965
+ });
1966
+
1967
+ test("a role selects a model family the global default does not use", async () => {
1968
+ // The case the split exists for: live voice on the streaming-only family
1969
+ // while file transcription keeps the one that can batch.
1970
+ applyConfig({
1971
+ provider: "deepgram",
1972
+ roles: { liveVoice: { provider: "deepgram", model: "flux" } },
1973
+ });
1974
+
1975
+ const live = await resolveStreamingTranscriber({
1976
+ role: "liveVoice",
1977
+ sampleRate: 16000,
1978
+ });
1979
+ const batch = await resolveBatchTranscriber({ role: "batch" });
1980
+
1981
+ expect(live?.providerId).toBe("deepgram-flux");
1982
+ expect(batch?.providerId).toBe("deepgram");
1983
+ });
1984
+
1985
+ test("the watch session resolves its own role", async () => {
1986
+ applyConfig({
1987
+ provider: "openai-whisper",
1988
+ roles: { watch: { provider: "deepgram" } },
1989
+ });
1990
+
1991
+ const transcriber = await resolveStreamingTranscriber({
1992
+ role: "watch",
1993
+ sampleRate: 16000,
1994
+ });
1995
+
1996
+ expect(transcriber?.providerId).toBe("deepgram");
1997
+ expect(whisperCtorCalls).toHaveLength(0);
1998
+ });
1999
+
2000
+ test("roles route two consumers to different providers at once", async () => {
2001
+ // The failure this whole change exists to prevent: one global forcing
2002
+ // every consumer onto a provider that only suits one of them.
2003
+ applyConfig({
2004
+ provider: "openai-whisper",
2005
+ roles: {
2006
+ dictation: { provider: "deepgram" },
2007
+ batch: { provider: "openai-whisper" },
2008
+ },
2009
+ });
2010
+
2011
+ const dictation = await resolveStreamingTranscriber({
2012
+ role: "dictation",
2013
+ sampleRate: 16000,
2014
+ });
2015
+ const batch = await resolveBatchTranscriber({ role: "batch" });
2016
+
2017
+ expect(dictation?.providerId).toBe("deepgram");
2018
+ expect(batch?.providerId).toBe("openai-whisper");
2019
+ });
2020
+ });
@@ -294,8 +294,13 @@ const EAGER_EOT_THRESHOLD_RANGE = { min: 0.3, max: 0.9 } as const;
294
294
  const EOT_TIMEOUT_MS_RANGE = { min: 500, max: 60_000 } as const;
295
295
 
296
296
  export interface FluxQueryParamOptions {
297
- /** Flux model to run, e.g. `"flux-general-en"`. Required by Deepgram. */
298
- model: string;
297
+ /**
298
+ * Flux model to run, e.g. `"flux-general-en"`. Required when dialing
299
+ * Deepgram directly. Omitted on the managed relay, which derives the model
300
+ * from the spoken language and prices it before dialing, and rejects a
301
+ * client-sent one.
302
+ */
303
+ model?: string;
299
304
  /**
300
305
  * Encoding of the raw audio being sent. Omit for containerized audio,
301
306
  * which carries its own format header.
@@ -316,6 +321,12 @@ export interface FluxQueryParamOptions {
316
321
  eagerEotThreshold?: number;
317
322
  /** Silence (ms) after which Flux force-ends a turn. */
318
323
  eotTimeoutMs?: number;
324
+ /**
325
+ * Language hint for the multilingual model. Meaningless on the English
326
+ * model, and omitted entirely for code-switching, which is what asks Flux
327
+ * to detect the language itself.
328
+ */
329
+ languageHint?: string;
319
330
  }
320
331
 
321
332
  /**
@@ -336,9 +347,78 @@ export interface FluxQueryParamOptions {
336
347
  * misconfigured pair from failing the whole session; the adjustment is logged
337
348
  * at debug when it fires.
338
349
  */
350
+ /** Flux's English model, and its multilingual sibling. */
351
+ const FLUX_MODEL_ENGLISH = "flux-general-en";
352
+ const FLUX_MODEL_MULTILINGUAL = "flux-general-multi";
353
+
354
+ /**
355
+ * Languages the multilingual Flux model serves, as base subtags.
356
+ *
357
+ * Deepgram documents this roster for `flux-general-multi`; anything outside it
358
+ * has no model to run on.
359
+ */
360
+ /**
361
+ * Exported so the provider catalog can report it. A client deciding whether to
362
+ * offer turn detection needs the same roster, and copying it there would make
363
+ * a third place to keep in step with this one and the relay's.
364
+ */
365
+ export const FLUX_MULTILINGUAL_SUBTAGS: ReadonlySet<string> = new Set([
366
+ "de",
367
+ "en",
368
+ "es",
369
+ "fr",
370
+ "hi",
371
+ "it",
372
+ "ja",
373
+ "nl",
374
+ "pt",
375
+ "ru",
376
+ ]);
377
+
378
+ /** The code that asks Flux to detect and code-switch rather than be told. */
379
+ const FLUX_CODE_SWITCHING = "multi";
380
+
381
+ /**
382
+ * The model and optional hint a spoken language selects.
383
+ *
384
+ * Deliberately the same mapping the managed relay applies server-side, so a
385
+ * language means the same thing whether Flux is reached with your own key or
386
+ * through the platform. Without it a BYOK session pins the English model and
387
+ * returns English-sounding nonsense for every other language, which is
388
+ * invisible from the transcript alone.
389
+ *
390
+ * Returns null when the language has no Flux model, leaving the caller to
391
+ * refuse rather than transcribe the wrong thing.
392
+ */
393
+ export function fluxModelForLanguage(
394
+ language: string | undefined,
395
+ ): { model: string; languageHint?: string } | null {
396
+ const normalized = language?.trim().toLowerCase();
397
+ if (!normalized) {
398
+ return { model: FLUX_MODEL_ENGLISH };
399
+ }
400
+ if (normalized === FLUX_CODE_SWITCHING) {
401
+ return { model: FLUX_MODEL_MULTILINGUAL };
402
+ }
403
+ const base = normalized.split("-")[0] ?? normalized;
404
+ if (!FLUX_MULTILINGUAL_SUBTAGS.has(base)) {
405
+ return null;
406
+ }
407
+ if (base === "en") {
408
+ return { model: FLUX_MODEL_ENGLISH };
409
+ }
410
+ return { model: FLUX_MODEL_MULTILINGUAL, languageHint: base };
411
+ }
412
+
339
413
  export function buildFluxQueryParams(opts: FluxQueryParamOptions): string {
340
414
  const params = new URLSearchParams();
341
- params.set("model", opts.model);
415
+ if (opts.model !== undefined) {
416
+ params.set("model", opts.model);
417
+ }
418
+
419
+ if (opts.languageHint !== undefined) {
420
+ params.set("language_hint", opts.languageHint);
421
+ }
342
422
 
343
423
  if (opts.encoding !== undefined) {
344
424
  params.set("encoding", opts.encoding);
@@ -52,6 +52,7 @@ import { getLogger } from "../../util/logger.js";
52
52
  import type { FluxEncoding } from "./deepgram-flux-frames.js";
53
53
  import {
54
54
  buildFluxQueryParams,
55
+ fluxModelForLanguage,
55
56
  parseFluxFrame,
56
57
  } from "./deepgram-flux-frames.js";
57
58
 
@@ -115,6 +116,43 @@ export interface DeepgramFluxRealtimeOptions {
115
116
  connectTimeoutMs?: number;
116
117
  /** Inactivity timeout in milliseconds. Default: 30_000. */
117
118
  inactivityTimeoutMs?: number;
119
+ /**
120
+ * WebSocket origin to dial. Defaults to Deepgram's own. The managed relay
121
+ * points this at the gateway, which forwards to velay.
122
+ */
123
+ baseUrl?: string;
124
+ /** Path on {@link baseUrl}. Defaults to Deepgram's `/v2/listen`. */
125
+ path?: string;
126
+ /**
127
+ * Carry the credential in a `key` query param rather than an
128
+ * `Authorization` header, which is how the gateway relay authenticates.
129
+ * Turns on URL redaction for logs.
130
+ */
131
+ queryAuth?: boolean;
132
+ /**
133
+ * Omit `model` from the query. The managed relay derives the Flux model
134
+ * from the spoken language and prices it before dialing, so it rejects a
135
+ * client-selected one.
136
+ */
137
+ omitModelParam?: boolean;
138
+ /**
139
+ * Ask the relay for native Flux frames instead of the v1-shaped Deepgram
140
+ * translation it serves released assistants by default. Meaningless when
141
+ * dialing Deepgram directly, which only speaks Flux.
142
+ */
143
+ nativeContract?: boolean;
144
+ /**
145
+ * Spoken language, forwarded to the relay so it can select the Flux model
146
+ * and language hint. Deepgram's own endpoint takes no `language`, so this
147
+ * is only sent when set, which is the managed path.
148
+ */
149
+ language?: string;
150
+ /**
151
+ * Called with any JSON frame the Flux parser produced no events from. The
152
+ * managed relay multiplexes its own `velay_error` control frames onto the
153
+ * same socket; this is how the wrapper sees them.
154
+ */
155
+ onUnhandledFrame?: (frame: Record<string, unknown>) => void;
118
156
  }
119
157
 
120
158
  // ---------------------------------------------------------------------------
@@ -165,6 +203,15 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
165
203
  private readonly sampleRate: number;
166
204
  private readonly connectTimeoutMs: number;
167
205
  private readonly inactivityTimeoutMs: number;
206
+ private readonly baseUrl: string;
207
+ private readonly path: string;
208
+ private readonly queryAuth: boolean;
209
+ private readonly omitModelParam: boolean;
210
+ private readonly nativeContract: boolean;
211
+ private readonly language: string | undefined;
212
+ private readonly onUnhandledFrame:
213
+ | ((frame: Record<string, unknown>) => void)
214
+ | undefined;
168
215
 
169
216
  /** The live WebSocket connection, set during start(). */
170
217
  private ws: WsLike | null = null;
@@ -211,6 +258,29 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
211
258
  options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
212
259
  this.inactivityTimeoutMs =
213
260
  options.inactivityTimeoutMs ?? DEFAULT_INACTIVITY_TIMEOUT_MS;
261
+ this.baseUrl = (options.baseUrl ?? WS_BASE_URL).replace(/\/+$/, "");
262
+ this.path = options.path ?? FLUX_PATH;
263
+ this.queryAuth = options.queryAuth ?? false;
264
+ this.omitModelParam = options.omitModelParam ?? false;
265
+ this.nativeContract = options.nativeContract ?? false;
266
+ this.language = options.language;
267
+ this.onUnhandledFrame = options.onUnhandledFrame;
268
+ }
269
+
270
+ /**
271
+ * Blank the credential in text bound for a log. Only query auth puts it in
272
+ * a URL; the header path never does, so this is a no-op there.
273
+ */
274
+ private redact(text: string): string {
275
+ if (!this.queryAuth || !text) {
276
+ return text;
277
+ }
278
+ return text
279
+ .split(this.apiKey)
280
+ .join("***")
281
+ .split(encodeURIComponent(this.apiKey))
282
+ .join("***")
283
+ .replace(/([?&]key=)[^&\s"']*/g, "$1***");
214
284
  }
215
285
 
216
286
  // ── StreamingTranscriber interface ──────────────────────────────────
@@ -222,7 +292,7 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
222
292
  this.onEvent = onEvent;
223
293
 
224
294
  const url = this.buildWebSocketUrl();
225
- log.info({ url }, "Opening Deepgram Flux session");
295
+ log.info({ url: this.redact(url) }, "Opening Deepgram Flux session");
226
296
 
227
297
  const ws = this.createWebSocket(url);
228
298
  this.ws = ws;
@@ -369,6 +439,10 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
369
439
  if (typeof WebSocketCtor !== "function") {
370
440
  throw new Error("global WebSocket is not available in this runtime");
371
441
  }
442
+ // Query auth carries the credential in the URL instead.
443
+ if (this.queryAuth) {
444
+ return new WebSocketCtor(url);
445
+ }
372
446
  return new WebSocketCtor(url, {
373
447
  headers: { Authorization: `Token ${this.apiKey}` },
374
448
  });
@@ -415,7 +489,13 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
415
489
  return;
416
490
  }
417
491
 
418
- for (const event of parseFluxFrame(raw)) {
492
+ const events = parseFluxFrame(raw);
493
+ if (events.length === 0) {
494
+ this.reportUnhandledFrame(raw);
495
+ return;
496
+ }
497
+
498
+ for (const event of events) {
419
499
  if (event.type === "error") {
420
500
  this.fatalErrorReported = true;
421
501
  }
@@ -423,6 +503,27 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
423
503
  }
424
504
  }
425
505
 
506
+ /**
507
+ * Hand a frame the Flux parser made nothing of to the transport wrapper.
508
+ * On the direct Deepgram path nothing listens and this is inert; on the
509
+ * managed relay it is how `velay_error` reaches the caller.
510
+ */
511
+ private reportUnhandledFrame(raw: string): void {
512
+ if (!this.onUnhandledFrame) {
513
+ return;
514
+ }
515
+ let frame: unknown;
516
+ try {
517
+ frame = JSON.parse(raw);
518
+ } catch {
519
+ return;
520
+ }
521
+ if (typeof frame !== "object" || frame === null || Array.isArray(frame)) {
522
+ return;
523
+ }
524
+ this.onUnhandledFrame(frame as Record<string, unknown>);
525
+ }
526
+
426
527
  /** Handle provider-side WebSocket close. */
427
528
  private handleProviderClose(code: number, reason: string): void {
428
529
  if (this.closed) {
@@ -635,15 +736,42 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
635
736
  * {@link buildFluxQueryParams}.
636
737
  */
637
738
  private buildWebSocketUrl(): string {
739
+ // Dialing Deepgram directly means choosing the model here. The relay
740
+ // derives it from the language instead, which is why the managed path
741
+ // sends neither.
742
+ const selection = this.omitModelParam
743
+ ? null
744
+ : // An explicit pin is the operator's call and wins; otherwise the
745
+ // language picks, matching what the relay does server-side.
746
+ this.flux.model
747
+ ? { model: this.flux.model }
748
+ : fluxModelForLanguage(this.language);
638
749
  const query = buildFluxQueryParams({
639
- model: this.flux.model,
750
+ ...(selection ? { model: selection.model } : {}),
751
+ ...(selection?.languageHint !== undefined
752
+ ? { languageHint: selection.languageHint }
753
+ : {}),
640
754
  encoding: AUDIO_ENCODING,
641
755
  sampleRate: this.sampleRate,
642
756
  eotThreshold: this.flux.eotThreshold,
643
757
  eagerEotThreshold: this.flux.eagerEotThreshold,
644
758
  eotTimeoutMs: this.flux.eotTimeoutMs,
645
759
  });
646
- return `${WS_BASE_URL}${FLUX_PATH}?${query}`;
760
+
761
+ // Relay-only params. Deepgram takes neither: `language` is how velay
762
+ // picks the Flux model and language hint, and `contract` asks it to stop
763
+ // translating Flux into the v1 Deepgram dialect.
764
+ const params = new URLSearchParams(query);
765
+ if (this.language) {
766
+ params.set("language", this.language);
767
+ }
768
+ if (this.nativeContract) {
769
+ params.set("contract", "flux");
770
+ }
771
+ if (this.queryAuth) {
772
+ params.set("key", this.apiKey);
773
+ }
774
+ return `${this.baseUrl}${this.path}?${params.toString()}`;
647
775
  }
648
776
  }
649
777