@vellumai/assistant 0.11.6 → 0.11.7-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +31 -19
- package/Dockerfile +1 -0
- package/docs/architecture/security.md +3 -3
- package/docs/flux-turn-detection-spike.md +8 -7
- package/docs/runbook-trusted-contacts.md +14 -0
- package/docs/stt-provider-onboarding.md +8 -4
- package/knip.json +1 -0
- package/node_modules/@vellumai/avatar-manifest/package.json +18 -0
- package/node_modules/@vellumai/avatar-manifest/src/__tests__/manifest.test.ts +173 -0
- package/node_modules/@vellumai/avatar-manifest/src/__tests__/read.test.ts +114 -0
- package/node_modules/@vellumai/avatar-manifest/src/index.ts +29 -0
- package/node_modules/@vellumai/avatar-manifest/src/layout.ts +18 -0
- package/node_modules/@vellumai/avatar-manifest/src/manifest.ts +156 -0
- package/node_modules/@vellumai/avatar-manifest/src/read.ts +109 -0
- package/node_modules/@vellumai/avatar-manifest/tsconfig.json +20 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
- package/node_modules/@vellumai/ces-client/src/http-credentials.ts +161 -0
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/environments/package.json +2 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +56 -0
- package/node_modules/@vellumai/environments/src/shell.ts +65 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/contact-read-contracts.test.ts +32 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/gateway-client.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/inbound-event-kind.test.ts +82 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/plugin-admission-denied-contract.test.ts +56 -0
- package/node_modules/@vellumai/gateway-client/src/__tests__/trust-verdict-contract.test.ts +18 -0
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +5 -11
- package/node_modules/@vellumai/gateway-client/src/channel-permission-contract.ts +3 -1
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +9 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +4 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-event-kind.ts +75 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +17 -0
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +8 -0
- package/node_modules/@vellumai/gateway-client/src/plugin-admission-denied-contract.ts +47 -0
- package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +11 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +37 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/remote-web-pairing.test.ts +389 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +22 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +508 -18
- package/node_modules/@vellumai/service-contracts/src/rpc.ts +174 -0
- package/openapi.yaml +419 -55
- package/package.json +4 -1
- package/scripts/sync-llm-catalog.ts +6 -0
- package/src/__tests__/approval-card-edit-inbound-handoff.test.ts +199 -0
- package/src/__tests__/approval-card-edit.test.ts +153 -0
- package/src/__tests__/attachments-store-heic-normalize.test.ts +1 -1
- package/src/__tests__/available-skills.test.ts +54 -0
- package/src/__tests__/avatar-identity-sync.test.ts +5 -1
- package/src/__tests__/catalog-install-normalize.test.ts +9 -1
- package/src/__tests__/channel-policy.test.ts +1 -3
- package/src/__tests__/channel-readiness-discord.test.ts +9 -70
- package/src/__tests__/compactor-call-site-logging.test.ts +61 -0
- package/src/__tests__/config-loader-backfill.test.ts +69 -0
- package/src/__tests__/config-schema.test.ts +2 -3
- package/src/__tests__/conversation-agent-loop.test.ts +238 -0
- package/src/__tests__/conversation-attachments.test.ts +0 -1
- package/src/__tests__/conversation-error.test.ts +44 -0
- package/src/__tests__/conversation-evictor.test.ts +19 -0
- package/src/__tests__/conversation-fork-referential.test.ts +73 -63
- package/src/__tests__/conversation-fork-retrospective.test.ts +99 -74
- package/src/__tests__/conversation-runtime-assembly.test.ts +23 -0
- package/src/__tests__/conversation-slash.test.ts +9 -0
- package/src/__tests__/conversation-title-service.test.ts +97 -0
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +16 -0
- package/src/__tests__/credential-broker-server-use.test.ts +0 -218
- package/src/__tests__/credential-broker.test.ts +43 -420
- package/src/__tests__/credential-record-write-through.test.ts +78 -0
- package/src/__tests__/credential-routes.test.ts +17 -1
- package/src/__tests__/credential-security-invariants.test.ts +8 -9
- package/src/__tests__/default-profile-catalog-fallback.test.ts +289 -0
- package/src/__tests__/delete-propagation.test.ts +92 -2
- package/src/__tests__/discord-callback-round-trip.test.ts +93 -0
- package/src/__tests__/discord-channel-config.test.ts +237 -0
- package/src/__tests__/edit-propagation.test.ts +42 -2
- package/src/__tests__/evict-conversations-for-reload.test.ts +51 -1
- package/src/__tests__/fallback-breaker.test.ts +1335 -0
- package/src/__tests__/gateway-threshold-reader-mock.ts +6 -0
- package/src/__tests__/guardian-card-withdrawal.test.ts +30 -0
- package/src/__tests__/handlers-user-message-approval-consumption.test.ts +0 -2
- package/src/__tests__/image-conversion.test.ts +38 -4
- package/src/__tests__/install-skill-routing.test.ts +25 -0
- package/src/__tests__/llm-catalog-parity.test.ts +4 -0
- package/src/__tests__/llm-resolver.test.ts +74 -0
- package/src/__tests__/llm-schema-fallback-profile.test.ts +359 -0
- package/src/__tests__/managed-fallback-dispatch.test.ts +430 -0
- package/src/__tests__/managed-profile-guard.test.ts +28 -0
- package/src/__tests__/managed-speech-defaults.test.ts +196 -1
- package/src/__tests__/mcp-cli.test.ts +23 -1
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +43 -34
- package/src/__tests__/media-stream-stt-session.test.ts +2 -0
- package/src/__tests__/notification-decision-fallback.test.ts +106 -0
- package/src/__tests__/notification-discord-adapter.test.ts +162 -0
- package/src/__tests__/oauth-connect-orchestrator.test.ts +95 -0
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +6 -8
- package/src/__tests__/persist-unsendable-image.test.ts +3 -3
- package/src/__tests__/platform.test.ts +19 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +5 -1
- package/src/__tests__/provider-catalog-visibility.test.ts +17 -0
- package/src/__tests__/provider-platform-proxy-integration.test.ts +8 -1
- package/src/__tests__/provider-usage-tracking.test.ts +56 -0
- package/src/__tests__/require-fresh-approval.test.ts +5 -2
- package/src/__tests__/retry-fallback-escalation.test.ts +1093 -0
- package/src/__tests__/schedule-retry.test.ts +183 -4
- package/src/__tests__/scheduler-reuse-conversation.test.ts +6 -6
- package/src/__tests__/secret-routes-platform-proxy.test.ts +55 -0
- package/src/__tests__/skill-load-tool.test.ts +34 -0
- package/src/__tests__/skill-tool-factory.test.ts +6 -1
- package/src/__tests__/skill-tool-manifest.test.ts +20 -0
- package/src/__tests__/skills.test.ts +25 -0
- package/src/__tests__/subagent-manager-notify.test.ts +38 -0
- package/src/__tests__/terminal-tools.test.ts +2 -0
- package/src/__tests__/tool-approval-handler.test.ts +146 -2
- package/src/__tests__/tool-approval-seed-content-blocks.test.ts +108 -2
- package/src/__tests__/tool-audit.test.ts +111 -40
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +228 -18
- package/src/__tests__/ui-shape-teaching.test.ts +1 -1
- package/src/__tests__/verification-control-plane-policy.test.ts +6 -2
- package/src/__tests__/voice-config-update.test.ts +43 -0
- package/src/__tests__/workspace-migration-147-rename-colliding-backup-profile-names.test.ts +480 -0
- package/src/__tests__/workspace-migration-148-strip-unsupported-fallback-profiles.test.ts +167 -0
- package/src/__tests__/workspace-migration-149-repoint-backup-profile-selections.test.ts +144 -0
- package/src/agent/loop.ts +31 -2
- package/src/approvals/guardian-card-withdrawal.ts +22 -0
- package/src/avatar/__tests__/ensure-raster.test.ts +291 -0
- package/src/avatar/avatar-manifest.ts +48 -100
- package/src/avatar/avatar-store.ts +7 -7
- package/src/avatar/ensure-raster.ts +146 -0
- package/src/avatar/resvg-lazy.test.ts +0 -1
- package/src/avatar/traits-png-sync.ts +9 -10
- package/src/background-wake/wake-intent-hooks.test.ts +0 -1
- package/src/backup/__tests__/paths.test.ts +6 -2
- package/src/backup/paths.ts +6 -4
- package/src/bundler/app-compiler.ts +2 -2
- package/src/bundler/compiler-tools.test.ts +36 -0
- package/src/bundler/compiler-tools.ts +54 -18
- package/src/calls/__tests__/telephony-synthesis-language.test.ts +81 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +57 -0
- package/src/calls/media-stream-stt-session.ts +2 -1
- package/src/calls/telephony-synthesis-language.ts +13 -5
- package/src/calls/voice-session-bridge.ts +19 -4
- package/src/channels/config.ts +23 -12
- package/src/cli/AGENTS.md +3 -1
- package/src/cli/commands/backup.help.ts +10 -13
- package/src/cli/commands/contacts.help.ts +16 -4
- package/src/cli/commands/contacts.ts +5 -1
- package/src/cli/commands/credentials.help.ts +1 -1
- package/src/cli/commands/inference-providers.ts +18 -53
- package/src/cli/commands/mcp.help.ts +6 -5
- package/src/cli/commands/mcp.ts +1 -1
- package/src/cli/commands/memory/memory-retrospective.ts +1 -0
- package/src/cli/commands/stt.help.ts +2 -2
- package/src/config/__tests__/backup-schema.test.ts +1 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +83 -2
- package/src/config/__tests__/loader-sparse-array-cleanup.test.ts +1 -1
- package/src/config/__tests__/memory-retrospective-schema.test.ts +12 -15
- package/src/config/bundled-skills/computer-use/SKILL.md +6 -6
- package/src/config/bundled-skills/computer-use/TOOLS.json +9 -6
- package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +3 -1
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +1 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +1 -1
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +16 -15
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +2 -1
- package/src/config/bundled-skills/settings/TOOLS.json +11 -5
- package/src/config/bundled-skills/settings/tools/open-system-settings.test.ts +85 -0
- package/src/config/bundled-skills/settings/tools/open-system-settings.ts +33 -8
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +54 -3
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +1 -1
- package/src/config/default-profile-catalog.ts +195 -10
- package/src/config/default-profile-names.ts +41 -0
- package/src/config/feature-flag-registry.json +34 -2
- package/src/config/inference-profile-validation.ts +11 -2
- package/src/config/llm-resolver.ts +25 -0
- package/src/config/loader.ts +3 -0
- package/src/config/managed-speech-defaults.ts +160 -10
- package/src/config/schema.ts +2 -0
- package/src/config/schemas/__tests__/live-voice.test.ts +7 -6
- package/src/config/schemas/__tests__/stt.test.ts +94 -0
- package/src/config/schemas/backup.ts +1 -1
- package/src/config/schemas/channels.ts +32 -0
- package/src/config/schemas/live-voice.ts +13 -4
- package/src/config/schemas/llm.ts +263 -15
- package/src/config/schemas/mcp.ts +10 -2
- package/src/config/schemas/memory-retrospective.ts +1 -19
- package/src/config/schemas/services.ts +24 -0
- package/src/config/schemas/stt.ts +128 -5
- package/src/config/seed-inference-profiles.ts +32 -3
- package/src/config/skill-state.ts +5 -0
- package/src/config/skills.ts +21 -0
- package/src/contacts/notify-contacts-changed.ts +5 -2
- package/src/context/compactor.ts +74 -1
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +138 -1
- package/src/daemon/config-watcher.ts +2 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +27 -0
- package/src/daemon/conversation-agent-loop.ts +86 -10
- package/src/daemon/conversation-error.ts +25 -0
- package/src/daemon/conversation-evictor.ts +2 -4
- package/src/daemon/conversation-runtime-assembly.ts +19 -7
- package/src/daemon/conversation-skill-tools.ts +5 -1
- package/src/daemon/conversation-slash.ts +2 -2
- package/src/daemon/conversation-store.ts +8 -1
- package/src/daemon/conversation-tool-setup.ts +37 -0
- package/src/daemon/conversation.ts +0 -4
- package/src/daemon/handlers/channel-config-result.ts +30 -0
- package/src/daemon/handlers/config-discord-channel.ts +301 -0
- package/src/daemon/handlers/config-ingress.ts +9 -0
- package/src/daemon/handlers/config-slack-channel.ts +13 -15
- package/src/daemon/handlers/config-telegram.ts +2 -5
- package/src/daemon/handlers/conversations.ts +0 -2
- package/src/daemon/handlers/skills.ts +25 -7
- package/src/daemon/lifecycle.ts +3 -1
- package/src/daemon/message-types/messages.ts +0 -2
- package/src/daemon/tool-setup-types.ts +5 -3
- package/src/daemon/trust-context-types.test.ts +33 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +0 -1
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +58 -4
- package/src/live-voice/__tests__/live-voice-managed-speech-fallback.test.ts +8 -3
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +73 -11
- package/src/live-voice/__tests__/live-voice-session-preflight.test.ts +148 -2
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +64 -0
- package/src/live-voice/__tests__/live-voice-stt.test.ts +3 -0
- package/src/live-voice/__tests__/live-voice-text-turn.test.ts +391 -0
- package/src/live-voice/__tests__/protocol.test.ts +224 -0
- package/src/live-voice/live-voice-credential-preflight.ts +5 -1
- package/src/live-voice/live-voice-session-manager.ts +39 -2
- package/src/live-voice/live-voice-session.ts +224 -9
- package/src/live-voice/protocol.ts +187 -1
- package/src/messaging/providers/discord/api.ts +2 -1
- package/src/messaging/providers/discord/withdraw.ts +62 -0
- package/src/messaging/read-provider-metadata.ts +32 -0
- package/src/monitoring/__tests__/file-descriptors.test.ts +32 -0
- package/src/monitoring/__tests__/process-memory.test.ts +34 -1
- package/src/monitoring/db-integrity-sample.ts +1 -0
- package/src/monitoring/file-descriptors.ts +39 -6
- package/src/monitoring/process-memory.ts +37 -5
- package/src/monitoring/resource-sampler.ts +25 -8
- package/src/notifications/__tests__/activity-failed-dedupe.test.ts +161 -0
- package/src/notifications/__tests__/background-failure-signal.test.ts +107 -0
- package/src/notifications/__tests__/connected-channels.test.ts +12 -0
- package/src/notifications/__tests__/copy-composer.test.ts +83 -0
- package/src/notifications/__tests__/destination-resolver.test.ts +38 -0
- package/src/notifications/activity-failed-dedupe.ts +95 -0
- package/src/notifications/adapters/discord.ts +132 -0
- package/src/notifications/adapters/shared.ts +16 -0
- package/src/notifications/adapters/slack.ts +6 -1
- package/src/notifications/adapters/telegram.ts +5 -8
- package/src/notifications/approval-card-data.ts +94 -1
- package/src/notifications/background-failure-signal.ts +116 -0
- package/src/notifications/broadcaster.ts +3 -9
- package/src/notifications/copy-composer.ts +41 -8
- package/src/notifications/decision-engine.ts +55 -0
- package/src/notifications/destination-resolver.ts +45 -5
- package/src/notifications/emit-signal.ts +19 -5
- package/src/notifications/guardian-question-mode.ts +57 -0
- package/src/notifications/types.ts +2 -8
- package/src/oauth/__tests__/seed-providers-managed.test.ts +18 -0
- package/src/oauth/connect-orchestrator.ts +16 -0
- package/src/oauth/seed-providers.ts +262 -0
- package/src/permissions/checker.test.ts +21 -0
- package/src/permissions/checker.ts +6 -0
- package/src/permissions/gateway-threshold-reader.test.ts +184 -0
- package/src/permissions/gateway-threshold-reader.ts +144 -10
- package/src/permissions/prompter.ts +0 -6
- package/src/permissions/types.ts +5 -4
- package/src/persistence/__tests__/bulk-write-gate.test.ts +0 -38
- package/src/persistence/bulk-write-gate.ts +9 -10
- package/src/persistence/conversation-crud.ts +88 -134
- package/src/persistence/conversation-title-service.ts +43 -29
- package/src/persistence/embeddings/embedding-local.ts +14 -85
- package/src/persistence/embeddings/embedding-runtime-manager.ts +1 -0
- package/src/persistence/lifecycle-events-store.test.ts +50 -0
- package/src/persistence/lifecycle-events-store.ts +75 -5
- package/src/platform/platform-patch-queue.test.ts +368 -0
- package/src/platform/platform-patch-queue.ts +190 -0
- package/src/platform/sync-avatar.test.ts +527 -0
- package/src/platform/sync-avatar.ts +209 -0
- package/src/platform/sync-identity.test.ts +75 -0
- package/src/platform/sync-identity.ts +17 -110
- package/src/plugin-api/model-profiles.test.ts +2 -2
- package/src/plugin-api/model-profiles.ts +7 -7
- package/src/plugin-api/transcription-session.ts +1 -1
- package/src/plugins/defaults/compaction/window-manager.ts +4 -0
- package/src/plugins/defaults/image-recovery/recover.ts +2 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +133 -34
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +21 -31
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +13 -9
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +31 -4
- package/src/plugins/defaults/memory/memory-retrospective-constants.ts +0 -11
- package/src/plugins/defaults/memory/memory-retrospective-fork-boundary.ts +27 -13
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +183 -113
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +13 -13
- package/src/plugins/defaults/memory/src/memory-v2-routes.ts +2 -2
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +1 -1
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +4 -2
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +9 -42
- package/src/plugins/defaults/memory/v2/rerank-local.ts +1 -0
- package/src/plugins/defaults/memory/worker.ts +7 -0
- package/src/providers/__tests__/provider-secret-catalog.test.ts +1 -0
- package/src/providers/__tests__/retry-callsite.test.ts +55 -0
- package/src/providers/__tests__/vellum-connection-routing.test.ts +21 -3
- package/src/providers/connection-resolution.ts +2 -17
- package/src/providers/fallback-breaker.ts +389 -0
- package/src/providers/inference/__tests__/adapter-factory-opencode.test.ts +92 -0
- package/src/providers/inference/__tests__/base-url-route-validation.test.ts +16 -0
- package/src/providers/inference/__tests__/base-url-security.test.ts +12 -0
- package/src/providers/inference/adapter-factory.ts +246 -3
- package/src/providers/inference/auth.ts +15 -14
- package/src/providers/inference/chatgpt-subscription-auth.ts +70 -0
- package/src/providers/inference/chatgpt-subscription-credentials.ts +84 -0
- package/src/providers/inference/resolve-auth.ts +1 -1
- package/src/providers/media-resolve.ts +2 -3
- package/src/providers/model-catalog.ts +100 -0
- package/src/providers/openai/__tests__/api-error-normalization.test.ts +22 -0
- package/src/providers/openai/__tests__/chat-template-flatten.test.ts +198 -0
- package/src/providers/openai/api-error-normalization.ts +13 -0
- package/src/providers/openai/chat-completions-provider.ts +84 -0
- package/src/providers/opencode/client.test.ts +150 -0
- package/src/providers/opencode/client.ts +61 -0
- package/src/providers/platform-proxy/constants.ts +5 -0
- package/src/providers/provider-resolvability.ts +20 -0
- package/src/providers/provider-secret-catalog.ts +3 -2
- package/src/providers/retry.ts +956 -68
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +42 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +136 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +120 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +316 -12
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +83 -3
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +132 -4
- package/src/providers/speech-to-text/provider-catalog.ts +191 -3
- package/src/providers/speech-to-text/resolve.ts +132 -24
- package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +296 -0
- package/src/providers/types.ts +39 -0
- package/src/providers/usage-tracking.ts +12 -1
- package/src/providers/vellum/client.ts +29 -0
- package/src/providers/vellum-model-routing.test.ts +2 -0
- package/src/providers/vellum-model-routing.ts +3 -3
- package/src/runtime/AGENTS.md +1 -2
- package/src/runtime/__tests__/agent-wake.test.ts +105 -3
- package/src/runtime/__tests__/background-job-runner.test.ts +89 -3
- package/src/runtime/agent-wake.ts +44 -5
- package/src/runtime/background-job-runner.ts +68 -50
- package/src/runtime/channel-readiness-service.ts +0 -39
- package/src/runtime/http-server.ts +12 -4
- package/src/runtime/routes/__tests__/avatar-state-routes.test.ts +18 -18
- package/src/runtime/routes/__tests__/backup-routes.test.ts +9 -5
- package/src/runtime/routes/__tests__/chatgpt-subscription-auth-routes.test.ts +372 -0
- package/src/runtime/routes/__tests__/contact-routes.test.ts +7 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +96 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +37 -0
- package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +2 -1
- package/src/runtime/routes/__tests__/mcp-add-default-risk.test.ts +87 -0
- package/src/runtime/routes/acp-claude-auth-routes.ts +25 -54
- package/src/runtime/routes/approval-routes.ts +0 -12
- package/src/runtime/routes/avatar-routes.ts +17 -30
- package/src/runtime/routes/backup-routes.ts +2 -2
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +296 -113
- package/src/runtime/routes/contact-prompt-routes.ts +65 -6
- package/src/runtime/routes/contact-routes.ts +20 -15
- package/src/runtime/routes/conversation-query-routes.ts +98 -10
- package/src/runtime/routes/conversation-routes.ts +2 -2
- package/src/runtime/routes/credential-routes.ts +2 -1
- package/src/runtime/routes/guardian-approval-interception.ts +12 -9
- package/src/runtime/routes/inbound-message-handler.ts +47 -48
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +18 -13
- package/src/runtime/routes/inbound-stages/admission-policy.ts +2 -2
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +20 -2
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +9 -8
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +1 -1
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +2 -2
- package/src/runtime/routes/inference-profiles-routes.ts +12 -7
- package/src/runtime/routes/inference-provider-connection-routes.ts +2 -2
- package/src/runtime/routes/inference-send-routes.ts +2 -2
- package/src/runtime/routes/ingress-status-routes.ts +2 -2
- package/src/runtime/routes/integrations/discord.ts +119 -0
- package/src/runtime/routes/llm-call-sites-routes.ts +2 -2
- package/src/runtime/routes/mcp-auth-routes.ts +12 -6
- package/src/runtime/routes/migration-routes.ts +3 -1
- package/src/runtime/routes/oauth-pending-flows.ts +136 -0
- package/src/runtime/routes/secret-routes.ts +7 -0
- package/src/runtime/routes/stt-routes.ts +38 -6
- package/src/runtime/routes/wake-conversation-routes.ts +8 -1
- package/src/runtime/routes/watch-routes.ts +9 -5
- package/src/runtime/sync/resource-sync-events.ts +2 -0
- package/src/runtime/trust-verdict-consumer.ts +3 -3
- package/src/schedule/__tests__/worker-feature-flags.test.ts +58 -0
- package/src/schedule/retry-policy.ts +8 -8
- package/src/schedule/run-script.ts +14 -23
- package/src/schedule/schedule-recovery.ts +0 -1
- package/src/schedule/scheduler.ts +89 -51
- package/src/schedule/worker.ts +57 -0
- package/src/security/__tests__/openai-device-auth.test.ts +355 -0
- package/src/security/ces-rpc-record-backend.ts +123 -0
- package/src/security/oauth2.ts +7 -0
- package/src/security/openai-device-auth.ts +364 -0
- package/src/security/secure-keys.ts +15 -0
- package/src/skills/available-skills.ts +10 -1
- package/src/skills/catalog-install.ts +19 -1
- package/src/skills/inline-command-runner.ts +6 -2
- package/src/skills/platform-compatibility.test.ts +49 -0
- package/src/skills/platform-compatibility.ts +65 -0
- package/src/skills/tool-manifest.ts +20 -0
- package/src/stt/__tests__/roles.test.ts +152 -0
- package/src/stt/daemon-batch-transcriber.ts +4 -1
- package/src/stt/roles.ts +180 -0
- package/src/stt/stt-stream-session.ts +3 -2
- package/src/stt/types.ts +33 -3
- package/src/subagent/manager.ts +12 -0
- package/src/telemetry/__tests__/live-voice-funnel.test.ts +34 -0
- package/src/telemetry/live-voice-funnel.ts +12 -2
- package/src/telemetry/telemetry-wire.generated.ts +6 -0
- package/src/telemetry/tool-audit.ts +83 -30
- package/src/telemetry/tool-executed-events-store.test.ts +0 -1
- package/src/telemetry/turn-outcome.ts +75 -4
- package/src/tools/client-os.ts +115 -0
- package/src/tools/computer-use/definitions.ts +11 -7
- package/src/tools/computer-use/skill-proxy-bridge.ts +2 -2
- package/src/tools/credentials/broker-types.ts +0 -71
- package/src/tools/credentials/broker.ts +6 -201
- package/src/tools/credentials/metadata-store.ts +65 -16
- package/src/tools/credentials/store.ts +2 -0
- package/src/tools/executor.ts +2 -8
- package/src/tools/host-shell.test.ts +75 -0
- package/src/tools/host-shell.ts +24 -0
- package/src/tools/host-terminal/host-shell.ts +15 -44
- package/src/tools/mcp/mcp-tool-factory.ts +44 -9
- package/src/tools/permission-checker.ts +73 -21
- package/src/tools/policy-context.ts +1 -0
- package/src/tools/shared/filesystem/image-read.ts +1 -1
- package/src/tools/skills/find-similar-skills.ts +10 -2
- package/src/tools/skills/load.ts +41 -6
- package/src/tools/skills/sandbox-runner.ts +8 -16
- package/src/tools/skills/skill-tool-factory.ts +1 -0
- package/src/tools/terminal/__tests__/safe-env.test.ts +64 -1
- package/src/tools/terminal/safe-env.ts +39 -9
- package/src/tools/terminal/shell.ts +12 -31
- package/src/tools/tool-approval-handler.ts +64 -10
- package/src/tools/tool-types.ts +10 -0
- package/src/tools/types.ts +19 -3
- package/src/tools/ui-surface/surface-shape-docs.ts +4 -3
- package/src/tools/workflows/manage-workflows.ts +2 -2
- package/src/tools/workflows/run-workflow.test.ts +2 -1
- package/src/util/__tests__/file-use.test.ts +61 -0
- package/src/util/__tests__/process-table.test.ts +66 -0
- package/src/util/__tests__/process-tree.test.ts +46 -0
- package/src/util/errors.ts +1 -0
- package/src/util/file-use.ts +91 -0
- package/src/util/host-process.test.ts +17 -0
- package/src/util/host-process.ts +75 -0
- package/src/util/image-conversion.ts +39 -7
- package/src/util/platform.ts +32 -13
- package/src/util/process-table.ts +249 -0
- package/src/util/process-tree.ts +85 -129
- package/src/util/provider-error-patterns.ts +20 -0
- package/src/util/spawn.ts +9 -2
- package/src/watch/__tests__/watch-retro.test.ts +48 -1
- package/src/watch/watch-retro.ts +46 -4
- package/src/workspace/git-service.ts +4 -14
- package/src/workspace/migrations/147-rename-colliding-backup-profile-names.ts +360 -0
- package/src/workspace/migrations/148-strip-unsupported-fallback-profiles.ts +117 -0
- package/src/workspace/migrations/149-repoint-backup-profile-selections.ts +204 -0
- package/src/workspace/migrations/150-stt-flux-provider-to-model-family.ts +139 -0
- package/src/workspace/migrations/__tests__/150-stt-flux-provider-to-model-family.test.ts +160 -0
- package/src/workspace/migrations/registry.ts +8 -0
- package/src/__tests__/permission-types.test.ts +0 -17
- package/src/channels/gateway-discord-admission.ts +0 -32
- package/src/persistence/fork-message-copy.ts +0 -239
- package/src/plugins/defaults/memory/__tests__/fork-message-copy.test.ts +0 -221
- package/src/security/__tests__/oauth2-device-code.test.ts +0 -431
- package/src/security/oauth2-device-code.ts +0 -312
|
@@ -105,6 +105,22 @@ mock.module("../vellum-managed-realtime.js", () => ({
|
|
|
105
105
|
},
|
|
106
106
|
}));
|
|
107
107
|
|
|
108
|
+
/** Constructor calls for the managed Flux transcriber (`vellum-flux`). */
|
|
109
|
+
const vellumFluxStreamCtorCalls: Array<{
|
|
110
|
+
connection: unknown;
|
|
111
|
+
options: unknown;
|
|
112
|
+
}> = [];
|
|
113
|
+
|
|
114
|
+
mock.module("../vellum-managed-flux-realtime.js", () => ({
|
|
115
|
+
VellumManagedFluxRealtimeTranscriber: class {
|
|
116
|
+
readonly providerId = "vellum-flux";
|
|
117
|
+
readonly boundaryId = "daemon-streaming";
|
|
118
|
+
constructor(connection: unknown, options: unknown) {
|
|
119
|
+
vellumFluxStreamCtorCalls.push({ connection, options });
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
}));
|
|
123
|
+
|
|
108
124
|
// -- Streaming adapter mocks ----------------------------------------------
|
|
109
125
|
|
|
110
126
|
/**
|
|
@@ -206,6 +222,7 @@ const {
|
|
|
206
222
|
function applyConfig(overrides: {
|
|
207
223
|
provider?: string;
|
|
208
224
|
language?: string;
|
|
225
|
+
roles?: Record<string, { provider: string; model?: string }>;
|
|
209
226
|
}): void {
|
|
210
227
|
const provider = overrides.provider ?? "openai-whisper";
|
|
211
228
|
// Seed a schema-valid base so the loader caches a fresh config object, then
|
|
@@ -230,6 +247,13 @@ function applyConfig(overrides: {
|
|
|
230
247
|
(getConfig().services.stt as { language?: string }).language =
|
|
231
248
|
overrides.language;
|
|
232
249
|
}
|
|
250
|
+
if (overrides.roles !== undefined) {
|
|
251
|
+
(
|
|
252
|
+
getConfig().services.stt as {
|
|
253
|
+
roles?: Record<string, { provider: string; model?: string }>;
|
|
254
|
+
}
|
|
255
|
+
).roles = overrides.roles;
|
|
256
|
+
}
|
|
233
257
|
}
|
|
234
258
|
|
|
235
259
|
// ---------------------------------------------------------------------------
|
|
@@ -486,6 +510,58 @@ describe("resolveTelephonySttCapability", () => {
|
|
|
486
510
|
expect(result.credentialProvider).toBe("gemini");
|
|
487
511
|
}
|
|
488
512
|
});
|
|
513
|
+
|
|
514
|
+
test("answers about the telephony role, not the global provider", async () => {
|
|
515
|
+
// The role's provider is the one a call dials, so preflight must judge
|
|
516
|
+
// that one: a credentialed role behind an uncredentialed global would
|
|
517
|
+
// otherwise be turned away from a call it can serve.
|
|
518
|
+
mockProviderKeys = { deepgram: "dg-key" };
|
|
519
|
+
applyConfig({
|
|
520
|
+
provider: "openai-whisper",
|
|
521
|
+
roles: { telephony: { provider: "deepgram" } },
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
const result = await resolveTelephonySttCapability();
|
|
525
|
+
|
|
526
|
+
expect(result.status).toBe("supported");
|
|
527
|
+
if (result.status === "supported") {
|
|
528
|
+
expect(result.providerId).toBe("deepgram");
|
|
529
|
+
expect(result.telephonyMode).toBe("realtime-ws");
|
|
530
|
+
}
|
|
531
|
+
});
|
|
532
|
+
|
|
533
|
+
test("reports the telephony role's missing credential, not the global's", async () => {
|
|
534
|
+
// The inverse: a credentialed global must not vouch for a role that
|
|
535
|
+
// cannot dial, which passes preflight and then fails at the dial.
|
|
536
|
+
mockProviderKeys = { openai: "oai-key" };
|
|
537
|
+
applyConfig({
|
|
538
|
+
provider: "openai-whisper",
|
|
539
|
+
roles: { telephony: { provider: "deepgram" } },
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
const result = await resolveTelephonySttCapability();
|
|
543
|
+
|
|
544
|
+
expect(result.status).toBe("missing-credentials");
|
|
545
|
+
if (result.status === "missing-credentials") {
|
|
546
|
+
expect(result.providerId).toBe("deepgram");
|
|
547
|
+
expect(result.credentialProvider).toBe("deepgram");
|
|
548
|
+
}
|
|
549
|
+
});
|
|
550
|
+
|
|
551
|
+
test("an unset telephony role still reads the global provider", async () => {
|
|
552
|
+
mockProviderKeys = { openai: "oai-key" };
|
|
553
|
+
applyConfig({
|
|
554
|
+
provider: "openai-whisper",
|
|
555
|
+
roles: { liveVoice: { provider: "deepgram" } },
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
const result = await resolveTelephonySttCapability();
|
|
559
|
+
|
|
560
|
+
expect(result.status).toBe("supported");
|
|
561
|
+
if (result.status === "supported") {
|
|
562
|
+
expect(result.providerId).toBe("openai-whisper");
|
|
563
|
+
}
|
|
564
|
+
});
|
|
489
565
|
});
|
|
490
566
|
|
|
491
567
|
// ---------------------------------------------------------------------------
|
|
@@ -511,11 +587,14 @@ describe("telephony capability catalog alignment", () => {
|
|
|
511
587
|
/**
|
|
512
588
|
* Providers that deliberately sit out telephony (`telephonyMode: "none"`).
|
|
513
589
|
* `deepgram-flux` is a streaming-only spike, and telephony is out of its
|
|
514
|
-
* scope.
|
|
515
|
-
*
|
|
590
|
+
* scope. `vellum-flux` sits out for a concrete reason: the relay maps
|
|
591
|
+
* `Finalize` onto Flux's `CloseStream`, and utterance-boundary finals are a
|
|
592
|
+
* nova-3 concept. Nothing reroutes a call to another provider, so opting
|
|
593
|
+
* out here means a Flux-configured assistant does not transcribe calls.
|
|
516
594
|
*/
|
|
517
595
|
const TELEPHONY_OPT_OUT: ReadonlySet<SttProviderId> = new Set([
|
|
518
596
|
"deepgram-flux",
|
|
597
|
+
"vellum-flux",
|
|
519
598
|
]);
|
|
520
599
|
|
|
521
600
|
test("deepgram catalog entry has realtime-ws telephonyMode", () => {
|
|
@@ -1046,6 +1125,90 @@ describe("resolveStreamingTranscriber diarize preference", () => {
|
|
|
1046
1125
|
});
|
|
1047
1126
|
});
|
|
1048
1127
|
|
|
1128
|
+
describe("vellum-flux managed resolution", () => {
|
|
1129
|
+
beforeEach(() => {
|
|
1130
|
+
mockVellumAvailable = false;
|
|
1131
|
+
mockProviderKeys = {};
|
|
1132
|
+
mockVelayConnection = {
|
|
1133
|
+
wsBaseUrl: "ws://gateway.test",
|
|
1134
|
+
httpBaseUrl: "http://gateway.test",
|
|
1135
|
+
mintServiceToken: () => "vk-test",
|
|
1136
|
+
};
|
|
1137
|
+
vellumFluxStreamCtorCalls.length = 0;
|
|
1138
|
+
});
|
|
1139
|
+
|
|
1140
|
+
test("streaming resolver constructs the managed Flux adapter over the same relay connection", async () => {
|
|
1141
|
+
mockVellumAvailable = true;
|
|
1142
|
+
applyConfig({ provider: "vellum-flux", language: "es" });
|
|
1143
|
+
|
|
1144
|
+
const transcriber = await resolveStreamingTranscriber({
|
|
1145
|
+
sampleRate: 16000,
|
|
1146
|
+
});
|
|
1147
|
+
|
|
1148
|
+
expect(transcriber?.providerId).toBe("vellum-flux");
|
|
1149
|
+
expect(vellumFluxStreamCtorCalls).toEqual([
|
|
1150
|
+
{
|
|
1151
|
+
connection: mockVelayConnection,
|
|
1152
|
+
// The relay picks flux-general-en vs flux-general-multi from the
|
|
1153
|
+
// language, so forwarding it is the only lever over the model.
|
|
1154
|
+
options: { sampleRate: 16000, language: "es" },
|
|
1155
|
+
},
|
|
1156
|
+
]);
|
|
1157
|
+
});
|
|
1158
|
+
|
|
1159
|
+
test("an unset language reaches the relay as code-switching, not English", async () => {
|
|
1160
|
+
// The relay resolves an absent language to flux-general-en, so unset
|
|
1161
|
+
// would be a silent English pin on non-English speech.
|
|
1162
|
+
mockVellumAvailable = true;
|
|
1163
|
+
applyConfig({ provider: "vellum-flux" });
|
|
1164
|
+
|
|
1165
|
+
await resolveStreamingTranscriber({ sampleRate: 16000 });
|
|
1166
|
+
|
|
1167
|
+
expect(
|
|
1168
|
+
(vellumFluxStreamCtorCalls[0]?.options as { language?: string })
|
|
1169
|
+
?.language,
|
|
1170
|
+
).toBe("multi");
|
|
1171
|
+
});
|
|
1172
|
+
|
|
1173
|
+
test("resolves null without a platform connection", async () => {
|
|
1174
|
+
// The credential is the platform connection, not a stored API key.
|
|
1175
|
+
mockVellumAvailable = false;
|
|
1176
|
+
applyConfig({ provider: "vellum-flux" });
|
|
1177
|
+
|
|
1178
|
+
expect(await resolveStreamingTranscriber({ sampleRate: 16000 })).toBeNull();
|
|
1179
|
+
expect(vellumFluxStreamCtorCalls).toHaveLength(0);
|
|
1180
|
+
});
|
|
1181
|
+
|
|
1182
|
+
test("has no batch boundary, like BYOK Flux", async () => {
|
|
1183
|
+
mockVellumAvailable = true;
|
|
1184
|
+
applyConfig({ provider: "vellum-flux" });
|
|
1185
|
+
|
|
1186
|
+
await expect(resolveBatchTranscriber()).rejects.toThrow(SttError);
|
|
1187
|
+
});
|
|
1188
|
+
|
|
1189
|
+
test("refuses a language Flux has no model for, like BYOK Flux", async () => {
|
|
1190
|
+
// The relay rejects the dial for a language outside Flux's roster, which
|
|
1191
|
+
// reaches the caller as an unexplained param error. Refusing here names
|
|
1192
|
+
// the reason where it can be read.
|
|
1193
|
+
mockVellumAvailable = true;
|
|
1194
|
+
applyConfig({ provider: "vellum-flux", language: "ko" });
|
|
1195
|
+
|
|
1196
|
+
expect(await resolveStreamingTranscriber({ sampleRate: 16000 })).toBeNull();
|
|
1197
|
+
expect(vellumFluxStreamCtorCalls).toHaveLength(0);
|
|
1198
|
+
});
|
|
1199
|
+
|
|
1200
|
+
test("serves a language that is on the Flux roster", async () => {
|
|
1201
|
+
mockVellumAvailable = true;
|
|
1202
|
+
applyConfig({ provider: "vellum-flux", language: "hi" });
|
|
1203
|
+
|
|
1204
|
+
const transcriber = await resolveStreamingTranscriber({
|
|
1205
|
+
sampleRate: 16000,
|
|
1206
|
+
});
|
|
1207
|
+
|
|
1208
|
+
expect(transcriber?.providerId).toBe("vellum-flux");
|
|
1209
|
+
});
|
|
1210
|
+
});
|
|
1211
|
+
|
|
1049
1212
|
describe("vellum managed resolution", () => {
|
|
1050
1213
|
beforeEach(() => {
|
|
1051
1214
|
mockVellumAvailable = false;
|
|
@@ -1452,6 +1615,8 @@ describe("streaming capability matches the streaming factory", () => {
|
|
|
1452
1615
|
xai: "xai",
|
|
1453
1616
|
// vellum's credential is the platform connection, mocked below.
|
|
1454
1617
|
vellum: "vellum",
|
|
1618
|
+
// Managed Flux rides the same platform connection.
|
|
1619
|
+
"vellum-flux": "vellum",
|
|
1455
1620
|
};
|
|
1456
1621
|
|
|
1457
1622
|
/** The exact list both user-facing error strings advertise. */
|
|
@@ -1464,16 +1629,18 @@ describe("streaming capability matches the streaming factory", () => {
|
|
|
1464
1629
|
function seedCredentialsFor(id: SttProviderId): void {
|
|
1465
1630
|
const credential = STREAMING_CREDENTIAL[id];
|
|
1466
1631
|
expect(credential).toBeDefined();
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1632
|
+
// Both managed entries authenticate with the platform connection rather
|
|
1633
|
+
// than a stored API key, and both dial through the same relay.
|
|
1634
|
+
const managed = id === "vellum" || id === "vellum-flux";
|
|
1635
|
+
mockVellumAvailable = managed;
|
|
1636
|
+
mockVelayConnection = managed
|
|
1637
|
+
? {
|
|
1638
|
+
wsBaseUrl: "ws://gateway.test",
|
|
1639
|
+
httpBaseUrl: "http://gateway.test",
|
|
1640
|
+
mintServiceToken: () => "vk-test",
|
|
1641
|
+
}
|
|
1642
|
+
: null;
|
|
1643
|
+
mockProviderKeys = managed ? {} : { [credential!]: `key-${id}` };
|
|
1477
1644
|
applyConfig({ provider: id });
|
|
1478
1645
|
}
|
|
1479
1646
|
|
|
@@ -1714,3 +1881,140 @@ describe("the multilingual default reaches configs with no stt block", () => {
|
|
|
1714
1881
|
expect(getConfig().services.stt.language).toBe("ta");
|
|
1715
1882
|
});
|
|
1716
1883
|
});
|
|
1884
|
+
|
|
1885
|
+
// ---------------------------------------------------------------------------
|
|
1886
|
+
// Tests: per-consumer role selection
|
|
1887
|
+
// ---------------------------------------------------------------------------
|
|
1888
|
+
|
|
1889
|
+
describe("services.stt.roles selection", () => {
|
|
1890
|
+
beforeEach(() => {
|
|
1891
|
+
mockVellumAvailable = false;
|
|
1892
|
+
mockVelayConnection = null;
|
|
1893
|
+
mockProviderKeys = { deepgram: "dg-key", openai: "oai-key" };
|
|
1894
|
+
deepgramCtorCalls.length = 0;
|
|
1895
|
+
whisperCtorCalls.length = 0;
|
|
1896
|
+
});
|
|
1897
|
+
|
|
1898
|
+
test("a role override wins over the global provider", async () => {
|
|
1899
|
+
applyConfig({
|
|
1900
|
+
provider: "openai-whisper",
|
|
1901
|
+
roles: { dictation: { provider: "deepgram" } },
|
|
1902
|
+
});
|
|
1903
|
+
|
|
1904
|
+
const transcriber = await resolveStreamingTranscriber({
|
|
1905
|
+
role: "dictation",
|
|
1906
|
+
sampleRate: 16000,
|
|
1907
|
+
});
|
|
1908
|
+
|
|
1909
|
+
expect(transcriber?.providerId).toBe("deepgram");
|
|
1910
|
+
expect(whisperCtorCalls).toHaveLength(0);
|
|
1911
|
+
});
|
|
1912
|
+
|
|
1913
|
+
test("a role with no override falls back to the global provider", async () => {
|
|
1914
|
+
// The invariant that keeps an install with no role set on one provider.
|
|
1915
|
+
applyConfig({
|
|
1916
|
+
provider: "openai-whisper",
|
|
1917
|
+
roles: { dictation: { provider: "deepgram" } },
|
|
1918
|
+
});
|
|
1919
|
+
|
|
1920
|
+
const transcriber = await resolveStreamingTranscriber({
|
|
1921
|
+
role: "liveVoice",
|
|
1922
|
+
sampleRate: 16000,
|
|
1923
|
+
});
|
|
1924
|
+
|
|
1925
|
+
expect(transcriber?.providerId).toBe("openai-whisper");
|
|
1926
|
+
});
|
|
1927
|
+
|
|
1928
|
+
test("a roleless caller still reads the global provider", async () => {
|
|
1929
|
+
applyConfig({
|
|
1930
|
+
provider: "openai-whisper",
|
|
1931
|
+
roles: { dictation: { provider: "deepgram" } },
|
|
1932
|
+
});
|
|
1933
|
+
|
|
1934
|
+
const transcriber = await resolveStreamingTranscriber({
|
|
1935
|
+
sampleRate: 16000,
|
|
1936
|
+
});
|
|
1937
|
+
|
|
1938
|
+
expect(transcriber?.providerId).toBe("openai-whisper");
|
|
1939
|
+
});
|
|
1940
|
+
|
|
1941
|
+
test("an explicit providerId beats the role, since that caller already resolved", async () => {
|
|
1942
|
+
applyConfig({
|
|
1943
|
+
provider: "openai-whisper",
|
|
1944
|
+
roles: { liveVoice: { provider: "openai-whisper" } },
|
|
1945
|
+
});
|
|
1946
|
+
|
|
1947
|
+
const transcriber = await resolveStreamingTranscriber({
|
|
1948
|
+
role: "liveVoice",
|
|
1949
|
+
providerId: "deepgram",
|
|
1950
|
+
sampleRate: 16000,
|
|
1951
|
+
});
|
|
1952
|
+
|
|
1953
|
+
expect(transcriber?.providerId).toBe("deepgram");
|
|
1954
|
+
});
|
|
1955
|
+
|
|
1956
|
+
test("the batch resolver honours its own role", async () => {
|
|
1957
|
+
applyConfig({
|
|
1958
|
+
provider: "openai-whisper",
|
|
1959
|
+
roles: { batch: { provider: "deepgram" } },
|
|
1960
|
+
});
|
|
1961
|
+
|
|
1962
|
+
const transcriber = await resolveBatchTranscriber({ role: "batch" });
|
|
1963
|
+
|
|
1964
|
+
expect(transcriber?.providerId).toBe("deepgram");
|
|
1965
|
+
});
|
|
1966
|
+
|
|
1967
|
+
test("a role selects a model family the global default does not use", async () => {
|
|
1968
|
+
// The case the split exists for: live voice on the streaming-only family
|
|
1969
|
+
// while file transcription keeps the one that can batch.
|
|
1970
|
+
applyConfig({
|
|
1971
|
+
provider: "deepgram",
|
|
1972
|
+
roles: { liveVoice: { provider: "deepgram", model: "flux" } },
|
|
1973
|
+
});
|
|
1974
|
+
|
|
1975
|
+
const live = await resolveStreamingTranscriber({
|
|
1976
|
+
role: "liveVoice",
|
|
1977
|
+
sampleRate: 16000,
|
|
1978
|
+
});
|
|
1979
|
+
const batch = await resolveBatchTranscriber({ role: "batch" });
|
|
1980
|
+
|
|
1981
|
+
expect(live?.providerId).toBe("deepgram-flux");
|
|
1982
|
+
expect(batch?.providerId).toBe("deepgram");
|
|
1983
|
+
});
|
|
1984
|
+
|
|
1985
|
+
test("the watch session resolves its own role", async () => {
|
|
1986
|
+
applyConfig({
|
|
1987
|
+
provider: "openai-whisper",
|
|
1988
|
+
roles: { watch: { provider: "deepgram" } },
|
|
1989
|
+
});
|
|
1990
|
+
|
|
1991
|
+
const transcriber = await resolveStreamingTranscriber({
|
|
1992
|
+
role: "watch",
|
|
1993
|
+
sampleRate: 16000,
|
|
1994
|
+
});
|
|
1995
|
+
|
|
1996
|
+
expect(transcriber?.providerId).toBe("deepgram");
|
|
1997
|
+
expect(whisperCtorCalls).toHaveLength(0);
|
|
1998
|
+
});
|
|
1999
|
+
|
|
2000
|
+
test("roles route two consumers to different providers at once", async () => {
|
|
2001
|
+
// The failure this whole change exists to prevent: one global forcing
|
|
2002
|
+
// every consumer onto a provider that only suits one of them.
|
|
2003
|
+
applyConfig({
|
|
2004
|
+
provider: "openai-whisper",
|
|
2005
|
+
roles: {
|
|
2006
|
+
dictation: { provider: "deepgram" },
|
|
2007
|
+
batch: { provider: "openai-whisper" },
|
|
2008
|
+
},
|
|
2009
|
+
});
|
|
2010
|
+
|
|
2011
|
+
const dictation = await resolveStreamingTranscriber({
|
|
2012
|
+
role: "dictation",
|
|
2013
|
+
sampleRate: 16000,
|
|
2014
|
+
});
|
|
2015
|
+
const batch = await resolveBatchTranscriber({ role: "batch" });
|
|
2016
|
+
|
|
2017
|
+
expect(dictation?.providerId).toBe("deepgram");
|
|
2018
|
+
expect(batch?.providerId).toBe("openai-whisper");
|
|
2019
|
+
});
|
|
2020
|
+
});
|
|
@@ -294,8 +294,13 @@ const EAGER_EOT_THRESHOLD_RANGE = { min: 0.3, max: 0.9 } as const;
|
|
|
294
294
|
const EOT_TIMEOUT_MS_RANGE = { min: 500, max: 60_000 } as const;
|
|
295
295
|
|
|
296
296
|
export interface FluxQueryParamOptions {
|
|
297
|
-
/**
|
|
298
|
-
|
|
297
|
+
/**
|
|
298
|
+
* Flux model to run, e.g. `"flux-general-en"`. Required when dialing
|
|
299
|
+
* Deepgram directly. Omitted on the managed relay, which derives the model
|
|
300
|
+
* from the spoken language and prices it before dialing, and rejects a
|
|
301
|
+
* client-sent one.
|
|
302
|
+
*/
|
|
303
|
+
model?: string;
|
|
299
304
|
/**
|
|
300
305
|
* Encoding of the raw audio being sent. Omit for containerized audio,
|
|
301
306
|
* which carries its own format header.
|
|
@@ -316,6 +321,12 @@ export interface FluxQueryParamOptions {
|
|
|
316
321
|
eagerEotThreshold?: number;
|
|
317
322
|
/** Silence (ms) after which Flux force-ends a turn. */
|
|
318
323
|
eotTimeoutMs?: number;
|
|
324
|
+
/**
|
|
325
|
+
* Language hint for the multilingual model. Meaningless on the English
|
|
326
|
+
* model, and omitted entirely for code-switching, which is what asks Flux
|
|
327
|
+
* to detect the language itself.
|
|
328
|
+
*/
|
|
329
|
+
languageHint?: string;
|
|
319
330
|
}
|
|
320
331
|
|
|
321
332
|
/**
|
|
@@ -336,9 +347,78 @@ export interface FluxQueryParamOptions {
|
|
|
336
347
|
* misconfigured pair from failing the whole session; the adjustment is logged
|
|
337
348
|
* at debug when it fires.
|
|
338
349
|
*/
|
|
350
|
+
/** Flux's English model, and its multilingual sibling. */
|
|
351
|
+
const FLUX_MODEL_ENGLISH = "flux-general-en";
|
|
352
|
+
const FLUX_MODEL_MULTILINGUAL = "flux-general-multi";
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Languages the multilingual Flux model serves, as base subtags.
|
|
356
|
+
*
|
|
357
|
+
* Deepgram documents this roster for `flux-general-multi`; anything outside it
|
|
358
|
+
* has no model to run on.
|
|
359
|
+
*/
|
|
360
|
+
/**
|
|
361
|
+
* Exported so the provider catalog can report it. A client deciding whether to
|
|
362
|
+
* offer turn detection needs the same roster, and copying it there would make
|
|
363
|
+
* a third place to keep in step with this one and the relay's.
|
|
364
|
+
*/
|
|
365
|
+
export const FLUX_MULTILINGUAL_SUBTAGS: ReadonlySet<string> = new Set([
|
|
366
|
+
"de",
|
|
367
|
+
"en",
|
|
368
|
+
"es",
|
|
369
|
+
"fr",
|
|
370
|
+
"hi",
|
|
371
|
+
"it",
|
|
372
|
+
"ja",
|
|
373
|
+
"nl",
|
|
374
|
+
"pt",
|
|
375
|
+
"ru",
|
|
376
|
+
]);
|
|
377
|
+
|
|
378
|
+
/** The code that asks Flux to detect and code-switch rather than be told. */
|
|
379
|
+
const FLUX_CODE_SWITCHING = "multi";
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* The model and optional hint a spoken language selects.
|
|
383
|
+
*
|
|
384
|
+
* Deliberately the same mapping the managed relay applies server-side, so a
|
|
385
|
+
* language means the same thing whether Flux is reached with your own key or
|
|
386
|
+
* through the platform. Without it a BYOK session pins the English model and
|
|
387
|
+
* returns English-sounding nonsense for every other language, which is
|
|
388
|
+
* invisible from the transcript alone.
|
|
389
|
+
*
|
|
390
|
+
* Returns null when the language has no Flux model, leaving the caller to
|
|
391
|
+
* refuse rather than transcribe the wrong thing.
|
|
392
|
+
*/
|
|
393
|
+
export function fluxModelForLanguage(
|
|
394
|
+
language: string | undefined,
|
|
395
|
+
): { model: string; languageHint?: string } | null {
|
|
396
|
+
const normalized = language?.trim().toLowerCase();
|
|
397
|
+
if (!normalized) {
|
|
398
|
+
return { model: FLUX_MODEL_ENGLISH };
|
|
399
|
+
}
|
|
400
|
+
if (normalized === FLUX_CODE_SWITCHING) {
|
|
401
|
+
return { model: FLUX_MODEL_MULTILINGUAL };
|
|
402
|
+
}
|
|
403
|
+
const base = normalized.split("-")[0] ?? normalized;
|
|
404
|
+
if (!FLUX_MULTILINGUAL_SUBTAGS.has(base)) {
|
|
405
|
+
return null;
|
|
406
|
+
}
|
|
407
|
+
if (base === "en") {
|
|
408
|
+
return { model: FLUX_MODEL_ENGLISH };
|
|
409
|
+
}
|
|
410
|
+
return { model: FLUX_MODEL_MULTILINGUAL, languageHint: base };
|
|
411
|
+
}
|
|
412
|
+
|
|
339
413
|
export function buildFluxQueryParams(opts: FluxQueryParamOptions): string {
|
|
340
414
|
const params = new URLSearchParams();
|
|
341
|
-
|
|
415
|
+
if (opts.model !== undefined) {
|
|
416
|
+
params.set("model", opts.model);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
if (opts.languageHint !== undefined) {
|
|
420
|
+
params.set("language_hint", opts.languageHint);
|
|
421
|
+
}
|
|
342
422
|
|
|
343
423
|
if (opts.encoding !== undefined) {
|
|
344
424
|
params.set("encoding", opts.encoding);
|
|
@@ -52,6 +52,7 @@ import { getLogger } from "../../util/logger.js";
|
|
|
52
52
|
import type { FluxEncoding } from "./deepgram-flux-frames.js";
|
|
53
53
|
import {
|
|
54
54
|
buildFluxQueryParams,
|
|
55
|
+
fluxModelForLanguage,
|
|
55
56
|
parseFluxFrame,
|
|
56
57
|
} from "./deepgram-flux-frames.js";
|
|
57
58
|
|
|
@@ -115,6 +116,43 @@ export interface DeepgramFluxRealtimeOptions {
|
|
|
115
116
|
connectTimeoutMs?: number;
|
|
116
117
|
/** Inactivity timeout in milliseconds. Default: 30_000. */
|
|
117
118
|
inactivityTimeoutMs?: number;
|
|
119
|
+
/**
|
|
120
|
+
* WebSocket origin to dial. Defaults to Deepgram's own. The managed relay
|
|
121
|
+
* points this at the gateway, which forwards to velay.
|
|
122
|
+
*/
|
|
123
|
+
baseUrl?: string;
|
|
124
|
+
/** Path on {@link baseUrl}. Defaults to Deepgram's `/v2/listen`. */
|
|
125
|
+
path?: string;
|
|
126
|
+
/**
|
|
127
|
+
* Carry the credential in a `key` query param rather than an
|
|
128
|
+
* `Authorization` header, which is how the gateway relay authenticates.
|
|
129
|
+
* Turns on URL redaction for logs.
|
|
130
|
+
*/
|
|
131
|
+
queryAuth?: boolean;
|
|
132
|
+
/**
|
|
133
|
+
* Omit `model` from the query. The managed relay derives the Flux model
|
|
134
|
+
* from the spoken language and prices it before dialing, so it rejects a
|
|
135
|
+
* client-selected one.
|
|
136
|
+
*/
|
|
137
|
+
omitModelParam?: boolean;
|
|
138
|
+
/**
|
|
139
|
+
* Ask the relay for native Flux frames instead of the v1-shaped Deepgram
|
|
140
|
+
* translation it serves released assistants by default. Meaningless when
|
|
141
|
+
* dialing Deepgram directly, which only speaks Flux.
|
|
142
|
+
*/
|
|
143
|
+
nativeContract?: boolean;
|
|
144
|
+
/**
|
|
145
|
+
* Spoken language, forwarded to the relay so it can select the Flux model
|
|
146
|
+
* and language hint. Deepgram's own endpoint takes no `language`, so this
|
|
147
|
+
* is only sent when set, which is the managed path.
|
|
148
|
+
*/
|
|
149
|
+
language?: string;
|
|
150
|
+
/**
|
|
151
|
+
* Called with any JSON frame the Flux parser produced no events from. The
|
|
152
|
+
* managed relay multiplexes its own `velay_error` control frames onto the
|
|
153
|
+
* same socket; this is how the wrapper sees them.
|
|
154
|
+
*/
|
|
155
|
+
onUnhandledFrame?: (frame: Record<string, unknown>) => void;
|
|
118
156
|
}
|
|
119
157
|
|
|
120
158
|
// ---------------------------------------------------------------------------
|
|
@@ -165,6 +203,15 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
|
|
|
165
203
|
private readonly sampleRate: number;
|
|
166
204
|
private readonly connectTimeoutMs: number;
|
|
167
205
|
private readonly inactivityTimeoutMs: number;
|
|
206
|
+
private readonly baseUrl: string;
|
|
207
|
+
private readonly path: string;
|
|
208
|
+
private readonly queryAuth: boolean;
|
|
209
|
+
private readonly omitModelParam: boolean;
|
|
210
|
+
private readonly nativeContract: boolean;
|
|
211
|
+
private readonly language: string | undefined;
|
|
212
|
+
private readonly onUnhandledFrame:
|
|
213
|
+
| ((frame: Record<string, unknown>) => void)
|
|
214
|
+
| undefined;
|
|
168
215
|
|
|
169
216
|
/** The live WebSocket connection, set during start(). */
|
|
170
217
|
private ws: WsLike | null = null;
|
|
@@ -211,6 +258,29 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
|
|
|
211
258
|
options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
|
|
212
259
|
this.inactivityTimeoutMs =
|
|
213
260
|
options.inactivityTimeoutMs ?? DEFAULT_INACTIVITY_TIMEOUT_MS;
|
|
261
|
+
this.baseUrl = (options.baseUrl ?? WS_BASE_URL).replace(/\/+$/, "");
|
|
262
|
+
this.path = options.path ?? FLUX_PATH;
|
|
263
|
+
this.queryAuth = options.queryAuth ?? false;
|
|
264
|
+
this.omitModelParam = options.omitModelParam ?? false;
|
|
265
|
+
this.nativeContract = options.nativeContract ?? false;
|
|
266
|
+
this.language = options.language;
|
|
267
|
+
this.onUnhandledFrame = options.onUnhandledFrame;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Blank the credential in text bound for a log. Only query auth puts it in
|
|
272
|
+
* a URL; the header path never does, so this is a no-op there.
|
|
273
|
+
*/
|
|
274
|
+
private redact(text: string): string {
|
|
275
|
+
if (!this.queryAuth || !text) {
|
|
276
|
+
return text;
|
|
277
|
+
}
|
|
278
|
+
return text
|
|
279
|
+
.split(this.apiKey)
|
|
280
|
+
.join("***")
|
|
281
|
+
.split(encodeURIComponent(this.apiKey))
|
|
282
|
+
.join("***")
|
|
283
|
+
.replace(/([?&]key=)[^&\s"']*/g, "$1***");
|
|
214
284
|
}
|
|
215
285
|
|
|
216
286
|
// ── StreamingTranscriber interface ──────────────────────────────────
|
|
@@ -222,7 +292,7 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
|
|
|
222
292
|
this.onEvent = onEvent;
|
|
223
293
|
|
|
224
294
|
const url = this.buildWebSocketUrl();
|
|
225
|
-
log.info({ url }, "Opening Deepgram Flux session");
|
|
295
|
+
log.info({ url: this.redact(url) }, "Opening Deepgram Flux session");
|
|
226
296
|
|
|
227
297
|
const ws = this.createWebSocket(url);
|
|
228
298
|
this.ws = ws;
|
|
@@ -369,6 +439,10 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
|
|
|
369
439
|
if (typeof WebSocketCtor !== "function") {
|
|
370
440
|
throw new Error("global WebSocket is not available in this runtime");
|
|
371
441
|
}
|
|
442
|
+
// Query auth carries the credential in the URL instead.
|
|
443
|
+
if (this.queryAuth) {
|
|
444
|
+
return new WebSocketCtor(url);
|
|
445
|
+
}
|
|
372
446
|
return new WebSocketCtor(url, {
|
|
373
447
|
headers: { Authorization: `Token ${this.apiKey}` },
|
|
374
448
|
});
|
|
@@ -415,7 +489,13 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
|
|
|
415
489
|
return;
|
|
416
490
|
}
|
|
417
491
|
|
|
418
|
-
|
|
492
|
+
const events = parseFluxFrame(raw);
|
|
493
|
+
if (events.length === 0) {
|
|
494
|
+
this.reportUnhandledFrame(raw);
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
for (const event of events) {
|
|
419
499
|
if (event.type === "error") {
|
|
420
500
|
this.fatalErrorReported = true;
|
|
421
501
|
}
|
|
@@ -423,6 +503,27 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
|
|
|
423
503
|
}
|
|
424
504
|
}
|
|
425
505
|
|
|
506
|
+
/**
|
|
507
|
+
* Hand a frame the Flux parser made nothing of to the transport wrapper.
|
|
508
|
+
* On the direct Deepgram path nothing listens and this is inert; on the
|
|
509
|
+
* managed relay it is how `velay_error` reaches the caller.
|
|
510
|
+
*/
|
|
511
|
+
private reportUnhandledFrame(raw: string): void {
|
|
512
|
+
if (!this.onUnhandledFrame) {
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
let frame: unknown;
|
|
516
|
+
try {
|
|
517
|
+
frame = JSON.parse(raw);
|
|
518
|
+
} catch {
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
if (typeof frame !== "object" || frame === null || Array.isArray(frame)) {
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
this.onUnhandledFrame(frame as Record<string, unknown>);
|
|
525
|
+
}
|
|
526
|
+
|
|
426
527
|
/** Handle provider-side WebSocket close. */
|
|
427
528
|
private handleProviderClose(code: number, reason: string): void {
|
|
428
529
|
if (this.closed) {
|
|
@@ -635,15 +736,42 @@ export class DeepgramFluxRealtimeTranscriber implements StreamingTranscriber {
|
|
|
635
736
|
* {@link buildFluxQueryParams}.
|
|
636
737
|
*/
|
|
637
738
|
private buildWebSocketUrl(): string {
|
|
739
|
+
// Dialing Deepgram directly means choosing the model here. The relay
|
|
740
|
+
// derives it from the language instead, which is why the managed path
|
|
741
|
+
// sends neither.
|
|
742
|
+
const selection = this.omitModelParam
|
|
743
|
+
? null
|
|
744
|
+
: // An explicit pin is the operator's call and wins; otherwise the
|
|
745
|
+
// language picks, matching what the relay does server-side.
|
|
746
|
+
this.flux.model
|
|
747
|
+
? { model: this.flux.model }
|
|
748
|
+
: fluxModelForLanguage(this.language);
|
|
638
749
|
const query = buildFluxQueryParams({
|
|
639
|
-
model:
|
|
750
|
+
...(selection ? { model: selection.model } : {}),
|
|
751
|
+
...(selection?.languageHint !== undefined
|
|
752
|
+
? { languageHint: selection.languageHint }
|
|
753
|
+
: {}),
|
|
640
754
|
encoding: AUDIO_ENCODING,
|
|
641
755
|
sampleRate: this.sampleRate,
|
|
642
756
|
eotThreshold: this.flux.eotThreshold,
|
|
643
757
|
eagerEotThreshold: this.flux.eagerEotThreshold,
|
|
644
758
|
eotTimeoutMs: this.flux.eotTimeoutMs,
|
|
645
759
|
});
|
|
646
|
-
|
|
760
|
+
|
|
761
|
+
// Relay-only params. Deepgram takes neither: `language` is how velay
|
|
762
|
+
// picks the Flux model and language hint, and `contract` asks it to stop
|
|
763
|
+
// translating Flux into the v1 Deepgram dialect.
|
|
764
|
+
const params = new URLSearchParams(query);
|
|
765
|
+
if (this.language) {
|
|
766
|
+
params.set("language", this.language);
|
|
767
|
+
}
|
|
768
|
+
if (this.nativeContract) {
|
|
769
|
+
params.set("contract", "flux");
|
|
770
|
+
}
|
|
771
|
+
if (this.queryAuth) {
|
|
772
|
+
params.set("key", this.apiKey);
|
|
773
|
+
}
|
|
774
|
+
return `${this.baseUrl}${this.path}?${params.toString()}`;
|
|
647
775
|
}
|
|
648
776
|
}
|
|
649
777
|
|