@vellumai/assistant 0.11.6 → 0.11.7-staging.2

package/src/providers/speech-to-text/provider-catalog.ts

@@ -13,11 +13,13 @@
 import type {
   ConversationStreamingMode,
   SttBoundaryId,
+  SttModelFamily,
   SttProviderId,
   SttTurnDetectionMode,
   TelephonySttMode,
 } from "../../stt/types.js";
 import { baseLanguageSubtag } from "../../util/language-subtag.js";
+import { FLUX_MULTILINGUAL_SUBTAGS } from "./deepgram-flux-frames.js";
 
 // ---------------------------------------------------------------------------
 // Client display metadata
@@ -122,6 +124,23 @@ interface SttProviderEntry {
 
   /** Guide for obtaining API credentials from this provider. */
   readonly credentialsGuide?: SttCredentialsGuide;
+
+  /**
+   * Set when this row is a model family of another provider rather than a
+   * provider in its own right: the id a user selects, plus the
+   * `services.stt.providers.<id>.model` value that reaches this row.
+   *
+   * Rows carrying this are not separately selectable. They share the parent's
+   * credential and appear nowhere in a provider picker.
+   */
+  readonly variantOf?: SttProviderId;
+  readonly variantModel?: SttModelFamily;
+
+  /**
+   * The family this row runs when no `model` is set. Present only on rows that
+   * have variants, so the default is nameable in config and in errors.
+   */
+  readonly baseModelFamily?: SttModelFamily;
 }
 
 // ---------------------------------------------------------------------------
@@ -156,6 +175,7 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
     "deepgram",
     {
       id: "deepgram",
+      baseModelFamily: "nova-3",
       displayName: "Deepgram",
       subtitle:
         "Fast, real-time speech-to-text with streaming support. Requires a Deepgram API key.",
@@ -178,6 +198,8 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
     "deepgram-flux",
     {
       id: "deepgram-flux",
+      variantOf: "deepgram",
+      variantModel: "flux",
       displayName: "Deepgram Flux",
       subtitle:
         "Conversational speech-to-text with model-native turn detection. Uses your Deepgram API key.",
@@ -198,9 +220,11 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
       // them, so a live-voice session may let them commit the turn.
       turnDetection: "provider",
       supportsDiarization: false,
-      // "no picker", not native detection: the Flux model is English-only and
-      // takes no language parameter, so audio in another language transcribes
-      // as English rather than being detected.
+      // "no picker", not native detection: this entry pins the English Flux
+      // model and sends no language parameter, so audio in another language
+      // transcribes as English. Flux itself has a multilingual model; the
+      // managed `vellum-flux` entry reaches it, and a BYOK picker would need
+      // the model to follow the selection.
       languageSelection: "auto",
       credentialsGuide: DEEPGRAM_CREDENTIALS_GUIDE,
     },
@@ -264,6 +288,7 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
     "vellum",
     {
       id: "vellum",
+      baseModelFamily: "nova-3",
       displayName: "Vellum",
       subtitle:
         "Speech-to-text through your Vellum account — billed to Vellum credits, no separate API key needed.",
@@ -283,6 +308,39 @@ const CATALOG: ReadonlyMap<SttProviderId, SttProviderEntry> = new Map<
       languageSelection: "manual",
     },
   ],
+  [
+    "vellum-flux",
+    {
+      id: "vellum-flux",
+      variantOf: "vellum",
+      variantModel: "flux",
+      displayName: "Vellum (Flux)",
+      subtitle:
+        "Conversational speech-to-text with model-native turn detection, through your Vellum account. No separate API key needed.",
+      setupMode: "connection",
+      setupHint: "Connect your Vellum account to enable managed transcription.",
+      // Shared with `vellum`: the platform connection is the credential, and
+      // Flux is a different endpoint on the same relay, not a second account.
+      credentialProvider: "vellum",
+      // Streaming only, same as the BYOK Flux entry: Flux has no batch
+      // endpoint, and the relay exposes none.
+      supportedBoundaries: new Set<SttBoundaryId>(["daemon-streaming"]),
+      // The relay maps `Finalize` onto Flux's `CloseStream`, which ends the
+      // session rather than flushing it, and telephony's utterance-boundary
+      // finals are a nova-3 concept. Calls stay on `vellum`.
+      telephonyMode: "none",
+      conversationStreamingMode: "realtime-ws",
+      // The relay is dialed with `contract=flux`, so Flux's turn lifecycle
+      // events arrive intact and a live-voice session may let them commit
+      // the turn. Without that opt-in the relay translates them away and
+      // this would be false.
+      turnDetection: "provider",
+      supportsDiarization: false,
+      // The relay selects `flux-general-en` or `flux-general-multi` from the
+      // language it is sent, so unlike BYOK Flux the picker is meaningful.
+      languageSelection: "manual",
+    },
+  ],
   [
     "xai",
     {
@@ -440,6 +498,136 @@ export function supportsDiarization(id: SttProviderId): boolean {
  * provider, so a new turn-detecting provider is a catalog entry rather than a
  * session change.
  */
+/**
+ * Whether a provider authenticates with the Vellum platform connection rather
+ * than a stored API key.
+ *
+ * Derived from the catalog rather than a hand-kept id list: every check that
+ * gates managed speech reads this, so adding a managed provider cannot leave
+ * one of them behind on a stale literal.
+ */
+export function isManagedSttProvider(id: SttProviderId): boolean {
+  return CATALOG.get(id)?.credentialProvider === "vellum";
+}
+
+/**
+ * The catalog row a configured provider plus model family resolves to.
+ *
+ * Falls back to the provider's own row when it offers no matching variant, so
+ * a model setting a provider does not implement is inert rather than fatal.
+ * The parameter is structural to keep the catalog free of config imports.
+ */
+export function resolveSttCatalogKey(stt: {
+  readonly provider: string;
+  readonly providers?:
+    | Record<string, { readonly model?: unknown } | undefined>
+    | undefined;
+}): SttProviderId {
+  const provider = stt.provider as SttProviderId;
+  return sttCatalogKeyFor(provider, stt.providers?.[provider]?.model);
+}
+
+/**
+ * The catalog row a provider and model family pair resolves to.
+ *
+ * Split out from {@link resolveSttCatalogKey} because a selection does not
+ * always come from the top-level config block: a per-consumer role names its
+ * own provider and family, and has to reach the same row that pair would
+ * reach anywhere else.
+ */
+export function sttCatalogKeyFor(
+  provider: SttProviderId,
+  model: unknown,
+): SttProviderId {
+  if (typeof model !== "string") {
+    return provider;
+  }
+  for (const entry of CATALOG.values()) {
+    if (entry.variantOf === provider && entry.variantModel === model) {
+      return entry.id;
+    }
+  }
+  return provider;
+}
+
+/**
+ * The model families a provider offers, as `services.stt.providers.<id>.model`
+ * values. A provider with a single family returns an empty list, and setting
+ * `model` on it is a configuration error rather than a silent no-op.
+ */
+export function listProviderModelFamilies(
+  id: SttProviderId,
+): readonly SttModelFamily[] {
+  const base = CATALOG.get(id);
+  if (!base || base.variantOf !== undefined) {
+    return [];
+  }
+  const families = [...CATALOG.values()]
+    .filter((entry) => entry.variantOf === id && entry.variantModel)
+    .map((entry) => entry.variantModel as SttModelFamily);
+  return families.length > 0
+    ? [base.baseModelFamily ?? "nova-3", ...families]
+    : [];
+}
+
+/**
+ * The config that selects a catalog row: the provider id a user sets, plus
+ * the model family when the row is a variant.
+ *
+ * The inverse of {@link resolveSttCatalogKey}. Anything persisting a resolved
+ * provider has to write these two fields rather than the key, which is not a
+ * valid `services.stt.provider` value for a variant row.
+ */
+export function sttConfigForCatalogKey(id: SttProviderId): {
+  provider: SttProviderId;
+  model?: SttModelFamily;
+} {
+  const entry = CATALOG.get(id);
+  if (!entry?.variantOf || !entry.variantModel) {
+    return { provider: id };
+  }
+  return { provider: entry.variantOf, model: entry.variantModel };
+}
+
+/**
+ * The family a provider runs when no `model` is set, for callers that must
+ * write an explicit value rather than leave a stale one in place. Providers
+ * with a single family have no name to write and return undefined.
+ */
+/**
+ * The spoken languages a provider's turn-detecting model family serves, or
+ * undefined when it has no such family.
+ *
+ * A family that owns the turn boundary does not necessarily cover the
+ * provider's whole language roster, and dialing it outside that set fails
+ * rather than degrades. Reported from here so the answer has one home.
+ */
+export function turnDetectionLanguagesFor(
+  id: SttProviderId,
+): readonly string[] | undefined {
+  const variant = [...CATALOG.values()].find(
+    (entry) => entry.variantOf === id && entry.turnDetection === "provider",
+  );
+  return variant ? [...FLUX_MULTILINGUAL_SUBTAGS].sort() : undefined;
+}
+
+export function baseModelFamilyFor(
+  id: SttProviderId,
+): SttModelFamily | undefined {
+  const entry = CATALOG.get(id);
+  return entry?.variantOf === undefined ? entry?.baseModelFamily : undefined;
+}
+
+/**
+ * Provider ids a user may select. Excludes model-family rows, which are
+ * reached through `services.stt.providers.<id>.model` instead.
+ */
+export function listSelectableProviderIds(): readonly SttProviderId[] {
+  return [...CATALOG.values()]
+    .filter((entry) => entry.variantOf === undefined)
+    .map((entry) => entry.id);
+}
+
 export function supportsProviderTurnDetection(id: SttProviderId): boolean {
   return CATALOG.get(id)?.turnDetection === "provider";
 }

package/src/providers/speech-to-text/resolve.ts

@@ -1,6 +1,7 @@
 import { getConfig } from "../../config/loader.js";
 import { getProviderKeyAsync } from "../../security/secure-keys.js";
 import { createDaemonBatchTranscriber } from "../../stt/daemon-batch-transcriber.js";
+import { sttCatalogKeyForRole, type SttRole } from "../../stt/roles.js";
 import type {
   BatchTranscriber,
   StreamingTranscriber,
@@ -12,6 +13,8 @@ import {
   batchBoundaryGapReason,
   getCredentialProvider,
   getProviderEntry,
+  isManagedSttProvider,
+  resolveSttCatalogKey,
   supportsBoundary,
   supportsDiarization,
 } from "./provider-catalog.js";
@@ -35,6 +38,10 @@ const log = getLogger("stt-resolver");
 const MULTILINGUAL_DEFAULT_PROVIDERS: ReadonlySet<SttProviderId> = new Set([
   "deepgram",
   "vellum",
+  // Managed Flux resolves an absent language to `flux-general-en` relay-side,
+  // so unset is an English pin here too. With "multi" the relay selects
+  // `flux-general-multi` and lets Flux detect and code-switch.
+  "vellum-flux",
 ] as SttProviderId[]);
 
 /**
@@ -76,6 +83,29 @@ export function effectiveSttLanguage(
     : undefined;
 }
 
+/**
+ * Record which provider a role actually dialed, when that differs from the
+ * global setting.
+ *
+ * `assistant config get` reports what was set, not what resolved, and managed
+ * defaulting already substitutes silently. A role override adds a second way
+ * for the two to diverge, so the divergence is logged rather than left to be
+ * inferred from a provider's behaviour.
+ */
+function logRoleSelection(
+  role: SttRole | undefined,
+  resolved: SttProviderId,
+  base: SttProviderId,
+): void {
+  if (role === undefined || resolved === base) {
+    return;
+  }
+  log.info(
+    { role, providerId: resolved, baseProviderId: base },
+    "STT role override selected a provider other than services.stt.provider",
+  );
+}
+
 // ---------------------------------------------------------------------------
 // Batch transcriber resolver (existing public API — unchanged contract)
 // ---------------------------------------------------------------------------
@@ -98,11 +128,14 @@ export function effectiveSttLanguage(
  * "no speech-to-text provider is configured" copy every caller pairs with
  * `null`, which is the opposite of what happened.
  */
-export async function resolveBatchTranscriber(): Promise<BatchTranscriber | null> {
+export async function resolveBatchTranscriber(
+  options: { role?: SttRole } = {},
+): Promise<BatchTranscriber | null> {
   // Snapshot the stt config once, before any await, so a concurrent config
   // change cannot pair one setting's old value with another's new value.
   const stt = getConfig().services.stt;
-  const provider = stt.provider;
+  const provider = sttCatalogKeyForRole(stt, options.role);
+  logRoleSelection(options.role, provider, resolveSttCatalogKey(stt));
   const language = effectiveSttLanguage(
     provider as SttProviderId,
     stt.language,
@@ -177,22 +210,27 @@ export type TelephonySttCapability =
     };
 
 /**
- * Validate whether the configured `services.stt` provider is eligible for
- * future real-time telephony call ingestion.
+ * Validate whether the STT provider serving telephony is eligible for
+ * real-time call ingestion.
  *
- * This resolver does **not** create a live transcriber — it only validates
+ * This resolver does **not** create a live transcriber: it only validates
  * that the configuration, catalog entry, and credentials are all in order.
- * The actual wiring is deferred to a future media-stream call adapter PR.
+ *
+ * The provider is the telephony role's, which is the one a call actually
+ * dials. Reading the global provider instead would answer about a provider
+ * no call uses: a credentialed role selection would be turned away for the
+ * global's missing key, and the inverse would pass preflight and fail at the
+ * dial.
  *
  * Callers can branch on the discriminated `status` field:
- * - `"supported"` — the provider is telephony-eligible and credentials exist.
- * - `"unsupported"` — the provider exists but has `telephonyMode: "none"`.
- * - `"unconfigured"` — the provider is unknown or missing from the catalog.
- * - `"missing-credentials"` — the provider is eligible but has no API key.
+ * - `"supported"`: the provider is telephony-eligible and credentials exist.
+ * - `"unsupported"`: the provider exists but has `telephonyMode: "none"`.
+ * - `"unconfigured"`: the provider is unknown or missing from the catalog.
+ * - `"missing-credentials"`: the provider is eligible but has no API key.
  */
 export async function resolveTelephonySttCapability(): Promise<TelephonySttCapability> {
   const config = getConfig();
-  const provider = config.services.stt.provider;
+  const provider = sttCatalogKeyForRole(config.services.stt, "telephony");
 
   const entry = getProviderEntry(provider as SttProviderId);
   if (!entry) {
@@ -281,7 +319,7 @@ export type ConversationStreamingSttCapability =
  */
 export async function resolveConversationStreamingSttCapability(): Promise<ConversationStreamingSttCapability> {
   const config = getConfig();
-  const provider = config.services.stt.provider;
+  const provider = resolveSttCatalogKey(config.services.stt);
 
   const entry = getProviderEntry(provider as SttProviderId);
   if (!entry) {
@@ -347,6 +385,13 @@ export interface ResolveStreamingTranscriberOptions {
    * readiness check approved.
    */
   providerId?: SttProviderId;
+  /**
+   * Consumer asking for the transcriber. Selects that role's
+   * `services.stt.roles` override; omitted falls back to the global
+   * `services.stt.provider`. Ignored when `providerId` is given, since that
+   * caller already resolved the provider itself.
+   */
+  role?: SttRole;
   /**
    * Speaker diarization preference. Default: `"off"`.
    *
@@ -409,7 +454,11 @@ export async function resolveStreamingTranscriber(
   // change cannot pair one setting's old value with another's new value
   // (e.g. the old provider with the new language).
   const stt = getConfig().services.stt;
-  const provider = options.providerId ?? (stt.provider as SttProviderId);
+  const provider =
+    options.providerId ?? sttCatalogKeyForRole(stt, options.role);
+  if (options.providerId === undefined) {
+    logRoleSelection(options.role, provider, resolveSttCatalogKey(stt));
+  }
   // Config-level language applies to every streaming caller (live voice,
   // dictation, telephony) unless one overrides it for a single session, so
   // the setting lands in one place rather than at each call site. An unset
@@ -466,11 +515,14 @@ export async function resolveStreamingTranscriber(
     (diarizePreference === "preferred" || diarizePreference === "required") &&
     providerSupportsDiarization;
 
-  const apiKey =
-    provider === "vellum"
-      ? null
-      : await getProviderKeyAsync(credentialProviderName);
-  if (provider === "vellum") {
+  // Both managed entries authenticate with the platform connection, not a
+  // stored API key: there is no key to fetch, and readiness is whether the
+  // connection resolves.
+  const managed = isManagedSttProvider(provider);
+  const apiKey = managed
+    ? null
+    : await getProviderKeyAsync(credentialProviderName);
+  if (managed) {
     if (!(await sttProviderKeyResolves("vellum"))) {
       return null;
     }
@@ -641,18 +693,74 @@ async function createStreamingTranscriber(
           : {}),
       });
     }
+    case "vellum-flux": {
+      // Managed Flux: same relay as `vellum`, but the STT v2 endpoint with
+      // `contract=flux` so the turn events arrive intact instead of being
+      // translated into the released v1 Deepgram dialect. Gated on the local
+      // platform connection for the same reason as `vellum` above.
+      const { vellumManagedSpeechAvailable } =
+        await import("./vellum-managed.js");
+      if (!(await vellumManagedSpeechAvailable())) {
+        return null;
+      }
+      // The relay maps the language onto a Flux model and refuses the dial
+      // when there is none, so a language outside Flux's roster fails as a
+      // relay param error the caller cannot explain. Refuse here instead, on
+      // the same rule the BYOK twin above applies, so the reason is named
+      // where it can be read.
+      const { fluxModelForLanguage: managedFluxModelForLanguage } =
+        await import("./deepgram-flux-frames.js");
+      if (managedFluxModelForLanguage(options.language) === null) {
+        log.warn(
+          { providerId, language: options.language },
+          "Managed Flux has no model for the configured language; refusing rather than transcribing it as another",
+        );
+        return null;
+      }
+      const { resolveSpeechRelayConnection } =
+        await import("./vellum-speech-relay-connection.js");
+      const connection = await resolveSpeechRelayConnection();
+      if (!connection) {
+        return null;
+      }
+      const { VellumManagedFluxRealtimeTranscriber } =
+        await import("./vellum-managed-flux-realtime.js");
+      return new VellumManagedFluxRealtimeTranscriber(connection, {
+        sampleRate: options.sampleRate,
+        // The relay picks `flux-general-en` or `flux-general-multi` from
+        // this, so it is the only lever over the Flux model from here.
+        // `utteranceBoundaryFinals` is a nova-3 concept and the catalog
+        // keeps this provider off telephony, so it never arrives.
+        ...(options.language ? { language: options.language } : {}),
+      });
+    }
     case "deepgram-flux": {
       // Flux is streaming-only and dials Deepgram's /v2/listen conversational
-      // endpoint. Turn-detection tuning comes from `liveVoice.flux`, which
-      // the adapter reads itself, so only the transport-level sample rate is
-      // passed here. No language is forwarded: the spike pins the
-      // English-only `flux-general-en`, and `language_hint` means nothing to
-      // a monolingual model. Diarization is off in the catalog, so a
-      // `"required"` caller never reaches this case.
+      // endpoint. Turn-detection tuning comes from `liveVoice.flux`, which the
+      // adapter reads itself. The language picks the model, so a language Flux
+      // has no model for resolves to nothing rather than being transcribed by
+      // the English one, which returns fluent-looking nonsense the transcript
+      // gives no sign of. Diarization is off in the catalog, so a `"required"`
+      // caller never reaches this case.
+      const { fluxModelForLanguage } =
+        await import("./deepgram-flux-frames.js");
+      // A pinned model is the operator saying which one to run, so the
+      // language check only guards the derived case.
+      if (
+        getConfig().liveVoice.flux.model === undefined &&
+        fluxModelForLanguage(options.language) === null
+      ) {
+        log.warn(
+          { providerId, language: options.language },
+          "Deepgram Flux has no model for the configured language; refusing rather than transcribing it as another",
+        );
+        return null;
+      }
       const { DeepgramFluxRealtimeTranscriber } =
         await import("./deepgram-flux-realtime.js");
       return new DeepgramFluxRealtimeTranscriber(apiKey, {
         sampleRate: options.sampleRate,
+        ...(options.language ? { language: options.language } : {}),
       });
     }
     default: {