@vellumai/assistant 0.11.3-staging.2 → 0.11.3-staging.3

package/src/providers/speech-to-text/deepgram-realtime.test.ts

@@ -102,7 +102,11 @@ function resultsFrame(
     is_final?: boolean;
     speech_final?: boolean;
     from_finalize?: boolean;
-    words?: { word: string; speaker?: number }[];
+    words?: { word: string; speaker?: number; language?: string }[];
+    /** Container-level detected languages on the alternative. */
+    alternativeLanguages?: string[];
+    /** Container-level detected languages on the channel. */
+    channelLanguages?: string[];
   } = {},
 ): string {
   return JSON.stringify({
@@ -121,8 +125,14 @@ function resultsFrame(
           transcript,
           confidence: 0.95,
           ...(options.words ? { words: options.words } : {}),
+          ...(options.alternativeLanguages
+            ? { languages: options.alternativeLanguages }
+            : {}),
         },
       ],
+      ...(options.channelLanguages
+        ? { languages: options.channelLanguages }
+        : {}),
     },
   });
 }
@@ -542,6 +552,154 @@ describe("DeepgramRealtimeTranscriber", () => {
     (globalThis as Record<string, unknown>).WebSocket = origWs;
   });
 
+  // ─────────────────────────────────────────────────────────────────
+  // Language metadata (nova-3 multi code-switching)
+  // ─────────────────────────────────────────────────────────────────
+
+  describe("language metadata", () => {
+    test("ranks per-word language tags by dominance on final events", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hello world hola", {
+          is_final: true,
+          words: [
+            { word: "hello", language: "en" },
+            { word: "world", language: "en" },
+            { word: "hola", language: "es" },
+          ],
+        }),
+      );
+
+      expect(events).toHaveLength(1);
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "hello world hola",
+        confidence: 0.95,
+        languages: ["en", "es"],
+      });
+    });
+
+    test("omits the languages field entirely when no language metadata is present", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hello world", {
+          is_final: true,
+          words: [{ word: "hello" }, { word: "world" }],
+        }),
+      );
+      mockWs.simulateMessage(resultsFrame("still typing", { is_final: false }));
+
+      expect(events).toHaveLength(2);
+      for (const event of events) {
+        // The keys must not exist at all, not just be undefined-valued.
+        expect("language" in event).toBe(false);
+        expect("languages" in event).toBe(false);
+      }
+    });
+
+    test("normalizes regional tags to their base subtag", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          words: [{ word: "hello", language: "en-US" }],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "hello",
+        confidence: 0.95,
+        languages: ["en"],
+      });
+    });
+
+    test("partial events carry the fields when interim results are enabled", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("namaste hello", {
+          is_final: false,
+          words: [
+            { word: "namaste", language: "hi" },
+            { word: "hello", language: "en" },
+            { word: "there", language: "en" },
+          ],
+        }),
+      );
+
+      expect(events).toHaveLength(1);
+      expect(events[0]).toEqual({
+        type: "partial",
+        text: "namaste hello",
+        confidence: 0.95,
+        languages: ["en", "hi"],
+      });
+    });
+
+    test("falls back to the alternative-level languages array when words carry no tags", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("mixed speech", {
+          is_final: true,
+          words: [{ word: "mixed" }, { word: "speech" }],
+          alternativeLanguages: ["es-419", "en", "ES"],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "mixed speech",
+        confidence: 0.95,
+        languages: ["es", "en"],
+      });
+    });
+
+    test("falls back to the channel-level languages array when the alternative has none", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("bonjour", {
+          is_final: true,
+          channelLanguages: ["fr", "en"],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "bonjour",
+        confidence: 0.95,
+        languages: ["fr", "en"],
+      });
+    });
+
+    test("per-word tags take precedence over container arrays", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hola amigo", {
+          is_final: true,
+          words: [
+            { word: "hola", language: "es" },
+            { word: "amigo", language: "es" },
+          ],
+          alternativeLanguages: ["en", "es"],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "hola amigo",
+        confidence: 0.95,
+        languages: ["es"],
+      });
+    });
+  });
+
   // ─────────────────────────────────────────────────────────────────
   // Multi-event sequence
   // ─────────────────────────────────────────────────────────────────
@@ -671,6 +829,130 @@ describe("DeepgramRealtimeTranscriber", () => {
         "second utterance",
       ]);
     });
+
+    test("aggregated final ranks language tags across all withheld frames", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      // Raw per-word tags are accumulated, so "es" (2 words) outranks
+      // "en" (1 word) even though "en" arrived in the earlier frame.
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          words: [{ word: "hello", language: "en" }],
+        }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hola amigo", {
+          is_final: true,
+          speech_final: true,
+          words: [
+            { word: "hola", language: "es" },
+            { word: "amigo", language: "es" },
+          ],
+        }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({
+        type: "final",
+        text: "hello hola amigo",
+        languages: ["es", "en"],
+      });
+    });
+
+    test("UtteranceEnd flush carries the accumulated language metadata", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      mockWs.simulateMessage(
+        resultsFrame("bonjour", {
+          is_final: true,
+          words: [{ word: "bonjour", language: "fr" }],
+        }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hello there", {
+          is_final: true,
+          alternativeLanguages: ["en"],
+        }),
+      );
+      mockWs.simulateMessage(utteranceEndFrame());
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({
+        type: "final",
+        text: "bonjour hello there",
+        languages: ["fr", "en"],
+      });
+    });
+
+    test("aggregated final omits language fields when no withheld frame carried tags", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      mockWs.simulateMessage(resultsFrame("no tags", { is_final: true }));
+      mockWs.simulateMessage(
+        resultsFrame("at all", { is_final: true, speech_final: true }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({ type: "final", text: "no tags at all" });
+      expect("language" in finals[0]!).toBe(false);
+      expect("languages" in finals[0]!).toBe(false);
+    });
+
+    test("tags on empty-text frames do not leak into the aggregated final", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      // A silence segment may still carry container-level tags; only
+      // frames that contributed transcript text feed the metadata.
+      mockWs.simulateMessage(
+        resultsFrame("", { is_final: true, alternativeLanguages: ["fr"] }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          speech_final: true,
+          words: [{ word: "hello", language: "en" }],
+        }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({
+        type: "final",
+        text: "hello",
+        languages: ["en"],
+      });
+    });
+
+    test("language tags reset between aggregated utterances", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      mockWs.simulateMessage(
+        resultsFrame("hola", {
+          is_final: true,
+          speech_final: true,
+          words: [{ word: "hola", language: "es" }],
+        }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          speech_final: true,
+          words: [{ word: "hello", language: "en" }],
+        }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(2);
+      expect(finals[1]).toEqual({
+        type: "final",
+        text: "hello",
+        languages: ["en"],
+      });
+    });
   });
 
   // ─────────────────────────────────────────────────────────────────

package/src/providers/speech-to-text/deepgram-realtime.ts

@@ -29,10 +29,12 @@
  * - All timers and listeners are cleaned up on close to prevent leaks.
  */
 
+import { rankLanguages } from "../../stt/language-metadata.js";
 import type {
   StreamingTranscriber,
   SttStreamServerEvent,
 } from "../../stt/types.js";
+import { baseLanguageSubtag } from "../../util/language-subtag.js";
 import { getLogger } from "../../util/logger.js";
 
 const log = getLogger("deepgram-realtime");
@@ -208,6 +210,11 @@ interface DeepgramStreamWord {
   confidence?: number;
   start?: number;
   end?: number;
+  /**
+   * BCP-47 tag of the language this word was spoken in. Present only on
+   * code-switching models (nova-3 with `language=multi`).
+   */
+  language?: string;
 }
 
 /**
@@ -225,11 +232,23 @@ interface DeepgramStreamAlternative {
   speaker?: number;
   /** Per-word speaker tags when diarization is enabled. */
   words?: DeepgramStreamWord[];
+  /**
+   * Detected languages for the chunk in dominance order. Emitted by
+   * code-switching models; the container varies by API version, so
+   * {@link DeepgramStreamChannel.languages} is checked as well.
+   */
+  languages?: string[];
 }
 
 /** A channel within a Deepgram streaming response. */
 interface DeepgramStreamChannel {
   alternatives?: DeepgramStreamAlternative[];
+  /**
+   * Detected languages for the chunk in dominance order. Alternate
+   * container for {@link DeepgramStreamAlternative.languages} on some
+   * API versions.
+   */
+  languages?: string[];
 }
 
 /**
@@ -343,6 +362,15 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
    */
   private pendingFinalSegments: string[] = [];
 
+  /**
+   * Raw detected-language tags for the withheld segments, accumulated
+   * alongside {@link pendingFinalSegments} and ranked into the event's
+   * `languages` when the utterance flushes. Cleared wherever the pending
+   * segments are cleared. Only populated when
+   * {@link utteranceBoundaryFinals} is enabled.
+   */
+  private pendingLanguageTags: string[] = [];
+
   /** The live WebSocket connection, set during start(). */
   private ws: WsLike | null = null;
 
@@ -748,6 +776,12 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
    * words — see {@link extractSpeakerLabel}. Confidence is taken from
    * the top alternative when present.
    *
+   * Code-switching models (nova-3 with `language=multi`) tag detected
+   * languages per word and per container. When present, these become the
+   * dominance-ranked `languages` field on the emitted events (see
+   * {@link extractLanguages}). The field is omitted when the frame
+   * carries no language metadata.
+   *
    * We emit:
    * - `partial` for `is_final: false` frames (if interim results enabled).
    * - `final` for `is_final: true` frames.
@@ -788,6 +822,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
       typeof alternative?.confidence === "number"
         ? alternative.confidence
         : undefined;
+    const languages = extractLanguages(frame.channel, alternative);
 
     if (frame.is_final) {
       if (this.utteranceBoundaryFinals) {
@@ -795,6 +830,16 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
         // Finalize flush is a forced boundary — flush what is pending.
         if (text.length > 0) {
           this.pendingFinalSegments.push(text);
+          // Collect language tags only from frames that contributed text
+          // so the flushed metadata stays aligned with the emitted
+          // transcript (empty frames may still carry tags, but they
+          // describe no emitted words). Raw per-word tags are preferred
+          // over the frame's ranked list so cross-frame frequency
+          // weighting survives until the flush ranks the whole utterance.
+          const wordTags = collectWordLanguageTags(alternative);
+          this.pendingLanguageTags.push(
+            ...(wordTags.length > 0 ? wordTags : languages),
+          );
         }
         if (frame.speech_final || fromFinalize) {
           this.flushPendingUtterance();
@@ -807,6 +852,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
           text,
           ...(speakerLabel !== undefined ? { speakerLabel } : {}),
           ...(confidence !== undefined ? { confidence } : {}),
+          ...(languages.length > 0 ? { languages } : {}),
           // Mark the finalize flush so consumers can attribute it to the
           // utterance that requested the flush rather than new speech.
           ...(fromFinalize ? { fromFinalize: true } : {}),
@@ -819,6 +865,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
         text,
         ...(speakerLabel !== undefined ? { speakerLabel } : {}),
         ...(confidence !== undefined ? { confidence } : {}),
+        ...(languages.length > 0 ? { languages } : {}),
       });
     }
 
@@ -917,16 +964,27 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
 
   /**
    * Emit a single aggregated `final` for the withheld `is_final` segments
-   * of the current utterance. No-op when nothing is pending, so boundary
-   * signals over silence emit nothing.
+   * of the current utterance, carrying the dominance-ranked detected
+   * languages accumulated alongside them (field omitted when no segment
+   * carried language metadata). No-op when nothing is pending, so
+   * boundary signals over silence emit nothing.
    */
   private flushPendingUtterance(): void {
     if (this.pendingFinalSegments.length === 0) {
+      // Tags accumulate only alongside text, but clear defensively so a
+      // future drift cannot leak one utterance's tags into the next.
+      this.pendingLanguageTags = [];
       return;
     }
     const text = this.pendingFinalSegments.join(" ");
+    const languages = rankLanguages(this.pendingLanguageTags);
     this.pendingFinalSegments = [];
-    this.emitEvent({ type: "final", text });
+    this.pendingLanguageTags = [];
+    this.emitEvent({
+      type: "final",
+      text,
+      ...(languages.length > 0 ? { languages } : {}),
+    });
   }
 
   /**
@@ -1222,6 +1280,62 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
  * contract on {@link SttStreamServerPartialEvent} /
  * {@link SttStreamServerFinalEvent}.
  */
+/**
+ * Derive the detected languages for a chunk, most dominant first.
+ *
+ * Code-switching models tag languages in two shapes:
+ *   1. Per-word `language` tags on `alternatives[0].words[]`, the richest
+ *      signal; ranked by frequency via {@link rankLanguages} (ties broken
+ *      by first appearance).
+ *   2. A container-level `languages` array in dominance order, attached to
+ *      the alternative or (on some API versions) the channel. Used as the
+ *      fallback when no word carries a tag; normalized and deduped with
+ *      the provider's order preserved.
+ *
+ * Returns `[]` when the frame carries no language metadata: callers omit
+ * the event fields entirely so absence stays distinguishable from a
+ * detected language.
+ */
+function extractLanguages(
+  channel: DeepgramStreamChannel | undefined,
+  alternative: DeepgramStreamAlternative | undefined,
+): string[] {
+  const wordTags = collectWordLanguageTags(alternative);
+  if (wordTags.length > 0) {
+    return rankLanguages(wordTags);
+  }
+
+  const container = Array.isArray(alternative?.languages)
+    ? alternative.languages
+    : Array.isArray(channel?.languages)
+      ? channel.languages
+      : [];
+  const deduped = new Set(
+    container
+      .filter((tag): tag is string => typeof tag === "string")
+      .flatMap((tag) => {
+        const base = baseLanguageSubtag(tag);
+        return base !== undefined ? [base] : [];
+      }),
+  );
+  return [...deduped];
+}
+
+/**
+ * Collect the raw per-word `language` tags of a chunk, in word order and
+ * without ranking or normalization. Used both for per-frame ranking in
+ * {@link extractLanguages} and for cross-frame accumulation in
+ * utterance-boundary mode, where ranking is deferred to the flush so
+ * frequency weighting spans the whole utterance.
+ */
+function collectWordLanguageTags(
+  alternative: DeepgramStreamAlternative | undefined,
+): string[] {
+  return (alternative?.words ?? []).flatMap((word) =>
+    typeof word.language === "string" ? [word.language] : [],
+  );
+}
+
 function extractSpeakerLabel(
   alternative: DeepgramStreamAlternative | undefined,
 ): string | undefined {

package/src/providers/speech-to-text/provider-catalog.ts

@@ -16,6 +16,7 @@ import type {
   SttProviderId,
   TelephonySttMode,
 } from "../../stt/types.js";
+import { baseLanguageSubtag } from "../../util/language-subtag.js";
 
 // ---------------------------------------------------------------------------
 // Client display metadata
@@ -282,6 +283,43 @@ export function listProviderEntries(): readonly SttProviderEntry[] {
   return [...CATALOG.values()];
 }
 
+/**
+ * A base-subtag regex over the pinned listening language. The pin is
+ * free-form workspace config, and it flows into prompt interpolation and
+ * per-language table lookups, so only a plausible ISO 639 base subtag
+ * passes; anything else (junk strings, prototype keys like "constructor")
+ * resolves as no pin.
+ */
+const PINNED_LANGUAGE_SUBTAG_REGEX = /^[a-z]{2,3}$/;
+
+/**
+ * The configured `services.stt.language` pin as the caller's listening
+ * language, or undefined when the pin carries no signal.
+ *
+ * A persisted pin only counts when the provider honors manual language
+ * selection: auto-detecting providers (gemini, whisper) ignore the setting
+ * entirely, so treating it as the caller's language would force every
+ * turn into a stale pin. "multi" and blank mean auto-detect (no pin), and
+ * the value must normalize to a plausible base subtag. Shared by the
+ * telephony pre-speech prompt rule (voice-session-bridge.ts), live
+ * voice's turn language (live-voice-session.ts), and telephony synthesis
+ * (telephony-synthesis-language.ts) so the gate cannot drift.
+ */
+export function pinnedListeningLanguage(
+  provider: string,
+  configuredLanguage: string | undefined,
+): string | undefined {
+  const providerHonorsLanguagePin =
+    getProviderEntry(provider as SttProviderId)?.languageSelection === "manual";
+  if (!providerHonorsLanguagePin || configuredLanguage?.trim() === "multi") {
+    return undefined;
+  }
+  const base = baseLanguageSubtag(configuredLanguage);
+  return base !== undefined && PINNED_LANGUAGE_SUBTAG_REGEX.test(base)
+    ? base
+    : undefined;
+}
+
 /**
  * Look up the credential-provider name for a given STT provider.
  *

package/src/runtime/__tests__/local-actor-identity-force-refresh.test.ts

@@ -0,0 +1,104 @@
+/**
+ * Tests for the cache-bypass option on the local guardian principal lookup.
+ *
+ * The guardian-delivery reader caches a successful read that finds no binding,
+ * and gateway-side binding writes do not invalidate the daemon's cache. A
+ * caller polling for a binding it expects to appear (the SSE actor-principal
+ * heal) therefore has to force the read, or every attempt re-reads the same
+ * empty answer until the TTL lapses.
+ */
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { GuardianDelivery } from "@vellumai/gateway-client";
+
+let cachedResult: GuardianDelivery[] | null = null;
+let freshResult: GuardianDelivery[] | null = null;
+let cachedCalls = 0;
+let freshCalls = 0;
+
+mock.module("../../config/env.js", () => ({
+  isHttpAuthDisabled: () => true,
+  hasUngatedHttpAuthDisabled: () => false,
+}));
+
+mock.module("../../contacts/guardian-delivery-reader.js", () => ({
+  getGuardianDelivery: () => {
+    cachedCalls++;
+    return Promise.resolve(cachedResult);
+  },
+  getGuardianDeliveryFresh: () => {
+    freshCalls++;
+    return Promise.resolve(freshResult);
+  },
+  peekCachedGuardianDelivery: () => undefined,
+  guardianForChannel: (list: GuardianDelivery[]) => list[0],
+}));
+
+import {
+  findLocalGuardianPrincipalId,
+  resolveActorPrincipalIdForLocalGuardian,
+} from "../local-actor-identity.js";
+
+afterAll(() => {
+  mock.restore();
+});
+
+/** Minimal guardian row: only `principalId` is read here. */
+function guardian(principalId: string): GuardianDelivery {
+  return { principalId } as unknown as GuardianDelivery;
+}
+
+describe("local guardian principal lookup — cache bypass", () => {
+  beforeEach(() => {
+    cachedCalls = 0;
+    freshCalls = 0;
+    cachedResult = [];
+    freshResult = [];
+  });
+
+  test("reads the cache by default", async () => {
+    cachedResult = [guardian("guardian-cached")];
+    freshResult = [guardian("guardian-fresh")];
+
+    expect(await findLocalGuardianPrincipalId()).toBe("guardian-cached");
+    expect(cachedCalls).toBe(1);
+    expect(freshCalls).toBe(0);
+  });
+
+  test("forceRefresh bypasses the cache and sees a binding the cache would miss", async () => {
+    // Cached read holds the empty result from before the binding existed.
+    cachedResult = [];
+    freshResult = [guardian("guardian-fresh")];
+
+    expect(await findLocalGuardianPrincipalId()).toBeUndefined();
+    expect(await findLocalGuardianPrincipalId({ forceRefresh: true })).toBe(
+      "guardian-fresh",
+    );
+    expect(freshCalls).toBe(1);
+  });
+
+  test("resolveActorPrincipalIdForLocalGuardian threads forceRefresh through", async () => {
+    cachedResult = [];
+    freshResult = [guardian("guardian-fresh")];
+
+    expect(
+      await resolveActorPrincipalIdForLocalGuardian("dev-bypass"),
+    ).toBeUndefined();
+    expect(
+      await resolveActorPrincipalIdForLocalGuardian("dev-bypass", {
+        forceRefresh: true,
+      }),
+    ).toBe("guardian-fresh");
+    expect(freshCalls).toBe(1);
+  });
+
+  test("a non-dev-bypass principal is passed through without any lookup", async () => {
+    expect(
+      await resolveActorPrincipalIdForLocalGuardian("actor-123", {
+        forceRefresh: true,
+      }),
+    ).toBe("actor-123");
+    expect(cachedCalls).toBe(0);
+    expect(freshCalls).toBe(0);
+  });
+});