@vellumai/assistant 0.11.3-staging.2 → 0.11.3-staging.4

package/src/live-voice/live-voice-tts.ts

@@ -1,3 +1,4 @@
+import { resolveLanguageVoiceOverride } from "../tts/language-voices.js";
 import { createPcmChunkAligner } from "../tts/pcm-chunk-aligner.js";
 import { getTtsProvider } from "../tts/provider-catalog.js";
 import { synthesizeAndEmit } from "../tts/synthesis-stream.js";
@@ -30,6 +31,7 @@ export interface LiveVoiceTtsOptions {
   useCase?: TtsUseCase;
   outputFormat?: TtsSynthesisRequest["outputFormat"];
   sampleRate?: number;
+  language?: string;
   config?: LiveVoiceTtsConfig;
   onAudioChunk: (chunk: LiveVoiceTtsAudioChunk) => void;
 }
@@ -72,11 +74,23 @@ interface ResolvedStreamingTtsProvider {
   providerConfig: Record<string, unknown>;
 }
 
+export { resolveLanguageVoiceOverride };
+
 export async function streamLiveVoiceTtsAudio(
   options: LiveVoiceTtsOptions,
 ): Promise<LiveVoiceTtsResult> {
   const { provider, providerId, providerConfig } =
     await resolveLiveVoiceStreamingTtsProvider(options.config);
+  // An explicit request voice wins outright; otherwise a language-known
+  // turn may select the provider's configured per-language voice. The cast
+  // recovers the schema-typed map that resolveTtsConfig's generic
+  // provider-block lookup erases.
+  const voiceId =
+    options.voiceId ??
+    resolveLanguageVoiceOverride(
+      providerConfig.languageVoices as Record<string, string> | undefined,
+      options.language,
+    );
   const useCase = options.useCase ?? "phone-call";
   const requestedSampleRate = resolveSampleRate(
     options.sampleRate,
@@ -89,9 +103,10 @@ export async function streamLiveVoiceTtsAudio(
   const providerSampleRate = provider.resolveOutputSampleRateHz?.({
     text: options.text,
     useCase,
-    voiceId: options.voiceId,
+    voiceId,
     outputFormat: options.outputFormat,
     sampleRateHz: requestedSampleRate,
+    language: options.language,
     signal: options.signal,
   });
   const sampleRate = providerSampleRate ?? requestedSampleRate;
@@ -138,9 +153,10 @@ export async function streamLiveVoiceTtsAudio(
       provider,
       text: options.text,
       useCase,
-      voiceId: options.voiceId,
+      voiceId,
       outputFormat: options.outputFormat,
       sampleRateHz: requestedSampleRate,
+      language: options.language,
       signal: options.signal,
       onChunk: (chunk) => {
         if (canStreamChunks) {

package/src/live-voice/progress-phrases.ts

@@ -1,3 +1,5 @@
+import { localizedOrDefault } from "../util/language-subtag.js";
+
 // Static fallbacks for an idle-triggered progress narration whose LLM
 // phrasing failed — the one case where prolonged silence is actively harmful.
 // The idle trigger can fire on a slow turn with zero tool activity, so every
@@ -10,8 +12,109 @@ export const PROGRESS_FALLBACK_PHRASES: readonly string[] = [
   "Almost there — thanks for waiting.",
 ];
 
+// Per-language fallback phrases, keyed by lowercased BCP 47 base subtag,
+// covering the Deepgram code-switching roster (DEEPGRAM_MULTI_LANGUAGE_CODES
+// in providers/speech-to-text/deepgram.ts). Every list carries the same
+// invariants as the English one above: persona-neutral, no claims about
+// running tools or tasks, at most 8 words per phrase.
+export const PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE: Readonly<
+  Record<string, readonly string[]>
+> = {
+  en: PROGRESS_FALLBACK_PHRASES,
+  es: [
+    "Sigo en ello, un momento.",
+    "Todavía lo estoy pensando.",
+    "Casi listo, gracias por esperar.",
+  ],
+  fr: [
+    "J'y suis encore, un instant.",
+    "J'y réfléchis encore.",
+    "Presque fini, merci de patienter.",
+  ],
+  de: [
+    "Bin noch dabei, einen Moment.",
+    "Ich denke noch darüber nach.",
+    "Fast fertig, danke fürs Warten.",
+  ],
+  hi: [
+    "बस एक पल रुकिए।",
+    "अभी इस पर विचार चल रहा है।",
+    "बस थोड़ा और इंतज़ार कीजिए, धन्यवाद।",
+  ],
+  ru: [
+    "Секундочку, я ещё здесь.",
+    "Я всё ещё думаю над этим.",
+    "Почти готово, спасибо за ожидание.",
+  ],
+  pt: [
+    "Ainda estou nisso, um momento.",
+    "Ainda estou pensando nisso.",
+    "Quase lá, agradeço a espera.",
+  ],
+  ja: [
+    "まだ対応中です。少々お待ちください。",
+    "まだ考えているところです。",
+    "もうすぐです。お待ちいただきありがとうございます。",
+  ],
+  it: [
+    "Ancora un attimo, per favore.",
+    "Ci sto ancora pensando.",
+    "Quasi fatto, grazie per l'attesa.",
+  ],
+  nl: [
+    "Ik ben er nog mee bezig.",
+    "Ik denk er nog over na.",
+    "Bijna klaar, bedankt voor het wachten.",
+  ],
+};
+
 // Deterministic rotation through the phrase list: callers hold a nonnegative
 // monotonic counter, so consecutive picks vary while tests stay reproducible.
-export function pickProgressPhrase(counter: number): string {
-  return PROGRESS_FALLBACK_PHRASES[counter % PROGRESS_FALLBACK_PHRASES.length];
+// `language` selects the per-language list by its lowercased base subtag
+// (e.g. "pt-BR" -> "pt"); unknown or absent languages fall back to English.
+export function pickProgressPhrase(counter: number, language?: string): string {
+  const phrases = localizedOrDefault(
+    PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
+    language,
+    PROGRESS_FALLBACK_PHRASES,
+  );
+  return phrases[counter % phrases.length];
+}
+
+// Spoken once when a turn starts waiting on the user's approval decision.
+// Fixed rather than generated: this is a statement about the system's
+// state, not about the work, and it has to be true every time. Kept in the
+// shape of the progress phrases it displaces (short, neutral, no claim
+// about tools).
+export const APPROVAL_PENDING_PHRASE =
+  "I need your okay for that one. Take a look.";
+
+// Per-language spellings of the approval-pending phrase, keyed by lowercased
+// BCP 47 base subtag, covering the Deepgram code-switching roster
+// (DEEPGRAM_MULTI_LANGUAGE_CODES in providers/speech-to-text/deepgram.ts).
+// Same invariants as the progress phrases: persona-neutral, no claims about
+// running tools or tasks.
+export const APPROVAL_PENDING_PHRASE_BY_LANGUAGE: Readonly<
+  Record<string, string>
+> = {
+  en: APPROVAL_PENDING_PHRASE,
+  es: "Necesito tu visto bueno para eso. Échale un vistazo.",
+  fr: "J'ai besoin de ton accord pour ça. Jette un œil.",
+  de: "Dafür brauche ich dein Okay. Schau mal drauf.",
+  hi: "इसके लिए मुझे आपकी मंज़ूरी चाहिए। एक नज़र डाल लीजिए।",
+  ru: "Для этого мне нужно твоё согласие. Взгляни, пожалуйста.",
+  pt: "Preciso do seu ok para isso. Dê uma olhada.",
+  ja: "これには許可が必要です。ご確認ください。",
+  it: "Mi serve il tuo via libera per questo. Dai un'occhiata.",
+  nl: "Hiervoor heb ik je akkoord nodig. Kijk even mee.",
+};
+
+// The approval-pending phrase in the turn's spoken language, defaulting to
+// English for unknown or absent languages.
+export function approvalPendingPhraseFor(language?: string): string {
+  return localizedOrDefault(
+    APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
+    language,
+    APPROVAL_PENDING_PHRASE,
+  );
 }

package/src/providers/speech-to-text/deepgram-realtime.test.ts

@@ -102,7 +102,11 @@ function resultsFrame(
     is_final?: boolean;
     speech_final?: boolean;
     from_finalize?: boolean;
-    words?: { word: string; speaker?: number }[];
+    words?: { word: string; speaker?: number; language?: string }[];
+    /** Container-level detected languages on the alternative. */
+    alternativeLanguages?: string[];
+    /** Container-level detected languages on the channel. */
+    channelLanguages?: string[];
   } = {},
 ): string {
   return JSON.stringify({
@@ -121,8 +125,14 @@ function resultsFrame(
           transcript,
           confidence: 0.95,
           ...(options.words ? { words: options.words } : {}),
+          ...(options.alternativeLanguages
+            ? { languages: options.alternativeLanguages }
+            : {}),
         },
       ],
+      ...(options.channelLanguages
+        ? { languages: options.channelLanguages }
+        : {}),
     },
   });
 }
@@ -542,6 +552,154 @@ describe("DeepgramRealtimeTranscriber", () => {
     (globalThis as Record<string, unknown>).WebSocket = origWs;
   });
 
+  // ─────────────────────────────────────────────────────────────────
+  // Language metadata (nova-3 multi code-switching)
+  // ─────────────────────────────────────────────────────────────────
+
+  describe("language metadata", () => {
+    test("ranks per-word language tags by dominance on final events", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hello world hola", {
+          is_final: true,
+          words: [
+            { word: "hello", language: "en" },
+            { word: "world", language: "en" },
+            { word: "hola", language: "es" },
+          ],
+        }),
+      );
+
+      expect(events).toHaveLength(1);
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "hello world hola",
+        confidence: 0.95,
+        languages: ["en", "es"],
+      });
+    });
+
+    test("omits the languages field entirely when no language metadata is present", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hello world", {
+          is_final: true,
+          words: [{ word: "hello" }, { word: "world" }],
+        }),
+      );
+      mockWs.simulateMessage(resultsFrame("still typing", { is_final: false }));
+
+      expect(events).toHaveLength(2);
+      for (const event of events) {
+        // The keys must not exist at all, not just be undefined-valued.
+        expect("language" in event).toBe(false);
+        expect("languages" in event).toBe(false);
+      }
+    });
+
+    test("normalizes regional tags to their base subtag", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          words: [{ word: "hello", language: "en-US" }],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "hello",
+        confidence: 0.95,
+        languages: ["en"],
+      });
+    });
+
+    test("partial events carry the fields when interim results are enabled", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("namaste hello", {
+          is_final: false,
+          words: [
+            { word: "namaste", language: "hi" },
+            { word: "hello", language: "en" },
+            { word: "there", language: "en" },
+          ],
+        }),
+      );
+
+      expect(events).toHaveLength(1);
+      expect(events[0]).toEqual({
+        type: "partial",
+        text: "namaste hello",
+        confidence: 0.95,
+        languages: ["en", "hi"],
+      });
+    });
+
+    test("falls back to the alternative-level languages array when words carry no tags", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("mixed speech", {
+          is_final: true,
+          words: [{ word: "mixed" }, { word: "speech" }],
+          alternativeLanguages: ["es-419", "en", "ES"],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "mixed speech",
+        confidence: 0.95,
+        languages: ["es", "en"],
+      });
+    });
+
+    test("falls back to the channel-level languages array when the alternative has none", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("bonjour", {
+          is_final: true,
+          channelLanguages: ["fr", "en"],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "bonjour",
+        confidence: 0.95,
+        languages: ["fr", "en"],
+      });
+    });
+
+    test("per-word tags take precedence over container arrays", async () => {
+      const { events } = await startSession();
+
+      mockWs.simulateMessage(
+        resultsFrame("hola amigo", {
+          is_final: true,
+          words: [
+            { word: "hola", language: "es" },
+            { word: "amigo", language: "es" },
+          ],
+          alternativeLanguages: ["en", "es"],
+        }),
+      );
+
+      expect(events[0]).toEqual({
+        type: "final",
+        text: "hola amigo",
+        confidence: 0.95,
+        languages: ["es"],
+      });
+    });
+  });
+
   // ─────────────────────────────────────────────────────────────────
   // Multi-event sequence
   // ─────────────────────────────────────────────────────────────────
@@ -671,6 +829,130 @@ describe("DeepgramRealtimeTranscriber", () => {
         "second utterance",
       ]);
     });
+
+    test("aggregated final ranks language tags across all withheld frames", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      // Raw per-word tags are accumulated, so "es" (2 words) outranks
+      // "en" (1 word) even though "en" arrived in the earlier frame.
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          words: [{ word: "hello", language: "en" }],
+        }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hola amigo", {
+          is_final: true,
+          speech_final: true,
+          words: [
+            { word: "hola", language: "es" },
+            { word: "amigo", language: "es" },
+          ],
+        }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({
+        type: "final",
+        text: "hello hola amigo",
+        languages: ["es", "en"],
+      });
+    });
+
+    test("UtteranceEnd flush carries the accumulated language metadata", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      mockWs.simulateMessage(
+        resultsFrame("bonjour", {
+          is_final: true,
+          words: [{ word: "bonjour", language: "fr" }],
+        }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hello there", {
+          is_final: true,
+          alternativeLanguages: ["en"],
+        }),
+      );
+      mockWs.simulateMessage(utteranceEndFrame());
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({
+        type: "final",
+        text: "bonjour hello there",
+        languages: ["fr", "en"],
+      });
+    });
+
+    test("aggregated final omits language fields when no withheld frame carried tags", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      mockWs.simulateMessage(resultsFrame("no tags", { is_final: true }));
+      mockWs.simulateMessage(
+        resultsFrame("at all", { is_final: true, speech_final: true }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({ type: "final", text: "no tags at all" });
+      expect("language" in finals[0]!).toBe(false);
+      expect("languages" in finals[0]!).toBe(false);
+    });
+
+    test("tags on empty-text frames do not leak into the aggregated final", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      // A silence segment may still carry container-level tags; only
+      // frames that contributed transcript text feed the metadata.
+      mockWs.simulateMessage(
+        resultsFrame("", { is_final: true, alternativeLanguages: ["fr"] }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          speech_final: true,
+          words: [{ word: "hello", language: "en" }],
+        }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(1);
+      expect(finals[0]).toEqual({
+        type: "final",
+        text: "hello",
+        languages: ["en"],
+      });
+    });
+
+    test("language tags reset between aggregated utterances", async () => {
+      const { events } = await startSession({ utteranceBoundaryFinals: true });
+
+      mockWs.simulateMessage(
+        resultsFrame("hola", {
+          is_final: true,
+          speech_final: true,
+          words: [{ word: "hola", language: "es" }],
+        }),
+      );
+      mockWs.simulateMessage(
+        resultsFrame("hello", {
+          is_final: true,
+          speech_final: true,
+          words: [{ word: "hello", language: "en" }],
+        }),
+      );
+
+      const finals = events.filter((e) => e.type === "final");
+      expect(finals).toHaveLength(2);
+      expect(finals[1]).toEqual({
+        type: "final",
+        text: "hello",
+        languages: ["en"],
+      });
+    });
   });
 
   // ─────────────────────────────────────────────────────────────────

package/src/providers/speech-to-text/deepgram-realtime.ts

@@ -29,10 +29,12 @@
  * - All timers and listeners are cleaned up on close to prevent leaks.
  */
 
+import { rankLanguages } from "../../stt/language-metadata.js";
 import type {
   StreamingTranscriber,
   SttStreamServerEvent,
 } from "../../stt/types.js";
+import { baseLanguageSubtag } from "../../util/language-subtag.js";
 import { getLogger } from "../../util/logger.js";
 
 const log = getLogger("deepgram-realtime");
@@ -208,6 +210,11 @@ interface DeepgramStreamWord {
   confidence?: number;
   start?: number;
   end?: number;
+  /**
+   * BCP-47 tag of the language this word was spoken in. Present only on
+   * code-switching models (nova-3 with `language=multi`).
+   */
+  language?: string;
 }
 
 /**
@@ -225,11 +232,23 @@ interface DeepgramStreamAlternative {
   speaker?: number;
   /** Per-word speaker tags when diarization is enabled. */
   words?: DeepgramStreamWord[];
+  /**
+   * Detected languages for the chunk in dominance order. Emitted by
+   * code-switching models; the container varies by API version, so
+   * {@link DeepgramStreamChannel.languages} is checked as well.
+   */
+  languages?: string[];
 }
 
 /** A channel within a Deepgram streaming response. */
 interface DeepgramStreamChannel {
   alternatives?: DeepgramStreamAlternative[];
+  /**
+   * Detected languages for the chunk in dominance order. Alternate
+   * container for {@link DeepgramStreamAlternative.languages} on some
+   * API versions.
+   */
+  languages?: string[];
 }
 
 /**
@@ -343,6 +362,15 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
    */
   private pendingFinalSegments: string[] = [];
 
+  /**
+   * Raw detected-language tags for the withheld segments, accumulated
+   * alongside {@link pendingFinalSegments} and ranked into the event's
+   * `languages` when the utterance flushes. Cleared wherever the pending
+   * segments are cleared. Only populated when
+   * {@link utteranceBoundaryFinals} is enabled.
+   */
+  private pendingLanguageTags: string[] = [];
+
   /** The live WebSocket connection, set during start(). */
   private ws: WsLike | null = null;
 
@@ -748,6 +776,12 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
    * words — see {@link extractSpeakerLabel}. Confidence is taken from
    * the top alternative when present.
    *
+   * Code-switching models (nova-3 with `language=multi`) tag detected
+   * languages per word and per container. When present, these become the
+   * dominance-ranked `languages` field on the emitted events (see
+   * {@link extractLanguages}). The field is omitted when the frame
+   * carries no language metadata.
+   *
    * We emit:
    * - `partial` for `is_final: false` frames (if interim results enabled).
    * - `final` for `is_final: true` frames.
@@ -788,6 +822,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
       typeof alternative?.confidence === "number"
         ? alternative.confidence
         : undefined;
+    const languages = extractLanguages(frame.channel, alternative);
 
     if (frame.is_final) {
       if (this.utteranceBoundaryFinals) {
@@ -795,6 +830,16 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
         // Finalize flush is a forced boundary — flush what is pending.
         if (text.length > 0) {
           this.pendingFinalSegments.push(text);
+          // Collect language tags only from frames that contributed text
+          // so the flushed metadata stays aligned with the emitted
+          // transcript (empty frames may still carry tags, but they
+          // describe no emitted words). Raw per-word tags are preferred
+          // over the frame's ranked list so cross-frame frequency
+          // weighting survives until the flush ranks the whole utterance.
+          const wordTags = collectWordLanguageTags(alternative);
+          this.pendingLanguageTags.push(
+            ...(wordTags.length > 0 ? wordTags : languages),
+          );
         }
         if (frame.speech_final || fromFinalize) {
           this.flushPendingUtterance();
@@ -807,6 +852,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
           text,
           ...(speakerLabel !== undefined ? { speakerLabel } : {}),
           ...(confidence !== undefined ? { confidence } : {}),
+          ...(languages.length > 0 ? { languages } : {}),
           // Mark the finalize flush so consumers can attribute it to the
           // utterance that requested the flush rather than new speech.
           ...(fromFinalize ? { fromFinalize: true } : {}),
@@ -819,6 +865,7 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
         text,
         ...(speakerLabel !== undefined ? { speakerLabel } : {}),
         ...(confidence !== undefined ? { confidence } : {}),
+        ...(languages.length > 0 ? { languages } : {}),
       });
     }
 
@@ -917,16 +964,27 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
 
   /**
    * Emit a single aggregated `final` for the withheld `is_final` segments
-   * of the current utterance. No-op when nothing is pending, so boundary
-   * signals over silence emit nothing.
+   * of the current utterance, carrying the dominance-ranked detected
+   * languages accumulated alongside them (field omitted when no segment
+   * carried language metadata). No-op when nothing is pending, so
+   * boundary signals over silence emit nothing.
    */
   private flushPendingUtterance(): void {
     if (this.pendingFinalSegments.length === 0) {
+      // Tags accumulate only alongside text, but clear defensively so a
+      // future drift cannot leak one utterance's tags into the next.
+      this.pendingLanguageTags = [];
       return;
     }
     const text = this.pendingFinalSegments.join(" ");
+    const languages = rankLanguages(this.pendingLanguageTags);
     this.pendingFinalSegments = [];
-    this.emitEvent({ type: "final", text });
+    this.pendingLanguageTags = [];
+    this.emitEvent({
+      type: "final",
+      text,
+      ...(languages.length > 0 ? { languages } : {}),
+    });
   }
 
   /**
@@ -1222,6 +1280,62 @@ export class DeepgramRealtimeTranscriber implements StreamingTranscriber {
  * contract on {@link SttStreamServerPartialEvent} /
  * {@link SttStreamServerFinalEvent}.
  */
+/**
+ * Derive the detected languages for a chunk, most dominant first.
+ *
+ * Code-switching models tag languages in two shapes:
+ *   1. Per-word `language` tags on `alternatives[0].words[]`, the richest
+ *      signal; ranked by frequency via {@link rankLanguages} (ties broken
+ *      by first appearance).
+ *   2. A container-level `languages` array in dominance order, attached to
+ *      the alternative or (on some API versions) the channel. Used as the
+ *      fallback when no word carries a tag; normalized and deduped with
+ *      the provider's order preserved.
+ *
+ * Returns `[]` when the frame carries no language metadata: callers omit
+ * the event fields entirely so absence stays distinguishable from a
+ * detected language.
+ */
+function extractLanguages(
+  channel: DeepgramStreamChannel | undefined,
+  alternative: DeepgramStreamAlternative | undefined,
+): string[] {
+  const wordTags = collectWordLanguageTags(alternative);
+  if (wordTags.length > 0) {
+    return rankLanguages(wordTags);
+  }
+
+  const container = Array.isArray(alternative?.languages)
+    ? alternative.languages
+    : Array.isArray(channel?.languages)
+      ? channel.languages
+      : [];
+  const deduped = new Set(
+    container
+      .filter((tag): tag is string => typeof tag === "string")
+      .flatMap((tag) => {
+        const base = baseLanguageSubtag(tag);
+        return base !== undefined ? [base] : [];
+      }),
+  );
+  return [...deduped];
+}
+
+/**
+ * Collect the raw per-word `language` tags of a chunk, in word order and
+ * without ranking or normalization. Used both for per-frame ranking in
+ * {@link extractLanguages} and for cross-frame accumulation in
+ * utterance-boundary mode, where ranking is deferred to the flush so
+ * frequency weighting spans the whole utterance.
+ */
+function collectWordLanguageTags(
+  alternative: DeepgramStreamAlternative | undefined,
+): string[] {
+  return (alternative?.words ?? []).flatMap((word) =>
+    typeof word.language === "string" ? [word.language] : [],
+  );
+}
+
 function extractSpeakerLabel(
   alternative: DeepgramStreamAlternative | undefined,
 ): string | undefined {