@vellumai/assistant 0.11.3-staging.2 → 0.11.3-staging.3

package/src/live-voice/live-voice-session.ts

@@ -25,7 +25,8 @@ import {
   classifyFrontDoorLeading,
   ESCALATE_VERDICT_TOKEN,
   ESCALATION_CONTINUATION_CONTENT,
-  FALLBACK_ESCALATION_BRIDGE,
+  FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
+  fallbackEscalationBridgeFor,
   isEscalationBridgeComplete,
   MIN_SPOKEN_BRIDGE_CHARS,
   type VoiceRoutingLeg,
@@ -46,14 +47,20 @@ import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
 import { ensureConversationExists } from "../persistence/conversation-crud.js";
 import {
   listProviderIds,
+  pinnedListeningLanguage,
   supportsBoundary,
 } from "../providers/speech-to-text/provider-catalog.js";
 import type { ResolveStreamingTranscriberOptions } from "../providers/speech-to-text/resolve.js";
 import { broadcastMessage } from "../runtime/assistant-event-hub.js";
 import { publishConversationListAndMetadataChanged } from "../runtime/sync/resource-sync-events.js";
+import {
+  dominantLanguageTag,
+  voteDominantLanguage,
+} from "../stt/language-metadata.js";
 import { detectPcm16SpeechActivity } from "../stt/speech-energy.js";
 import type {
   StreamingTranscriber,
+  SttProviderId,
   SttStreamServerErrorEvent,
   SttStreamServerEvent,
 } from "../stt/types.js";
@@ -62,6 +69,7 @@ import { liveVoiceEndScreen } from "../telemetry/live-voice-funnel.js";
 import { getToolOwner } from "../tools/registry.js";
 import { extractSpeakableSegments } from "../tts/speakable-segments.js";
 import { createAbortReason } from "../util/abort-reasons.js";
+import { hasLocalizedEntry } from "../util/language-subtag.js";
 import { getLogger } from "../util/logger.js";
 import {
   activityLabelForTool,
@@ -101,7 +109,12 @@ import type {
   LiveVoiceTtsOptions,
   LiveVoiceTtsResult,
 } from "./live-voice-tts.js";
-import { pickProgressPhrase } from "./progress-phrases.js";
+import {
+  APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
+  approvalPendingPhraseFor,
+  pickProgressPhrase,
+  PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
+} from "./progress-phrases.js";
 import {
   type LiveVoiceClientAttachImageFrame,
   type LiveVoiceClientFrame,
@@ -369,6 +382,23 @@ interface UtteranceCycle {
   // text replays the boundary immediately — the hold was judged on stale
   // text, so waiting out the extension only adds silence.
   heldSpeculativeContent: string | null;
+  // Count per detected-language base subtag (see voteDominantLanguage)
+  // across this cycle's final transcript events. Resolves the turn's spoken
+  // language (see turnLanguageFor); empty when the provider tags nothing.
+  languageTally: Map<string, number>;
+  // Detected languages of the most recent partial that carried any, already
+  // normalized, dominance order. Speculative turns dispatch from partials
+  // before the first tagged final lands, so turnLanguageFor falls back to
+  // this when the final tally is still empty. Never cleared: the tally
+  // outranks it once finals arrive, and a revising partial without tags
+  // must not wipe an earlier partial's detection.
+  latestPartialLanguages: readonly string[] | null;
+  // The provider that actually transcribed this cycle, recorded when its
+  // transcriber is assigned and kept after teardown nulls `transcriber`.
+  // The resolver can silently dial managed vellum when a BYOK provider has
+  // no credential, so the language-pin gate in turnLanguageFor must follow
+  // this, not the configured provider.
+  dialedSttProvider: SttProviderId | null;
   turnId: string | null;
   userMessageId: string | null;
   userAudioChunks: Buffer[];
@@ -399,6 +429,11 @@ type UtteranceStartResult =
 // client in job-list order.
 interface TtsSegmentJob {
   readonly text: string;
+  // Per-segment language-hint override, preferred over the turn's language.
+  // Set on fixed phrases whose localized table lacks the turn's language:
+  // the English fallback text carries "en" so an enforcing provider never
+  // renders English words as ar/ko/ta. Undefined means the turn language.
+  readonly language: string | undefined;
   // The provider stream was started (the job holds an open-job slot).
   started: boolean;
   // Emission finished; the slot is free for the next queued segment.
@@ -483,6 +518,13 @@ interface ActiveAssistantTurn {
   token: symbol;
   turnId: string;
   utterance: UtteranceCycle;
+  // The caller's spoken language for this turn as a lowercase base subtag
+  // (see turnLanguageFor): the dominant STT-detected language, else a
+  // monolingual services.stt.language pin. Undefined when unknown, which
+  // disables every language-aware path (prompt note, TTS hint, localized
+  // fallbacks). Re-resolved when a speculative turn commits, since finals
+  // can land between dispatch and verdict.
+  language: string | undefined;
   abortController: AbortController;
   handle: VoiceTurnHandle | null;
   // When the turn launched, for narration's turnElapsedMs.
@@ -681,14 +723,8 @@ function createControlMarkerHoldback(
 // barge-in, the interruption merge note is appended to it (see
 // buildInterruptionMergeNote) so the model reconciles the interrupted request
 // with the new utterance.
-// Spoken once when a turn starts waiting on the user's decision. Fixed rather
-// than generated: this is a statement about the system's state, not about the
-// work, and it has to be true every time. Kept in the shape of the progress
-// phrases it displaces (short, neutral, no claim about tools).
-const APPROVAL_PENDING_PHRASE = "I need your okay for that one. Take a look.";
-
 const LIVE_VOICE_CONTROL_PROMPT_BASE =
-  "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. ";
+  "You are speaking in a local live voice session. Keep replies brief and conversational. Speech is the main channel: say the answer, and do not narrate a surface instead of answering. You can also put something on screen when it genuinely helps (a form, a list to pick from, a progress card for long work); the call overlay minimizes by itself once you finish speaking, so the user sees it without doing anything. Never tell the user you cannot show them something. Reply in the language the caller is speaking; if they switch languages, switch with them. ";
 
 // Appended for the legs that can actually put something on screen: the main
 // leg and the escalated leg. The front-door (fast) leg never receives it, for
@@ -777,6 +813,9 @@ function buildVoiceControlPrompt(
     LIVE_VOICE_CONTROL_PROMPT_BASE +
     (leg.frontDoor === true ? "" : LIVE_VOICE_SCREEN_REVEAL_TEACHING) +
     VOICE_NO_SETUP_FLOWS_RULE;
+  if (turn.language !== undefined) {
+    prompt = `${prompt}\n\nThe caller has been speaking the language with code "${turn.language}" this turn. Reply in that language unless they clearly switch to another.`;
+  }
   if (turn.interruptedRequest) {
     prompt = `${prompt}\n\n${buildInterruptionMergeNote(turn.interruptedRequest)}`;
   }
@@ -825,6 +864,9 @@ function createUtteranceCycle(): UtteranceCycle {
     latestPartialText: null,
     endpointExtensionCount: 0,
     heldSpeculativeContent: null,
+    languageTally: new Map(),
+    latestPartialLanguages: null,
+    dialedSttProvider: null,
     turnId: null,
     userMessageId: null,
     userAudioChunks: [],
@@ -1387,6 +1429,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         // Persistent re-arm: the shared stream is already open, so the cycle
         // goes straight to streaming with no resolve/start round-trip.
         utterance.transcriber = shared;
+        utterance.dialedSttProvider = shared.providerId;
         return await this.activateUtterance(utterance, replayTurnEnd);
       }
       // The shared stream is pinned to the old language, so retire it and
@@ -1419,6 +1462,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       }
 
       utterance.transcriber = transcriber;
+      utterance.dialedSttProvider = transcriber.providerId;
       if (
         this.turnDetector &&
         typeof transcriber.finalizeUtterance === "function"
@@ -2527,8 +2571,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     // Spoken, because opening the room is only a cue for someone looking at
     // the screen, and the case this exists for is a phone the user has put
     // down. One line, not narration: the turn is not working, it is waiting,
-    // and it says which.
-    this.enqueueFillerPhrase(turn, APPROVAL_PENDING_PHRASE);
+    // and it says which, in the turn's spoken language, like every other
+    // filler phrase.
+    this.enqueueFillerPhrase(
+      turn,
+      approvalPendingPhraseFor(turn.language),
+      this.fixedPhraseLanguage(turn, APPROVAL_PENDING_PHRASE_BY_LANGUAGE),
+    );
   }
 
   /** Clear the wait once a decision lands, so the turn narrates normally again. */
@@ -2851,6 +2900,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     // are skipped; the thinking frame and timers still apply.
     const alreadyReleased = utterance.released;
     turn.speculativePending = false;
+    // Finals can land between the speculative dispatch and this verdict.
+    // Fill the language only when dispatch had none: the model request was
+    // already issued with the dispatch language, so overwriting here would
+    // hint TTS (and any voice override) in a different language than the
+    // text it speaks. The tally still carries the corrected detection into
+    // the next turn.
+    turn.language ??= this.turnLanguageFor(utterance);
     if (turn.verdictDeadlineTimer !== null) {
       clearTimeout(turn.verdictDeadlineTimer);
       turn.verdictDeadlineTimer = null;
@@ -3189,11 +3245,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     switch (event.type) {
       case "partial":
         utterance.latestPartialText = event.text;
+        this.capturePartialLanguages(utterance, event.languages);
         this.markFirstPartial(utterance);
         await this.sendFrame({ type: "stt_partial", text: event.text });
         return;
       case "final":
-        await this.recordFinalTranscript(utterance, event.text);
+        await this.recordFinalTranscript(
+          utterance,
+          event.text,
+          event.languages,
+        );
         return;
       case "finalized":
         // Per-cycle transcribers are torn down with stop(); the finalize
@@ -3262,6 +3323,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
           return;
         }
         target.latestPartialText = event.text;
+        this.capturePartialLanguages(target, event.languages);
         this.markFirstPartial(target);
         await this.sendFrame({ type: "stt_partial", text: event.text });
         return;
@@ -3277,7 +3339,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
           // newer cycle.
           const owner = this.finalizeQueue[0];
           if (owner && !owner.assistantTurnStarted && !owner.completed) {
-            await this.recordFinalTranscript(owner, event.text);
+            await this.recordFinalTranscript(
+              owner,
+              event.text,
+              event.languages,
+            );
           } else {
             log.warn(
               "Dropping a late finalize flush segment: its assistant turn already dispatched",
@@ -3294,7 +3360,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
           );
           return;
         }
-        await this.recordFinalTranscript(target, event.text);
+        await this.recordFinalTranscript(target, event.text, event.languages);
         return;
       }
       case "finalized": {
@@ -3363,10 +3429,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   private async recordFinalTranscript(
     utterance: UtteranceCycle,
     text: string,
+    languages?: readonly string[],
   ): Promise<void> {
     const transcript = text.trim();
     if (transcript.length > 0) {
       utterance.finalTranscriptSegments.push(transcript);
+      // Tally only finals that committed transcript: empty silence frames
+      // can still carry container-level language tags describing no emitted
+      // words, and counting those would let silence outvote real speech
+      // (same choice as the adapter's boundary-final aggregation).
+      voteDominantLanguage(utterance.languageTally, languages);
     }
     // The final commits (and supersedes) whatever partial was trailing it.
     utterance.latestPartialText = null;
@@ -3405,6 +3477,59 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     await this.startAssistantTurnIfReady();
   }
 
+  // Record a partial event's detected languages so speculative dispatch
+  // has a detection before the first tagged final. The event contract
+  // (stt/types.ts) guarantees the tags arrive as normalized base subtags
+  // in dominance order, so they are stored as-is. Partials revise each
+  // other, so this overwrites rather than tallies, and a tag-less partial
+  // keeps the previous value.
+  private capturePartialLanguages(
+    utterance: UtteranceCycle,
+    languages: readonly string[] | undefined,
+  ): void {
+    if (!languages || languages.length === 0) {
+      return;
+    }
+    utterance.latestPartialLanguages = languages;
+  }
+
+  /**
+   * The caller's spoken language for a turn on this utterance, as a
+   * lowercase base subtag: the dominant tallied STT-detected language
+   * (most final-event counts, ties by first appearance), else the latest
+   * tagged partial's dominant language (speculative turns dispatch from
+   * partials), else a monolingual `services.stt.language` pin (a pinned
+   * language IS the spoken language), else undefined ("multi" with no tags,
+   * non-tagging providers, silence).
+   */
+  private turnLanguageFor(utterance: UtteranceCycle): string | undefined {
+    const dominant = dominantLanguageTag(utterance.languageTally);
+    if (dominant !== undefined) {
+      return dominant;
+    }
+    // No tagged final yet (speculative turns dispatch from partials): the
+    // latest tagged partial is the best detection available and outranks a
+    // static pin for the same reason the tally does.
+    const partialDominant = utterance.latestPartialLanguages?.[0];
+    if (partialDominant !== undefined) {
+      return partialDominant;
+    }
+    // A persisted pin only counts when the provider that actually
+    // transcribed honors manual language selection (the shared
+    // pinnedListeningLanguage gate). The DIALED transcriber's providerId
+    // is authoritative, because the resolver silently falls back to
+    // managed vellum (which honors the pin) when a BYOK provider has no
+    // credential; the configured provider is only the last resort when no
+    // transcriber reference survives.
+    const { language: configured, provider: sttProvider } =
+      getConfig().services.stt;
+    const dialedProvider =
+      utterance.dialedSttProvider ??
+      this.sharedTranscriber?.providerId ??
+      (sttProvider as SttProviderId);
+    return pinnedListeningLanguage(dialedProvider, configured);
+  }
+
   // Providers emit `error` mid-stream and may keep streaming; `closed` /
   // `final` still drive turn lifecycle. Only transient categories are
   // recoverable — auth/rate-limit/invalid-audio will not self-heal, so
@@ -3660,6 +3785,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       token,
       turnId,
       utterance,
+      language: this.turnLanguageFor(utterance),
       abortController,
       handle: null,
       launchedAtMs: Date.now(),
@@ -4312,7 +4438,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     // deleted row for a bridge the model never produced).
     const usesFallbackBridge = cappedBridge.length < MIN_SPOKEN_BRIDGE_CHARS;
     const spokenBridge = usesFallbackBridge
-      ? FALLBACK_ESCALATION_BRIDGE
+      ? fallbackEscalationBridgeFor(activeTurn.language)
       : cappedBridge;
     if (!usesFallbackBridge) {
       this.markFirstAssistantDelta(activeTurn.utterance, activeTurn.turnId);
@@ -4321,12 +4447,25 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         { type: "assistant_text_delta", text: spokenBridge },
         () => !activeTurn.abortController.signal.aborted && !this.isClosed,
       );
+      this.bufferAssistantTextForTts(activeTurn.token, `${spokenBridge} `);
+      // Force-flush now: on the TTS path an unpunctuated bridge would
+      // otherwise sit buffered until a sentence boundary and leave the
+      // caller in silence during the escalated model's call.
+      this.flushTtsBuffer(activeTurn.token, true);
+    } else {
+      // The canned bridge is a fixed localized-table phrase, enqueued
+      // directly (it is already one complete sentence) so the segment can
+      // carry the "en" override when the table lacks the turn's language.
+      const speakable = sanitizeForTts(spokenBridge).trim();
+      if (speakable.length > 0) {
+        this.enqueueTtsSegment(activeTurn.token, speakable, {
+          language: this.fixedPhraseLanguage(
+            activeTurn,
+            FALLBACK_ESCALATION_BRIDGE_BY_LANGUAGE,
+          ),
+        });
+      }
     }
-    this.bufferAssistantTextForTts(activeTurn.token, `${spokenBridge} `);
-    // Force-flush now: on the TTS path an unpunctuated bridge would otherwise
-    // sit buffered until a sentence boundary and leave the caller in silence
-    // during the escalated model's call.
-    this.flushTtsBuffer(activeTurn.token, true);
 
     // No overrideProfile: the escalated leg runs on the call-site default —
     // the exact profile an un-routed voice turn would use (see
@@ -4690,6 +4829,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
           : null,
         turnElapsedMs: now - turn.launchedAtMs,
         updateIndex: progress.updatesSpoken + 1,
+        ...(turn.language !== undefined ? { languageHint: turn.language } : {}),
       };
       const generated = await frontDecider
         .generateProgressText(input, turn.abortController.signal)
@@ -4718,13 +4858,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         return;
       }
       let raw = generated;
+      // Decider text is generated in the turn's language; only the static
+      // fallback comes from a localized table and may need the "en" override.
+      let fillerLanguage: string | undefined;
       if (raw === null) {
         if (trigger !== "idle") {
           return;
         }
-        raw = pickProgressPhrase(this.progressPhraseCounter++);
+        raw = pickProgressPhrase(this.progressPhraseCounter++, turn.language);
+        fillerLanguage = this.fixedPhraseLanguage(
+          turn,
+          PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
+        );
       }
-      if (!this.enqueueFillerPhrase(turn, raw)) {
+      if (!this.enqueueFillerPhrase(turn, raw, fillerLanguage)) {
         return;
       }
       progress.opsSinceNarration = 0;
@@ -4784,6 +4931,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
           {
             transcriptSoFar: transcript,
             toolName,
+            ...(activeTurn.language !== undefined
+              ? { languageHint: activeTurn.language }
+              : {}),
           },
           activeTurn.abortController.signal,
         )
@@ -4821,20 +4971,42 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   // Sanitize and enqueue one filler sentence (spoken ack or progress
   // narration) on the turn's ordered TTS queue — the shared tail of every
   // filler path. Returns whether a phrase actually enqueued; per-kind metric
-  // marks and bookkeeping are the caller's.
-  private enqueueFillerPhrase(turn: ActiveAssistantTurn, raw: string): boolean {
+  // marks and bookkeeping are the caller's. `language` is a per-segment
+  // hint override (see fixedPhraseLanguage); omit it for generated text,
+  // which is already in the turn's language.
+  private enqueueFillerPhrase(
+    turn: ActiveAssistantTurn,
+    raw: string,
+    language?: string,
+  ): boolean {
     const phrase = sanitizeForTts(raw).trim();
     if (phrase.length === 0) {
       return false;
     }
     this.enqueueTtsSegment(turn.token, phrase, {
       countsAsFirstSegment: false,
+      ...(language !== undefined ? { language } : {}),
     });
     // A spoken filler holds the floor, so narration's minGapMs spaces from it.
     turn.progress.lastFloorHolderAtMs = Date.now();
     return true;
   }
 
+  // The TTS hint override for a fixed phrase picked from a localized table:
+  // "en" when the turn has a language the table does not cover (the picker
+  // fell back to English text, which must not be synthesized under an
+  // ar/ko/ta hint), undefined otherwise (the segment rides the turn's
+  // language, or no hint at all when the language is unknown).
+  private fixedPhraseLanguage(
+    turn: ActiveAssistantTurn,
+    table: Readonly<Record<string, unknown>>,
+  ): string | undefined {
+    return turn.language !== undefined &&
+      !hasLocalizedEntry(table, turn.language)
+      ? "en"
+      : undefined;
+  }
+
   private bufferAssistantTextForTts(token: symbol, text: string): void {
     if (!this.streamTtsAudio || text.length === 0) {
       return;
@@ -4955,7 +5127,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   private enqueueTtsSegment(
     token: symbol,
     segment: string,
-    options: { countsAsFirstSegment?: boolean } = {},
+    options: { countsAsFirstSegment?: boolean; language?: string } = {},
   ): void {
     const activeTurn = this.activeAssistantTurn;
     if (activeTurn?.token !== token || !this.streamTtsAudio) {
@@ -4969,6 +5141,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     }
     const job: TtsSegmentJob = {
       text: segment,
+      language: options.language,
       started: false,
       settled: false,
       emitting: false,
@@ -5008,10 +5181,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         return;
       }
       job.started = true;
+      // The segment's own language override (fixed English fallback text)
+      // wins over the turn's language.
+      const language = job.language ?? activeTurn.language;
       let synthesis: Promise<void>;
       try {
         synthesis = streamTtsAudio({
           text: job.text,
+          ...(language !== undefined ? { language } : {}),
           signal: activeTurn.abortController.signal,
           outputFormat: "pcm",
           sampleRate: this.context.startFrame.audio.sampleRate,

package/src/live-voice/live-voice-tts.ts

@@ -1,3 +1,4 @@
+import { resolveLanguageVoiceOverride } from "../tts/language-voices.js";
 import { createPcmChunkAligner } from "../tts/pcm-chunk-aligner.js";
 import { getTtsProvider } from "../tts/provider-catalog.js";
 import { synthesizeAndEmit } from "../tts/synthesis-stream.js";
@@ -30,6 +31,7 @@ export interface LiveVoiceTtsOptions {
   useCase?: TtsUseCase;
   outputFormat?: TtsSynthesisRequest["outputFormat"];
   sampleRate?: number;
+  language?: string;
   config?: LiveVoiceTtsConfig;
   onAudioChunk: (chunk: LiveVoiceTtsAudioChunk) => void;
 }
@@ -72,11 +74,23 @@ interface ResolvedStreamingTtsProvider {
   providerConfig: Record<string, unknown>;
 }
 
+export { resolveLanguageVoiceOverride };
+
 export async function streamLiveVoiceTtsAudio(
   options: LiveVoiceTtsOptions,
 ): Promise<LiveVoiceTtsResult> {
   const { provider, providerId, providerConfig } =
     await resolveLiveVoiceStreamingTtsProvider(options.config);
+  // An explicit request voice wins outright; otherwise a language-known
+  // turn may select the provider's configured per-language voice. The cast
+  // recovers the schema-typed map that resolveTtsConfig's generic
+  // provider-block lookup erases.
+  const voiceId =
+    options.voiceId ??
+    resolveLanguageVoiceOverride(
+      providerConfig.languageVoices as Record<string, string> | undefined,
+      options.language,
+    );
   const useCase = options.useCase ?? "phone-call";
   const requestedSampleRate = resolveSampleRate(
     options.sampleRate,
@@ -89,9 +103,10 @@ export async function streamLiveVoiceTtsAudio(
   const providerSampleRate = provider.resolveOutputSampleRateHz?.({
     text: options.text,
     useCase,
-    voiceId: options.voiceId,
+    voiceId,
     outputFormat: options.outputFormat,
     sampleRateHz: requestedSampleRate,
+    language: options.language,
     signal: options.signal,
   });
   const sampleRate = providerSampleRate ?? requestedSampleRate;
@@ -138,9 +153,10 @@ export async function streamLiveVoiceTtsAudio(
       provider,
       text: options.text,
       useCase,
-      voiceId: options.voiceId,
+      voiceId,
       outputFormat: options.outputFormat,
       sampleRateHz: requestedSampleRate,
+      language: options.language,
       signal: options.signal,
       onChunk: (chunk) => {
         if (canStreamChunks) {

package/src/live-voice/progress-phrases.ts

@@ -1,3 +1,5 @@
+import { localizedOrDefault } from "../util/language-subtag.js";
+
 // Static fallbacks for an idle-triggered progress narration whose LLM
 // phrasing failed — the one case where prolonged silence is actively harmful.
 // The idle trigger can fire on a slow turn with zero tool activity, so every
@@ -10,8 +12,109 @@ export const PROGRESS_FALLBACK_PHRASES: readonly string[] = [
   "Almost there — thanks for waiting.",
 ];
 
+// Per-language fallback phrases, keyed by lowercased BCP 47 base subtag,
+// covering the Deepgram code-switching roster (DEEPGRAM_MULTI_LANGUAGE_CODES
+// in providers/speech-to-text/deepgram.ts). Every list carries the same
+// invariants as the English one above: persona-neutral, no claims about
+// running tools or tasks, at most 8 words per phrase.
+export const PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE: Readonly<
+  Record<string, readonly string[]>
+> = {
+  en: PROGRESS_FALLBACK_PHRASES,
+  es: [
+    "Sigo en ello, un momento.",
+    "Todavía lo estoy pensando.",
+    "Casi listo, gracias por esperar.",
+  ],
+  fr: [
+    "J'y suis encore, un instant.",
+    "J'y réfléchis encore.",
+    "Presque fini, merci de patienter.",
+  ],
+  de: [
+    "Bin noch dabei, einen Moment.",
+    "Ich denke noch darüber nach.",
+    "Fast fertig, danke fürs Warten.",
+  ],
+  hi: [
+    "बस एक पल रुकिए।",
+    "अभी इस पर विचार चल रहा है।",
+    "बस थोड़ा और इंतज़ार कीजिए, धन्यवाद।",
+  ],
+  ru: [
+    "Секундочку, я ещё здесь.",
+    "Я всё ещё думаю над этим.",
+    "Почти готово, спасибо за ожидание.",
+  ],
+  pt: [
+    "Ainda estou nisso, um momento.",
+    "Ainda estou pensando nisso.",
+    "Quase lá, agradeço a espera.",
+  ],
+  ja: [
+    "まだ対応中です。少々お待ちください。",
+    "まだ考えているところです。",
+    "もうすぐです。お待ちいただきありがとうございます。",
+  ],
+  it: [
+    "Ancora un attimo, per favore.",
+    "Ci sto ancora pensando.",
+    "Quasi fatto, grazie per l'attesa.",
+  ],
+  nl: [
+    "Ik ben er nog mee bezig.",
+    "Ik denk er nog over na.",
+    "Bijna klaar, bedankt voor het wachten.",
+  ],
+};
+
 // Deterministic rotation through the phrase list: callers hold a nonnegative
 // monotonic counter, so consecutive picks vary while tests stay reproducible.
-export function pickProgressPhrase(counter: number): string {
-  return PROGRESS_FALLBACK_PHRASES[counter % PROGRESS_FALLBACK_PHRASES.length];
+// `language` selects the per-language list by its lowercased base subtag
+// (e.g. "pt-BR" -> "pt"); unknown or absent languages fall back to English.
+export function pickProgressPhrase(counter: number, language?: string): string {
+  const phrases = localizedOrDefault(
+    PROGRESS_FALLBACK_PHRASES_BY_LANGUAGE,
+    language,
+    PROGRESS_FALLBACK_PHRASES,
+  );
+  return phrases[counter % phrases.length];
+}
+
+// Spoken once when a turn starts waiting on the user's approval decision.
+// Fixed rather than generated: this is a statement about the system's
+// state, not about the work, and it has to be true every time. Kept in the
+// shape of the progress phrases it displaces (short, neutral, no claim
+// about tools).
+export const APPROVAL_PENDING_PHRASE =
+  "I need your okay for that one. Take a look.";
+
+// Per-language spellings of the approval-pending phrase, keyed by lowercased
+// BCP 47 base subtag, covering the Deepgram code-switching roster
+// (DEEPGRAM_MULTI_LANGUAGE_CODES in providers/speech-to-text/deepgram.ts).
+// Same invariants as the progress phrases: persona-neutral, no claims about
+// running tools or tasks.
+export const APPROVAL_PENDING_PHRASE_BY_LANGUAGE: Readonly<
+  Record<string, string>
+> = {
+  en: APPROVAL_PENDING_PHRASE,
+  es: "Necesito tu visto bueno para eso. Échale un vistazo.",
+  fr: "J'ai besoin de ton accord pour ça. Jette un œil.",
+  de: "Dafür brauche ich dein Okay. Schau mal drauf.",
+  hi: "इसके लिए मुझे आपकी मंज़ूरी चाहिए। एक नज़र डाल लीजिए।",
+  ru: "Для этого мне нужно твоё согласие. Взгляни, пожалуйста.",
+  pt: "Preciso do seu ok para isso. Dê uma olhada.",
+  ja: "これには許可が必要です。ご確認ください。",
+  it: "Mi serve il tuo via libera per questo. Dai un'occhiata.",
+  nl: "Hiervoor heb ik je akkoord nodig. Kijk even mee.",
+};
+
+// The approval-pending phrase in the turn's spoken language, defaulting to
+// English for unknown or absent languages.
+export function approvalPendingPhraseFor(language?: string): string {
+  return localizedOrDefault(
+    APPROVAL_PENDING_PHRASE_BY_LANGUAGE,
+    language,
+    APPROVAL_PENDING_PHRASE,
+  );
 }