@elizaos/ui 2.0.0-beta.2 → 2.0.3-beta.2

package/hooks/useVoiceChat.js

@@ -6,7 +6,8 @@
  *     the stream grows (no sentence-boundary wait — lower time-to-first-audio).
  *  2. Browser SpeechSynthesis — fallback when ElevenLabs isn't configured.
  *
- * STT: Web Speech API (SpeechRecognition) for user voice input.
+ * STT: local-inference ASR on local desktop, then native TalkMode or browser
+ * SpeechRecognition fallback.
  */
 import { Capacitor } from "@capacitor/core";
 import { useCallback, useEffect, useEffectEvent, useMemo, useRef, useState, } from "react";
@@ -17,14 +18,38 @@ import { resolveApiUrl } from "../utils/index.js";
 import { getElizaApiToken } from "../utils/eliza-globals.js";
 import { isTtsDebugEnabled, ttsDebug, ttsDebugTextPreview, } from "../utils/tts-debug.js";
 import { hasConfiguredApiKey } from "../voice/index.js";
-import { collapseWhitespace, nextIdleMouthOpen, normalizeCacheText, normalizeMouthOpen, queueableSpeechPrefix, remainderAfter, splitFirstSentence, toSpeakableText, } from "../voice/voice-chat-playback.js";
+import { isLocalAsrCaptureSupported, startLocalAsrRecorder, } from "../voice/local-asr-capture.js";
+import { isLocalInferenceAsrReady, transcribeLocalInferenceWav, } from "../voice/local-asr-transcribe.js";
+import { collapseWhitespace, nextIdleMouthOpen, normalizeCacheText, normalizeMouthOpen, queueableSpeechPrefix, remainderAfter, shouldCacheGeneratedSpeech, splitFirstSentence, toSpeakableText, } from "../voice/voice-chat-playback.js";
 import { mergeTranscriptWindows } from "../voice/voice-chat-recording.js";
 import { ASSISTANT_TTS_DEBOUNCE_MS, ASSISTANT_TTS_FINAL_ONLY, ASSISTANT_TTS_FIRST_FLUSH_CHARS, ASSISTANT_TTS_MIN_CHUNK_CHARS, DEFAULT_ELEVEN_MODEL, DEFAULT_ELEVEN_VOICE, describeTtsCloudFetchTargetForDebug, getSpeechRecognitionCtor, globalAudioCache, isAbortError, localePrefix, MAX_CACHED_SEGMENTS, matchesVoiceLocale, normalizeSpeechLocale, resolveEffectiveVoiceConfig, resolveVoiceMode, resolveVoiceProxyEndpoint, TALKMODE_STOP_SETTLE_MS, toArrayBuffer, webSpeechVoiceDebugFields, } from "../voice/voice-chat-types.js";
 // ── Re-exports (public API) ──────────────────────────────────────────
 export { nextIdleMouthOpen } from "../voice/voice-chat-playback.js";
 // ── Shared mutable state ─────────────────────────────────────────────
 let sharedAudioCtx = null;
+const AUDIO_CONTEXT_RESUME_TIMEOUT_MS = 1200;
+const LOCAL_INFERENCE_TTS_TIMEOUT_MS = 60_000;
+/** How long the transient `micReconnected` pulse stays set after an auto-restart. */
+const MIC_RECONNECT_PULSE_MS = 1500;
 // ── Internal helpers ─────────────────────────────────────────────────
+async function resumeAudioContextForPlayback(ctx, timeoutMs = AUDIO_CONTEXT_RESUME_TIMEOUT_MS) {
+    if (ctx.state !== "suspended")
+        return true;
+    let timeoutId;
+    try {
+        const resumed = await Promise.race([
+            ctx.resume().then(() => true, () => false),
+            new Promise((resolve) => {
+                timeoutId = setTimeout(() => resolve(false), timeoutMs);
+            }),
+        ]);
+        return resumed && ctx.state !== "suspended";
+    }
+    finally {
+        if (timeoutId)
+            clearTimeout(timeoutId);
+    }
+}
 function shouldPreferNativeTalkMode() {
     if (typeof window === "undefined")
         return false;
@@ -44,11 +69,28 @@ function shouldAutoRestartBrowserRecognition() {
     }
     return true;
 }
+function shouldUseLocalInferenceAsr(config) {
+    return config?.asr?.provider === "local-inference";
+}
+const ACTIVE_VOICE_SESSION_MODES = new Set([
+    "compose",
+    "push-to-talk",
+    "hands-free",
+    "passive",
+]);
+function normalizeActiveVoiceSessionMode(mode) {
+    return typeof mode === "string" &&
+        ACTIVE_VOICE_SESSION_MODES.has(mode)
+        ? mode
+        : null;
+}
 // ── Test-visible internals ───────────────────────────────────────────
 export const __voiceChatInternals = {
     isWindowsElectrobunRenderer,
     shouldPreferNativeTalkMode,
     shouldAutoRestartBrowserRecognition,
+    shouldUseLocalInferenceAsr,
+    resumeAudioContextForPlayback,
     splitFirstSentence,
     remainderAfter,
     queueableSpeechPrefix,
@@ -72,8 +114,18 @@ export function useVoiceChat(options) {
     const [supported, setSupported] = useState(false);
     const [usingAudioAnalysis, setUsingAudioAnalysis] = useState(false);
     const [voiceUnlockedGeneration, setVoiceUnlockedGeneration] = useState(0);
+    // True when a TTS clip was blocked because the AudioContext is still
+    // suspended (browser autoplay policy). Callers surface an "tap to enable
+    // sound" hint. Cleared on the next user-gesture unlock.
+    const [needsAudioUnlock, setNeedsAudioUnlock] = useState(false);
+    // Transient pulse: set when browser SpeechRecognition silently auto-restarts
+    // (the engine ends a segment mid-session and we restart it). Lets callers
+    // flash a brief "reconnected" indicator instead of the restart being silent.
+    const [micReconnected, setMicReconnected] = useState(false);
+    const micReconnectTimerRef = useRef(null);
     // Refs — stable across renders, read from animation loop & callbacks
     const recognitionRef = useRef(null);
+    const localAsrRecorderRef = useRef(null);
     const sttBackendRef = useRef(null);
     const talkModeHandlesRef = useRef([]);
     const synthRef = useRef(null);
@@ -84,14 +136,21 @@ export function useVoiceChat(options) {
     const enabledRef = useRef(false);
     const listeningModeRef = useRef("idle");
     const transcriptBufferRef = useRef("");
-    const emitTranscript = useEffectEvent((text) => {
-        options.onTranscript(text);
+    const latestTranscriptTurnRef = useRef(null);
+    const emitTranscript = useEffectEvent((text, event) => {
+        options.onTranscript(text, event);
     });
     const emitTranscriptPreview = useEffectEvent((text, event) => {
         options.onTranscriptPreview?.(text, event);
     });
     const emitPlaybackStart = useEffectEvent((event) => {
         options.onPlaybackStart?.(event);
+        // Headless-test observability: a real TTS playback actually started. The
+        // voice self-test + real-audio e2e poll this to prove audio flowed (the
+        // only honest "reply was spoken" signal in a headless browser).
+        if (typeof window !== "undefined") {
+            window.__voicePlaybackStarted = true;
+        }
     });
     const effectiveVoiceConfig = useMemo(() => resolveEffectiveVoiceConfig(options.voiceConfig, {
         cloudConnected: options.cloudConnected,
@@ -155,6 +214,31 @@ export function useVoiceChat(options) {
             speechTimeoutRef.current = null;
         }
     }, []);
+    // Playback was rejected because the AudioContext is still suspended (autoplay
+    // policy). Raise the unlock hint so a caller can prompt for a user gesture.
+    const markAudioBlocked = useCallback(() => {
+        setNeedsAudioUnlock(true);
+    }, []);
+    // A clip played (or the context was already running). Any prior unlock hint is
+    // stale, so clear it.
+    const markAudioPlaying = useCallback(() => {
+        setNeedsAudioUnlock(false);
+    }, []);
+    // Explicitly unlock audio playback in response to a user gesture (e.g. the
+    // status-bar "enable sound" button). Warms/resumes the shared AudioContext,
+    // bumps the unlock generation so a blocked greeting retries, and clears the
+    // hint. The passive first-gesture listener covers the common case; this covers
+    // a context that re-suspended after the initial unlock (tab switch, etc.).
+    const unlockAudio = useCallback(() => {
+        if (typeof window === "undefined")
+            return;
+        if (!sharedAudioCtx) {
+            sharedAudioCtx = new AudioContext({ latencyHint: "interactive" });
+        }
+        void sharedAudioCtx.resume().catch(() => { });
+        setVoiceUnlockedGeneration((g) => g + 1);
+        setNeedsAudioUnlock(false);
+    }, []);
     const rememberCachedSegment = useCallback((key, bytes) => {
         globalAudioCache.delete(key);
         globalAudioCache.set(key, bytes);
@@ -184,6 +268,7 @@ export function useVoiceChat(options) {
             normalizeCacheText(text),
         ].join("|");
     }, []);
+    const makeLocalInferenceCacheKey = useCallback((text) => ["local-inference", normalizeCacheText(text)].join("|"), []);
     const updateMouthOpen = useCallback((value) => {
         const previousValue = mouthOpenRef.current;
         const resolvedValue = typeof value === "function" ? value(previousValue) : value;
@@ -199,6 +284,14 @@ export function useVoiceChat(options) {
         let cancelled = false;
         const syncVoiceSupport = async () => {
             const browserSpeechSupported = !!getSpeechRecognitionCtor();
+            const localAsrSupported = shouldUseLocalInferenceAsr(voiceConfigRef.current) &&
+                isLocalAsrCaptureSupported();
+            if (localAsrSupported) {
+                if (!cancelled) {
+                    setSupported(true);
+                }
+                return;
+            }
             if (!shouldPreferNativeTalkMode()) {
                 if (!cancelled) {
                     setSupported(browserSpeechSupported);
@@ -293,8 +386,8 @@ export function useVoiceChat(options) {
         };
     }, [isSpeaking, updateMouthOpen]);
     // ── STT (Speech Recognition) ──────────────────────────────────────
-    const applyTranscriptUpdate = useCallback((transcript, isFinal) => {
-        const mode = listeningModeRef.current;
+    const applyTranscriptUpdate = useCallback((transcript, isFinal, metadata = {}) => {
+        const mode = metadata.mode ?? listeningModeRef.current;
         if (mode === "idle")
             return;
         const normalized = collapseWhitespace(transcript);
@@ -305,13 +398,42 @@ export function useVoiceChat(options) {
             return;
         transcriptBufferRef.current = nextText;
         setInterimTranscript(nextText);
+        const turn = {
+            ...metadata.turn,
+            text: nextText,
+            mode,
+            isFinal,
+            speaker: metadata.speaker ?? metadata.turn?.speaker,
+            source: metadata.source ??
+                metadata.turn?.source ??
+                sttBackendRef.current ??
+                undefined,
+            confidence: metadata.confidence ?? metadata.turn?.confidence,
+            metadata: metadata.metadata ?? metadata.turn?.metadata,
+        };
+        latestTranscriptTurnRef.current = turn;
         emitTranscriptPreview(nextText, {
+            text: nextText,
             mode,
             isFinal,
+            turn,
+            speaker: turn.speaker,
         });
         if (interruptOnSpeechRef.current) {
             interruptSpeechRef.current();
         }
+        if (isFinal && mode === "passive") {
+            emitTranscript(nextText, {
+                text: nextText,
+                mode,
+                isFinal: true,
+                turn,
+                speaker: turn.speaker,
+            });
+            transcriptBufferRef.current = "";
+            latestTranscriptTurnRef.current = null;
+            setInterimTranscript("");
+        }
     }, []);
     const removeTalkModeListeners = useCallback(async () => {
         const handles = talkModeHandlesRef.current;
@@ -322,7 +444,9 @@ export function useVoiceChat(options) {
     }, []);
     const resetListeningState = useCallback(() => {
         transcriptBufferRef.current = "";
+        latestTranscriptTurnRef.current = null;
         recognitionRef.current = null;
+        localAsrRecorderRef.current = null;
         sttBackendRef.current = null;
         enabledRef.current = false;
         listeningModeRef.current = "idle";
@@ -335,7 +459,15 @@ export function useVoiceChat(options) {
             return;
         const talkMode = getTalkModePlugin();
         const transcriptHandle = await talkMode.addListener("transcript", (event) => {
-            applyTranscriptUpdate(event.transcript ?? "", event.isFinal === true);
+            const typedEvent = event;
+            applyTranscriptUpdate(event.transcript ?? "", event.isFinal === true, {
+                mode: normalizeActiveVoiceSessionMode(typedEvent.mode) ?? undefined,
+                speaker: typedEvent.speaker,
+                source: typedEvent.source,
+                confidence: typedEvent.confidence,
+                turn: typedEvent.turn,
+                metadata: typedEvent.metadata,
+            });
         });
         const errorHandle = await talkMode.addListener("error", (event) => {
             if (sttBackendRef.current === "talkmode" ||
@@ -356,6 +488,39 @@ export function useVoiceChat(options) {
         });
         talkModeHandlesRef.current = [transcriptHandle, errorHandle, stateHandle];
     }, [applyTranscriptUpdate, resetListeningState]);
+    const transcribeLocalInferenceAudio = useCallback(async (audio, signal) => {
+        const { text } = await transcribeLocalInferenceWav(audio, { signal });
+        return text;
+    }, []);
+    const startLocalInferenceRecognition = useCallback(async (mode) => {
+        if (!shouldUseLocalInferenceAsr(voiceConfigRef.current)) {
+            return false;
+        }
+        if (!isLocalAsrCaptureSupported()) {
+            return false;
+        }
+        // Defer to the next backend (talk-mode / browser) when the server can't
+        // transcribe right now — capturing here would only 502 at stop() with no
+        // recoverable fallback (no whisper model / native adapter installed).
+        if (!(await isLocalInferenceAsrReady())) {
+            return false;
+        }
+        try {
+            const recorder = await startLocalAsrRecorder();
+            localAsrRecorderRef.current = recorder;
+            sttBackendRef.current = "local-inference";
+            enabledRef.current = true;
+            listeningModeRef.current = mode;
+            setSupported(true);
+            setCaptureMode(mode);
+            setIsListening(true);
+            return true;
+        }
+        catch {
+            localAsrRecorderRef.current = null;
+            return false;
+        }
+    }, []);
     const startBrowserRecognition = useCallback((mode) => {
         const SpeechRecognitionAPI = getSpeechRecognitionCtor();
         if (!SpeechRecognitionAPI)
@@ -395,6 +560,16 @@ export function useVoiceChat(options) {
                 listeningModeRef.current === mode) {
                 try {
                     recognition.start();
+                    // Surface the silent restart as a brief pulse so the UI can show a
+                    // "mic reconnected" indicator instead of an unexplained gap.
+                    setMicReconnected(true);
+                    if (micReconnectTimerRef.current !== null) {
+                        clearTimeout(micReconnectTimerRef.current);
+                    }
+                    micReconnectTimerRef.current = setTimeout(() => {
+                        micReconnectTimerRef.current = null;
+                        setMicReconnected(false);
+                    }, MIC_RECONNECT_PULSE_MS);
                 }
                 catch {
                     /* already started */
@@ -420,14 +595,13 @@ export function useVoiceChat(options) {
         if (!shouldPreferNativeTalkMode()) {
             return false;
         }
-        await ensureTalkModeListeners();
         try {
+            await ensureTalkModeListeners();
             const talkMode = getTalkModePlugin();
             const browserSpeechSupported = !!getSpeechRecognitionCtor();
             let permissions = await talkMode.checkPermissions().catch(() => null);
             const nativeSpeechSupported = permissions?.speechRecognition !== "not_supported";
             if (!nativeSpeechSupported && !browserSpeechSupported) {
-                console.warn("[useVoiceChat] No desktop or browser speech backend is available.");
                 setSupported(false);
                 return false;
             }
@@ -443,7 +617,7 @@ export function useVoiceChat(options) {
             const result = await talkMode.start({
                 config: {
                     stt: {
-                        ...(directRpc ? { engine: "whisper" } : {}),
+                        ...(directRpc ? { engine: "web" } : {}),
                         language: options.lang ?? "en-US",
                         modelSize: "base",
                         sampleRate: 16000,
@@ -453,10 +627,6 @@ export function useVoiceChat(options) {
                 },
             });
             if (!result.started) {
-                console.warn("[useVoiceChat] TalkMode start returned not started.", {
-                    browserSpeechSupported,
-                    error: result.error,
-                });
                 if (!browserSpeechSupported) {
                     setSupported(false);
                 }
@@ -470,15 +640,29 @@ export function useVoiceChat(options) {
             setIsListening(true);
             return true;
         }
-        catch (error) {
-            console.warn("[useVoiceChat] TalkMode start failed.", error);
+        catch {
             return false;
         }
     }, [ensureTalkModeListeners, options.lang]);
     const finalizeRecognition = useCallback((submit) => {
+        const mode = normalizeActiveVoiceSessionMode(listeningModeRef.current) ?? "compose";
         const transcript = collapseWhitespace(transcriptBufferRef.current);
         if (submit && transcript) {
-            emitTranscript(transcript);
+            const latestTurn = latestTranscriptTurnRef.current;
+            const turn = {
+                ...latestTurn,
+                text: transcript,
+                mode,
+                isFinal: true,
+                endedAtMs: latestTurn?.endedAtMs ?? Date.now(),
+            };
+            emitTranscript(transcript, {
+                text: transcript,
+                mode,
+                isFinal: true,
+                turn,
+                speaker: turn.speaker,
+            });
         }
         resetListeningState();
     }, [resetListeningState]);
@@ -490,17 +674,22 @@ export function useVoiceChat(options) {
         if (interruptOnSpeechRef.current) {
             interruptSpeechRef.current();
         }
+        const localStarted = await startLocalInferenceRecognition(mode);
+        if (localStarted) {
+            return;
+        }
         if (shouldPreferNativeTalkMode()) {
             const started = await startTalkModeRecognition(mode);
             if (started) {
                 return;
             }
         }
-        const startedInBrowser = startBrowserRecognition(mode);
-        if (!startedInBrowser) {
-            console.warn("[useVoiceChat] Voice capture failed to start in both desktop and browser backends.");
-        }
-    }, [startBrowserRecognition, startTalkModeRecognition]);
+        startBrowserRecognition(mode);
+    }, [
+        startBrowserRecognition,
+        startLocalInferenceRecognition,
+        startTalkModeRecognition,
+    ]);
     const stopListening = useCallback(async (options) => {
         const mode = listeningModeRef.current;
         if (mode === "idle")
@@ -515,12 +704,34 @@ export function useVoiceChat(options) {
             });
             await new Promise((resolve) => window.setTimeout(resolve, TALKMODE_STOP_SETTLE_MS));
         }
+        else if (sttBackendRef.current === "local-inference") {
+            const recorder = localAsrRecorderRef.current;
+            localAsrRecorderRef.current = null;
+            if (recorder) {
+                try {
+                    const audio = await recorder.stop();
+                    const transcript = await transcribeLocalInferenceAudio(audio);
+                    applyTranscriptUpdate(transcript, true, {
+                        mode: normalizeActiveVoiceSessionMode(mode) ??
+                            normalizeActiveVoiceSessionMode(listeningModeRef.current) ??
+                            "compose",
+                        source: "local-inference",
+                        metadata: { source: "local-inference" },
+                    });
+                }
+                catch (error) {
+                    ttsDebug("asr:local-inference:error", {
+                        message: error instanceof Error ? error.message : String(error),
+                    });
+                }
+            }
+        }
         else {
             recognitionRef.current?.stop();
             await new Promise((resolve) => window.setTimeout(resolve, TALKMODE_STOP_SETTLE_MS));
         }
         finalizeRecognition(submit);
-    }, [finalizeRecognition]);
+    }, [applyTranscriptUpdate, finalizeRecognition, transcribeLocalInferenceAudio]);
     const toggleListening = useCallback(() => {
         if (enabledRef.current && listeningModeRef.current === "compose") {
             void stopListening();
@@ -558,6 +769,14 @@ export function useVoiceChat(options) {
             }
             audioSourceRef.current = null;
         }
+        // Native TalkMode TTS — interrupt any in-flight native speak so barge-in
+        // actually silences the agent. Without this the awaited TalkMode.speak()
+        // plays to completion and the agent talks over the user.
+        if (Capacitor.isNativePlatform()) {
+            void getTalkModePlugin()
+                .stopSpeaking?.()
+                .catch(() => { });
+        }
         clearSpeechTimers();
         usingAudioAnalysisRef.current = false;
         setUsingAudioAnalysis(false);
@@ -577,29 +796,31 @@ export function useVoiceChat(options) {
     const speakElevenLabs = useCallback(async (text, elConfig, task, generation) => {
         let ctx = sharedAudioCtx;
         if (!ctx) {
-            ctx = new AudioContext();
+            ctx = new AudioContext({ latencyHint: "interactive" });
             sharedAudioCtx = ctx;
         }
         if (ctx.state === "suspended") {
-            try {
-                await ctx.resume();
-            }
-            catch {
-                // Force a fresh context if resume fails
-                ctx.close().catch((err) => {
-                    console.warn("[useVoiceChat] AudioContext.close() failed", err);
+            const resumed = await resumeAudioContextForPlayback(ctx);
+            if (!resumed) {
+                ttsDebug("play:audio-context-blocked", {
+                    provider: "elevenlabs",
+                    state: ctx.state,
                 });
-                ctx = new AudioContext();
-                sharedAudioCtx = ctx;
+                markAudioBlocked();
+                throw new DOMException("Audio playback is blocked until a user gesture unlocks the audio context", "NotAllowedError");
             }
         }
+        markAudioPlaying();
         const voiceId = elConfig.voiceId ?? DEFAULT_ELEVEN_VOICE;
         const modelId = elConfig.modelId ?? DEFAULT_ELEVEN_MODEL;
-        const cacheKey = task.cacheKey ?? makeElevenCacheKey(text, elConfig);
-        const cachedBytes = globalAudioCache.get(cacheKey);
+        const cacheKey = task.cacheKey ??
+            (shouldCacheGeneratedSpeech(text, task.segment)
+                ? makeElevenCacheKey(text, elConfig)
+                : undefined);
+        const cachedBytes = cacheKey ? globalAudioCache.get(cacheKey) : undefined;
         let audioBytes = null;
         let cached = false;
-        if (cachedBytes) {
+        if (cacheKey && cachedBytes) {
             rememberCachedSegment(cacheKey, cachedBytes);
             audioBytes = cachedBytes.slice();
             cached = true;
@@ -678,7 +899,7 @@ export function useVoiceChat(options) {
                         error: error instanceof Error ? error.message : String(error),
                     });
                 }
-                return await fetchWithCsrf(resolveApiUrl("/api/tts/elevenlabs"), makeProxyRequestInit());
+                return fetchWithCsrf(resolveApiUrl("/api/tts/elevenlabs"), makeProxyRequestInit());
             };
             const trimmedApiKey = typeof elConfig.apiKey === "string" ? elConfig.apiKey.trim() : "";
             const hasDirectKey = hasConfiguredApiKey(trimmedApiKey);
@@ -727,7 +948,9 @@ export function useVoiceChat(options) {
             }
             const audioData = await res.arrayBuffer();
             audioBytes = new Uint8Array(audioData);
-            rememberCachedSegment(cacheKey, audioBytes.slice());
+            if (cacheKey) {
+                rememberCachedSegment(cacheKey, audioBytes.slice());
+            }
         }
         if (generation !== generationRef.current)
             return;
@@ -799,9 +1022,146 @@ export function useVoiceChat(options) {
                 provider: "elevenlabs",
                 cached,
                 startedAtMs: playStartMs,
+                ...task.telemetry,
             });
         });
-    }, [clearSpeechTimers, makeElevenCacheKey, rememberCachedSegment]);
+    }, [
+        clearSpeechTimers,
+        makeElevenCacheKey,
+        markAudioBlocked,
+        markAudioPlaying,
+        rememberCachedSegment,
+    ]);
+    // ── Local inference TTS ───────────────────────────────────────────────
+    const speakLocalInference = useCallback(async (text, task, generation) => {
+        let ctx = sharedAudioCtx;
+        if (!ctx) {
+            ctx = new AudioContext({ latencyHint: "interactive" });
+            sharedAudioCtx = ctx;
+        }
+        if (ctx.state === "suspended") {
+            const resumed = await resumeAudioContextForPlayback(ctx);
+            if (!resumed) {
+                ttsDebug("play:audio-context-blocked", {
+                    provider: "local-inference",
+                    state: ctx.state,
+                });
+                markAudioBlocked();
+                throw new DOMException("Audio playback is blocked until a user gesture unlocks the audio context", "NotAllowedError");
+            }
+        }
+        markAudioPlaying();
+        const cacheKey = task.cacheKey ??
+            (shouldCacheGeneratedSpeech(text, task.segment)
+                ? makeLocalInferenceCacheKey(text)
+                : undefined);
+        const cachedBytes = cacheKey ? globalAudioCache.get(cacheKey) : undefined;
+        let audioBytes = null;
+        let cached = false;
+        if (cacheKey && cachedBytes) {
+            rememberCachedSegment(cacheKey, cachedBytes);
+            audioBytes = cachedBytes.slice();
+            cached = true;
+        }
+        if (!audioBytes) {
+            const controller = new AbortController();
+            activeFetchAbortRef.current = controller;
+            const timeoutId = setTimeout(() => {
+                controller.abort(new DOMException("Local inference TTS timed out", "TimeoutError"));
+            }, LOCAL_INFERENCE_TTS_TIMEOUT_MS);
+            let res;
+            try {
+                res = await fetchWithCsrf(resolveApiUrl("/api/tts/local-inference"), {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        Accept: "audio/wav, audio/*;q=0.9",
+                    },
+                    body: JSON.stringify({ text }),
+                    signal: controller.signal,
+                });
+            }
+            finally {
+                clearTimeout(timeoutId);
+                if (activeFetchAbortRef.current === controller) {
+                    activeFetchAbortRef.current = null;
+                }
+            }
+            if (!res.ok) {
+                const body = await res.text().catch(() => "");
+                throw new Error(`Local inference TTS ${res.status}: ${body.slice(0, 200)}`);
+            }
+            audioBytes = new Uint8Array(await res.arrayBuffer());
+            if (cacheKey) {
+                rememberCachedSegment(cacheKey, audioBytes.slice());
+            }
+        }
+        if (generation !== generationRef.current)
+            return;
+        const audioBuffer = await ctx.decodeAudioData(toArrayBuffer(audioBytes));
+        if (generation !== generationRef.current)
+            return;
+        const analyser = ctx.createAnalyser();
+        analyser.fftSize = 2048;
+        analyser.smoothingTimeConstant = 0.8;
+        analyserRef.current = analyser;
+        timeDomainDataRef.current = new Float32Array(new ArrayBuffer(analyser.fftSize * Float32Array.BYTES_PER_ELEMENT));
+        const source = ctx.createBufferSource();
+        source.buffer = audioBuffer;
+        source.connect(analyser);
+        analyser.connect(ctx.destination);
+        audioSourceRef.current = source;
+        await new Promise((resolve) => {
+            let finished = false;
+            const playStartMs = performance.now();
+            let wrappedFinish = null;
+            const finish = () => {
+                if (finished)
+                    return;
+                finished = true;
+                if (wrappedFinish && activeTaskFinishRef.current === wrappedFinish) {
+                    activeTaskFinishRef.current = null;
+                }
+                if (audioSourceRef.current === source) {
+                    audioSourceRef.current = null;
+                }
+                source.onended = null;
+                try {
+                    source.disconnect();
+                }
+                catch {
+                    /* ok */
+                }
+                try {
+                    analyser.disconnect();
+                }
+                catch {
+                    /* ok */
+                }
+                clearSpeechTimers();
+                resolve();
+            };
+            wrappedFinish = finish;
+            activeTaskFinishRef.current = wrappedFinish;
+            source.onended = wrappedFinish;
+            speechTimeoutRef.current = setTimeout(wrappedFinish, Math.max(2500, Math.ceil(audioBuffer.duration * 1000) + 1200));
+            source.start(0);
+            emitPlaybackStart({
+                text,
+                segment: task.segment,
+                provider: "local-inference",
+                cached,
+                startedAtMs: playStartMs,
+                ...task.telemetry,
+            });
+        });
+    }, [
+        clearSpeechTimers,
+        makeLocalInferenceCacheKey,
+        markAudioBlocked,
+        markAudioPlaying,
+        rememberCachedSegment,
+    ]);
     // ── Browser SpeechSynthesis TTS ───────────────────────────────────
     const speakBrowser = useCallback((text, task, generation) => {
         const config = voiceConfigRef.current;
@@ -815,7 +1175,7 @@ export function useVoiceChat(options) {
                 ? "speechSynthesis"
                 : useTalkModeTts
                     ? "talkmode-bridge"
-                    : "no-synth-timer-only",
+                    : "no-synth",
             segment: task.segment,
             append: task.append,
             textChars: text.trim().length,
@@ -840,45 +1200,45 @@ export function useVoiceChat(options) {
             };
             activeTaskFinishRef.current = finish;
             if (!synth) {
-                if (getElectrobunRendererRpc()) {
-                    ttsDebug("play:talkmode:dispatch", {
+                if (!getElectrobunRendererRpc()) {
+                    ttsDebug("play:browser:no-synth", {
                         segment: task.segment,
-                        append: task.append,
                         textChars: text.trim().length,
                         preview: ttsDebugTextPreview(text),
-                        engine: "native-talkmode-bridge",
-                        note: "No window.speechSynthesis — routing TTS to main-process talkmodeSpeak",
-                    });
-                    void invokeDesktopBridgeRequest({
-                        rpcMethod: "talkmodeSpeak",
-                        ipcChannel: "talkmode:speak",
-                        params: { text: text.trim() },
-                    }).catch((err) => {
-                        ttsDebug("play:talkmode:speak-failed", {
-                            segment: task.segment,
-                            preview: ttsDebugTextPreview(text),
-                            err: err instanceof Error
-                                ? `${err.name}: ${err.message.slice(0, 200)}`
-                                : String(err).slice(0, 200),
-                        });
-                        console.warn("[useVoiceChat] Desktop speech bridge failed:", err);
+                        engine: "none",
+                        note: "No SpeechSynthesis or Talk Mode bridge; no playback emitted",
                     });
+                    finish();
+                    return;
                 }
-                else {
-                    ttsDebug("play:browser:no-synth", {
+                ttsDebug("play:talkmode:dispatch", {
+                    segment: task.segment,
+                    append: task.append,
+                    textChars: text.trim().length,
+                    preview: ttsDebugTextPreview(text),
+                    engine: "native-talkmode-bridge",
+                    note: "No window.speechSynthesis — routing TTS to main-process talkmodeSpeak",
+                });
+                void invokeDesktopBridgeRequest({
+                    rpcMethod: "talkmodeSpeak",
+                    ipcChannel: "talkmode:speak",
+                    params: { text: text.trim() },
+                }).catch((err) => {
+                    ttsDebug("play:talkmode:speak-failed", {
                         segment: task.segment,
-                        textChars: text.trim().length,
                         preview: ttsDebugTextPreview(text),
-                        engine: "none",
-                        note: "No SpeechSynthesis — playback may be silent until Talk Mode or synth is available",
+                        err: err instanceof Error
+                            ? `${err.name}: ${err.message.slice(0, 200)}`
+                            : String(err).slice(0, 200),
                     });
-                }
+                });
                 emitPlaybackStart({
                     text,
                     segment: task.segment,
                     provider: "browser",
                     cached: false,
                     startedAtMs: performance.now(),
+                    ...task.telemetry,
                 });
                 speechTimeoutRef.current = setTimeout(finish, estimatedMs);
                 return;
@@ -907,7 +1267,7 @@ export function useVoiceChat(options) {
                             }
                             else {
                                 return (nameLower.includes("female") ||
-                                    nameLower.includes("samantha") ||
+                                    nameLower.includes("sam") ||
                                     nameLower.includes("victoria") ||
                                     nameLower.includes("zira") ||
                                     nameLower.includes("karen"));
@@ -962,6 +1322,7 @@ export function useVoiceChat(options) {
                     provider: "browser",
                     cached: false,
                     startedAtMs: browserPlayStartMsRef.value,
+                    ...task.telemetry,
                 });
             };
             const endBrowserUtterance = () => {
@@ -995,6 +1356,7 @@ export function useVoiceChat(options) {
         queueWorkerRunningRef.current = true;
         const workerGeneration = generationRef.current;
         void (async () => {
+            let workerError = null;
             try {
                 while (queueRef.current.length > 0) {
                     if (workerGeneration !== generationRef.current)
@@ -1005,8 +1367,10 @@ export function useVoiceChat(options) {
                     const config = voiceConfigRef.current;
                     const elConfig = config?.elevenlabs;
                     const useElevenLabs = config?.provider === "elevenlabs";
+                    const useLocalInference = config?.provider === "local-inference";
                     ttsDebug("processQueue:task", {
                         useElevenLabs,
+                        useLocalInference,
                         hasElConfig: Boolean(elConfig),
                         segment: task.segment,
                         append: task.append,
@@ -1019,6 +1383,85 @@ export function useVoiceChat(options) {
                             }
                             : {}),
                     });
+                    // Native mobile (Android/iOS Capacitor): route the reply through the
+                    // native TalkMode engine (Kotlin AudioTrack / on-device local-inference
+                    // TTS) per the Android TTS-owner decision, instead of the WebView
+                    // AudioContext path. Falls through to the web TTS path on a native error.
+                    if (Capacitor.isNativePlatform()) {
+                        const trimmed = task.text.trim();
+                        if (!trimmed)
+                            continue;
+                        usingAudioAnalysisRef.current = false;
+                        setUsingAudioAnalysis(false);
+                        emitPlaybackStart({
+                            text: task.text,
+                            segment: task.segment,
+                            provider: "browser",
+                            cached: false,
+                            startedAtMs: performance.now(),
+                            ...task.telemetry,
+                        });
+                        try {
+                            const result = await getTalkModePlugin().speak({
+                                text: trimmed,
+                                useLocalInferenceTts: true,
+                            });
+                            if (workerGeneration !== generationRef.current)
+                                break;
+                            if (result?.error)
+                                throw new Error(result.error);
+                            continue;
+                        }
+                        catch (error) {
+                            if (workerGeneration !== generationRef.current ||
+                                isAbortError(error)) {
+                                break;
+                            }
+                            ttsDebug("useVoiceChat:native-talkmode-failed-fallback", {
+                                err: error instanceof Error
+                                    ? `${error.name}: ${error.message.slice(0, 200)}`
+                                    : String(error).slice(0, 200),
+                            });
+                            // Fall through to the web TTS path (elevenlabs/local/browser).
+                        }
+                    }
+                    if (useLocalInference) {
+                        usingAudioAnalysisRef.current = true;
+                        setUsingAudioAnalysis(true);
+                        try {
+                            await speakLocalInference(task.text, task, workerGeneration);
+                            continue;
+                        }
+                        catch (error) {
+                            if (workerGeneration !== generationRef.current ||
+                                isAbortError(error)) {
+                                break;
+                            }
+                            usingAudioAnalysisRef.current = false;
+                            setUsingAudioAnalysis(false);
+                            ttsDebug("useVoiceChat:local-inference-failed", {
+                                err: error instanceof Error
+                                    ? `${error.name}: ${error.message.slice(0, 200)}`
+                                    : String(error).slice(0, 200),
+                            });
+                            // Local TTS failed — fall back to browser SpeechSynthesis so the
+                            // reply stays audible rather than going silent. Only hard-error
+                            // if the fallback ALSO fails.
+                            try {
+                                await speakBrowser(task.text, task, workerGeneration);
+                                continue;
+                            }
+                            catch (fallbackError) {
+                                if (workerGeneration !== generationRef.current ||
+                                    isAbortError(fallbackError)) {
+                                    break;
+                                }
+                                workerError = fallbackError;
+                                queueRef.current = [];
+                                break;
+                            }
+                        }
+                    }
                     if (useElevenLabs && elConfig) {
                         usingAudioAnalysisRef.current = true;
                         setUsingAudioAnalysis(true);
@@ -1031,9 +1474,6 @@ export function useVoiceChat(options) {
                                 isAbortError(error)) {
                                 break;
                             }
-                            console.warn("[useVoiceChat] ElevenLabs TTS failed:", error instanceof Error
-                                ? `${error.name}: ${error.message}`
-                                : error);
                             ttsDebug("useVoiceChat:elevenlabs-failed", {
                                 err: error instanceof Error
                                     ? `${error.name}: ${error.message.slice(0, 200)}`
@@ -1043,7 +1483,23 @@ export function useVoiceChat(options) {
                             });
                             usingAudioAnalysisRef.current = false;
                             setUsingAudioAnalysis(false);
-                            throw error;
+                            // Cloud TTS failed (commonly: no ElevenLabs / cloud key on a
+                            // default deploy). Fall back to the browser's built-in
+                            // SpeechSynthesis so the reply is still AUDIBLE instead of going
+                            // silent. Only hard-error if that fallback ALSO fails.
+                            try {
+                                await speakBrowser(task.text, task, workerGeneration);
+                                continue;
+                            }
+                            catch (fallbackError) {
+                                if (workerGeneration !== generationRef.current ||
+                                    isAbortError(fallbackError)) {
+                                    break;
+                                }
+                                workerError = fallbackError;
+                                queueRef.current = [];
+                                break;
+                            }
                         }
                     }
                     else {
@@ -1060,11 +1516,30 @@ export function useVoiceChat(options) {
                     await speakBrowser(task.text, task, workerGeneration);
                 }
             }
+            catch (error) {
+                workerError = error;
+                queueRef.current = [];
+                ttsDebug("processQueue:error", {
+                    err: error instanceof Error
+                        ? `${error.name}: ${error.message.slice(0, 200)}`
+                        : String(error).slice(0, 200),
+                });
+            }
             finally {
                 queueWorkerRunningRef.current = false;
             }
-            if (workerGeneration !== generationRef.current)
+            if (workerGeneration !== generationRef.current) {
+                if (queueRef.current.length > 0) {
+                    processQueue();
+                }
+                return;
+            }
+            if (workerError) {
+                usingAudioAnalysisRef.current = false;
+                setUsingAudioAnalysis(false);
+                setIsSpeaking(false);
                 return;
+            }
             if (queueRef.current.length > 0) {
                 processQueue();
                 return;
@@ -1073,7 +1548,7 @@ export function useVoiceChat(options) {
             setUsingAudioAnalysis(false);
             setIsSpeaking(false);
         })();
-    }, [speakBrowser, speakElevenLabs]);
+    }, [speakBrowser, speakElevenLabs, speakLocalInference]);
     const enqueueSpeech = useCallback((task) => {
         const speakable = toSpeakableText(task.text);
         if (!speakable)
@@ -1081,7 +1556,18 @@ export function useVoiceChat(options) {
         if (!task.append) {
             cancelPlayback();
         }
-        queueRef.current.push({ ...task, text: speakable });
+        queueRef.current.push({
+            ...task,
+            text: speakable,
+            telemetry: task.telemetry
+                ? {
+                    ...task.telemetry,
+                    queuedAtMs: typeof performance !== "undefined"
+                        ? performance.now()
+                        : Date.now(),
+                }
+                : undefined,
+        });
         ttsDebug("enqueueSpeech", {
             segment: task.segment,
             append: task.append,
@@ -1124,7 +1610,11 @@ export function useVoiceChat(options) {
         if (!unsent)
             return;
         const elConfig = voiceConfigRef.current?.elevenlabs;
-        const cacheKey = voiceConfigRef.current?.provider === "elevenlabs" && elConfig
+        const isFirstClip = state.queuedSpeakablePrefix.length === 0;
+        const segment = isFirstClip ? "full" : "remainder";
+        const cacheKey = voiceConfigRef.current?.provider === "elevenlabs" &&
+            elConfig &&
+            shouldCacheGeneratedSpeech(unsent, segment)
             ? makeElevenCacheKey(unsent, elConfig)
             : undefined;
         const dbgUtterance = isTtsDebugEnabled()
@@ -1133,17 +1623,17 @@ export function useVoiceChat(options) {
                 fullAssistTextPreview: ttsDebugTextPreview(latest, 220),
             }
             : undefined;
-        const isFirstClip = state.queuedSpeakablePrefix.length === 0;
         enqueueSpeech({
             text: unsent,
-            append: !isFirstClip,
-            segment: isFirstClip ? "full" : "remainder",
+            append: !isFirstClip || !state.replacePlaybackOnFirstClip,
+            segment,
             cacheKey,
             debugUtteranceContext: dbgUtterance,
+            telemetry: state.telemetry,
         });
         state.queuedSpeakablePrefix = latest;
     }, [enqueueSpeech, makeElevenCacheKey]);
-    const queueAssistantSpeech = useCallback((messageId, text, isFinal) => {
+    const queueAssistantSpeech = useCallback((messageId, text, isFinal, queueOptions) => {
         if (!messageId)
             return;
         const speakable = toSpeakableText(text);
@@ -1165,6 +1655,14 @@ export function useVoiceChat(options) {
                 queuedSpeakablePrefix: "",
                 latestSpeakable: "",
                 finalQueued: false,
+                replacePlaybackOnFirstClip: queueOptions?.replace !== false,
+                telemetry: queueOptions?.telemetry,
+            };
+        }
+        else if (queueOptions?.telemetry) {
+            current.telemetry = {
+                ...current.telemetry,
+                ...queueOptions.telemetry,
             };
         }
         const state = assistantSpeechRef.current;
@@ -1180,7 +1678,9 @@ export function useVoiceChat(options) {
                 return;
             clearAssistantTtsDebounce();
             const elConfig = voiceConfigRef.current?.elevenlabs;
-            const cacheKey = voiceConfigRef.current?.provider === "elevenlabs" && elConfig
+            const cacheKey = voiceConfigRef.current?.provider === "elevenlabs" &&
+                elConfig &&
+                shouldCacheGeneratedSpeech(speakable, "full")
                 ? makeElevenCacheKey(speakable, elConfig)
                 : undefined;
             const dbgUtterance = isTtsDebugEnabled()
@@ -1196,6 +1696,7 @@ export function useVoiceChat(options) {
                 segment: "full",
                 cacheKey,
                 debugUtteranceContext: dbgUtterance,
+                telemetry: state.telemetry,
             });
             state.queuedSpeakablePrefix = speakable;
             state.finalQueued = true;
@@ -1210,8 +1711,12 @@ export function useVoiceChat(options) {
             state.finalQueued = true;
             return;
         }
-        const unsent = remainderAfter(speakable, state.queuedSpeakablePrefix);
-        if (!unsent) {
+        const boundaryPrefix = queueableSpeechPrefix(speakable, isFinal);
+        const boundaryUnsent = boundaryPrefix
+            ? remainderAfter(boundaryPrefix, state.queuedSpeakablePrefix)
+            : "";
+        const rawUnsent = remainderAfter(speakable, state.queuedSpeakablePrefix);
+        if (!rawUnsent) {
             if (isFinal) {
                 clearAssistantTtsDebounce();
                 state.finalQueued = true;
@@ -1219,13 +1724,25 @@ export function useVoiceChat(options) {
             return;
         }
         const isFirstClip = state.queuedSpeakablePrefix.length === 0;
-        const flushNow = isFinal ||
-            (isFirstClip && unsent.length >= ASSISTANT_TTS_FIRST_FLUSH_CHARS) ||
-            (!isFirstClip && unsent.length >= ASSISTANT_TTS_MIN_CHUNK_CHARS);
+        const thresholdFlush = isFinal ||
+            (isFirstClip && rawUnsent.length >= ASSISTANT_TTS_FIRST_FLUSH_CHARS) ||
+            (!isFirstClip && rawUnsent.length >= ASSISTANT_TTS_MIN_CHUNK_CHARS);
+        const targetPrefix = boundaryUnsent
+            ? boundaryPrefix
+            : thresholdFlush
+                ? speakable
+                : "";
+        const unsent = targetPrefix
+            ? remainderAfter(targetPrefix, state.queuedSpeakablePrefix)
+            : "";
+        const flushNow = Boolean(unsent);
         if (flushNow) {
             clearAssistantTtsDebounce();
             const elConfig = voiceConfigRef.current?.elevenlabs;
-            const cacheKey = voiceConfigRef.current?.provider === "elevenlabs" && elConfig
+            const segment = isFirstClip ? "full" : "remainder";
+            const cacheKey = voiceConfigRef.current?.provider === "elevenlabs" &&
+                elConfig &&
+                shouldCacheGeneratedSpeech(unsent, segment)
                 ? makeElevenCacheKey(unsent, elConfig)
                 : undefined;
             const dbgUtterance = isTtsDebugEnabled()
@@ -1236,12 +1753,13 @@ export function useVoiceChat(options) {
                 : undefined;
             enqueueSpeech({
                 text: unsent,
-                append: !isFirstClip,
-                segment: isFirstClip ? "full" : "remainder",
+                append: !isFirstClip || !state.replacePlaybackOnFirstClip,
+                segment,
                 cacheKey,
                 debugUtteranceContext: dbgUtterance,
+                telemetry: state.telemetry,
             });
-            state.queuedSpeakablePrefix = speakable;
+            state.queuedSpeakablePrefix = targetPrefix;
             if (isFinal)
                 state.finalQueued = true;
             return;
@@ -1267,14 +1785,9 @@ export function useVoiceChat(options) {
         const handleUserGesture = () => {
             window.removeEventListener("pointerdown", handleUserGesture, true);
             window.removeEventListener("keydown", handleUserGesture, true);
-            // Warm AudioContext for ElevenLabs
-            if (!sharedAudioCtx) {
-                sharedAudioCtx = new AudioContext();
-            }
-            void sharedAudioCtx.resume().catch(() => { });
-            // Signal that audio is now unlocked so callers can retry speech
-            // that was silently blocked by browser autoplay policy.
-            setVoiceUnlockedGeneration((g) => g + 1);
+            // Warm/resume the context and clear any stale unlock hint. Shared with the
+            // explicit `unlockAudio` action so both paths behave identically.
+            unlockAudio();
         };
         window.addEventListener("pointerdown", handleUserGesture, true);
         window.addEventListener("keydown", handleUserGesture, true);
@@ -1282,13 +1795,17 @@ export function useVoiceChat(options) {
             window.removeEventListener("pointerdown", handleUserGesture, true);
             window.removeEventListener("keydown", handleUserGesture, true);
         };
-    }, []);
+    }, [unlockAudio]);
     // ── Cleanup on unmount ────────────────────────────────────────────
     useEffect(() => {
         return () => {
             void stopListening();
             void removeTalkModeListeners();
             stopSpeaking();
+            if (micReconnectTimerRef.current !== null) {
+                clearTimeout(micReconnectTimerRef.current);
+                micReconnectTimerRef.current = null;
+            }
         };
     }, [removeTalkModeListeners, stopListening, stopSpeaking]);
     return {
@@ -1306,6 +1823,9 @@ export function useVoiceChat(options) {
         queueAssistantSpeech,
         stopSpeaking,
         voiceUnlockedGeneration,
+        needsAudioUnlock,
+        micReconnected,
+        unlockAudio,
         assistantTtsQuality,
     };
 }