@elizaos/ui 2.0.0-alpha.85 → 2.0.0-beta.2

package/voice/voice-chat-playback.js

@@ -0,0 +1,209 @@
+/**
+ * Playback / TTS logic for voice chat — text processing, sentence splitting,
+ * speech text extraction, and mouth animation helpers.
+ */
+import { sanitizeSpeechText } from "@elizaos/shared";
+import { MAX_SPOKEN_CHARS, MOUTH_OPEN_STEP } from "./voice-chat-types.js";
+// ── Text processing helpers ───────────────────────────────────────────
+export function collapseWhitespace(input) {
+    return input.replace(/\s+/g, " ").trim();
+}
+export function normalizeCacheText(input) {
+    return collapseWhitespace(input.normalize("NFKC")).toLowerCase();
+}
+export function capSpeechLength(input) {
+    if (input.length <= MAX_SPOKEN_CHARS)
+        return input;
+    const clipped = input.slice(0, MAX_SPOKEN_CHARS);
+    const splitAt = clipped.lastIndexOf(" ");
+    const body = splitAt > 120 ? clipped.slice(0, splitAt) : clipped;
+    return `${body.trim()}...`;
+}
+// ── Hidden model block stripping ──────────────────────────────────────
+/**
+ * Hidden model block tags whose content should never be spoken. During
+ * streaming the closing tag may not have arrived yet, so we strip from
+ * the opening tag to end-of-string.
+ *
+ * The upstream `sanitizeSpeechText` only strips *closed* `<think>` blocks,
+ * so an in-progress `<think>reasoning so far` leaks "reasoning so far"
+ * into the voice output.  We handle it here before sanitization.
+ */
+const HIDDEN_VOICE_BLOCK_RE = /<(think|thought|analysis|reasoning|tool_calls?|tools?)\b[^>]*>[\s\S]*?(?:<\/\1>|$)/gi;
+function parseJsonObject(input) {
+    try {
+        const parsed = JSON.parse(input.trim());
+        return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+            ? parsed
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
+function extractStructuredVoiceText(input) {
+    const parsed = parseJsonObject(input);
+    if (!parsed)
+        return null;
+    if (typeof parsed.text === "string") {
+        return parsed.text;
+    }
+    if ("actions" in parsed || "params" in parsed || "providers" in parsed) {
+        return "";
+    }
+    return null;
+}
+export function extractVoiceText(input) {
+    let text = extractStructuredVoiceText(input) ?? input;
+    text = text.replace(HIDDEN_VOICE_BLOCK_RE, " ");
+    text = text.replace(/<\/?[a-zA-Z][^>]*$|<\/?$/s, "");
+    return text;
+}
+export function toSpeakableText(input) {
+    const extracted = extractVoiceText(input);
+    if (!extracted)
+        return "";
+    const normalized = sanitizeSpeechText(extracted);
+    if (!normalized)
+        return "";
+    return capSpeechLength(normalized);
+}
+// ── Sentence splitting ────────────────────────────────────────────────
+/** Common abbreviations that end with a period but are not sentence endings. */
+const ABBREV_RE = /(?:Mr|Mrs|Ms|Dr|Jr|Sr|St|vs|etc|approx|Prof|Rev|Gen|Sgt|Lt|Col|Maj|Capt|Corp|Pvt|Ave|Blvd|dept|est|govt|assn)$/;
+/**
+ * Replace URLs with placeholders so their internal dots are not treated as
+ * sentence boundaries.  Returns the cleaned string and a restore function.
+ */
+export function shelterUrls(input) {
+    const urls = [];
+    const text = input.replace(/https?:\/\/\S+/g, (m) => {
+        urls.push(m);
+        return `__URL${urls.length - 1}__`;
+    });
+    return {
+        text,
+        restore: (s) => s.replace(/__URL(\d+)__/g, (_, i) => urls[Number(i)] ?? _),
+    };
+}
+/**
+ * Test whether a period match at `index` inside `value` is a real sentence
+ * boundary (not an abbreviation or decimal).
+ */
+export function isRealSentenceEnd(value, matchIndex) {
+    const previousChar = matchIndex > 0 ? value[matchIndex - 1] : undefined;
+    const nextChar = matchIndex + 1 < value.length ? value[matchIndex + 1] : undefined;
+    if (previousChar !== undefined && /\d/.test(previousChar)) {
+        if (nextChar !== undefined && /\d/.test(nextChar)) {
+            return false;
+        }
+    }
+    const before = value.slice(0, matchIndex);
+    if (ABBREV_RE.test(before))
+        return false;
+    return true;
+}
+export function splitFirstSentence(text) {
+    const value = collapseWhitespace(text);
+    if (!value)
+        return { complete: false, firstSentence: "", remainder: "" };
+    const { text: sheltered, restore } = shelterUrls(value);
+    const boundary = /([.!?]+(?:["')\]]+)?)(?:\s|$)/g;
+    let match = null;
+    while (true) {
+        match = boundary.exec(sheltered);
+        if (!match || typeof match.index !== "number")
+            break;
+        const punctChar = match[1]?.[0];
+        if (punctChar === ".") {
+            if (match[1]?.length >= 3)
+                continue;
+            if (!isRealSentenceEnd(sheltered, match.index))
+                continue;
+        }
+        const endIndex = match.index + match[0].length;
+        const firstSentence = restore(sheltered.slice(0, endIndex).trim());
+        const remainder = restore(sheltered.slice(endIndex).trim());
+        if (firstSentence.length > 0) {
+            return { complete: true, firstSentence, remainder };
+        }
+    }
+    if (value.length >= 180) {
+        const window = value.slice(0, 180);
+        const splitAt = window.lastIndexOf(" ");
+        if (splitAt > 100) {
+            return {
+                complete: true,
+                firstSentence: window.slice(0, splitAt).trim(),
+                remainder: value.slice(splitAt).trim(),
+            };
+        }
+    }
+    return { complete: false, firstSentence: value, remainder: "" };
+}
+export function remainderAfter(fullText, firstSentence) {
+    const full = collapseWhitespace(fullText);
+    const first = collapseWhitespace(firstSentence);
+    if (!full || !first)
+        return full;
+    if (full.startsWith(first))
+        return full.slice(first.length).trim();
+    const lowerFull = full.toLowerCase();
+    const lowerFirst = first.toLowerCase();
+    if (lowerFull.startsWith(lowerFirst)) {
+        return full.slice(first.length).trim();
+    }
+    const idx = lowerFull.indexOf(lowerFirst);
+    if (idx >= 0) {
+        return full.slice(idx + first.length).trim();
+    }
+    return "";
+}
+export function queueableSpeechPrefix(text, isFinal) {
+    const value = collapseWhitespace(text);
+    if (!value)
+        return "";
+    if (isFinal)
+        return value;
+    const { text: sheltered, restore } = shelterUrls(value);
+    let lastSentenceEnd = 0;
+    const boundary = /([.!?]+(?:["')\]]+)?)(?:\s|$)/g;
+    let match = null;
+    while (true) {
+        match = boundary.exec(sheltered);
+        if (!match || typeof match.index !== "number")
+            break;
+        const punctChar = match[1]?.[0];
+        if (punctChar === ".") {
+            if (match[1]?.length >= 3)
+                continue;
+            if (!isRealSentenceEnd(sheltered, match.index))
+                continue;
+        }
+        lastSentenceEnd = match.index + match[0].length;
+    }
+    if (lastSentenceEnd > 0) {
+        return restore(sheltered.slice(0, lastSentenceEnd).trim());
+    }
+    if (value.length >= 180) {
+        const window = value.slice(0, 180);
+        const splitAt = window.lastIndexOf(" ");
+        if (splitAt > 100) {
+            return window.slice(0, splitAt).trim();
+        }
+    }
+    return "";
+}
+// ── Mouth animation ───────────────────────────────────────────────────
+export function normalizeMouthOpen(value) {
+    const clamped = Math.max(0, Math.min(1, value));
+    const stepped = Math.round(clamped / MOUTH_OPEN_STEP) * MOUTH_OPEN_STEP;
+    return stepped < MOUTH_OPEN_STEP ? 0 : Math.min(1, stepped);
+}
+export function nextIdleMouthOpen(currentValue) {
+    const current = normalizeMouthOpen(currentValue);
+    if (current <= MOUTH_OPEN_STEP) {
+        return 0;
+    }
+    return Math.max(0, Math.min(current * 0.85, current - MOUTH_OPEN_STEP));
+}

package/voice/voice-chat-recording.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Recording / STT logic for voice chat — transcript merging and normalization.
+ */
+export declare function normalizeTranscriptWord(word: string): string;
+export declare function mergeTranscriptWindows(existing: string, incoming: string): string;
+//# sourceMappingURL=voice-chat-recording.d.ts.map

package/voice/voice-chat-recording.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-chat-recording.d.ts","sourceRoot":"","sources":["../../../../../src/voice/voice-chat-recording.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAK5D;AAED,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,MAAM,CAuCR"}

package/voice/voice-chat-recording.js

@@ -0,0 +1,46 @@
+/**
+ * Recording / STT logic for voice chat — transcript merging and normalization.
+ */
+import { mergeStreamingText } from "../utils/streaming-text.js";
+import { collapseWhitespace } from "./voice-chat-playback.js";
+// ── Transcript merging ────────────────────────────────────────────────
+export function normalizeTranscriptWord(word) {
+    return word
+        .normalize("NFKC")
+        .toLowerCase()
+        .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
+}
+export function mergeTranscriptWindows(existing, incoming) {
+    const left = collapseWhitespace(existing);
+    const right = collapseWhitespace(incoming);
+    if (!left)
+        return right;
+    if (!right)
+        return left;
+    const exactMerged = mergeStreamingText(left, right);
+    if (exactMerged === right ||
+        exactMerged === left ||
+        exactMerged === `${left}${right}`) {
+        const leftWords = left.split(" ");
+        const rightWords = right.split(" ");
+        const maxOverlap = Math.min(leftWords.length, rightWords.length);
+        for (let overlap = maxOverlap; overlap > 0; overlap -= 1) {
+            let matches = true;
+            for (let index = 0; index < overlap; index += 1) {
+                const leftWord = normalizeTranscriptWord(leftWords[leftWords.length - overlap + index] ?? "");
+                const rightWord = normalizeTranscriptWord(rightWords[index] ?? "");
+                if (!leftWord || !rightWord || leftWord !== rightWord) {
+                    matches = false;
+                    break;
+                }
+            }
+            if (!matches)
+                continue;
+            if (overlap === rightWords.length) {
+                return left;
+            }
+            return [...leftWords, ...rightWords.slice(overlap)].join(" ");
+        }
+    }
+    return exactMerged;
+}

package/voice/voice-chat-types.d.ts

@@ -0,0 +1,193 @@
+/**
+ * Types, constants, and config interfaces for the voice chat system.
+ */
+import type { VoiceConfig, VoiceMode } from "../api/client";
+export interface SpeechRecognitionInstance extends EventTarget {
+    continuous: boolean;
+    interimResults: boolean;
+    lang: string;
+    onstart: (() => void) | null;
+    onend: (() => void) | null;
+    onerror: ((event: {
+        error: string;
+    }) => void) | null;
+    onresult: ((event: SpeechRecognitionResultEvent) => void) | null;
+    start(): void;
+    stop(): void;
+    abort(): void;
+}
+export interface SpeechRecognitionResultEvent {
+    results: SpeechRecognitionResultList;
+    resultIndex: number;
+}
+export interface SpeechRecognitionResultList {
+    length: number;
+    [index: number]: {
+        isFinal: boolean;
+        0: {
+            transcript: string;
+            confidence: number;
+        };
+    };
+}
+export type SpeechRecognitionCtor = new () => SpeechRecognitionInstance;
+export interface WindowWithSpeechRecognition extends Window {
+    SpeechRecognition?: SpeechRecognitionCtor;
+    webkitSpeechRecognition?: SpeechRecognitionCtor;
+}
+/** Access browser SpeechRecognition APIs which may live under a vendor prefix. */
+export declare function getSpeechRecognitionCtor(): SpeechRecognitionCtor | undefined;
+export type SpeechSegmentKind = "full" | "first-sentence" | "remainder";
+export type SpeechProviderKind = "elevenlabs" | "browser";
+export type VoiceCaptureMode = "idle" | "compose" | "push-to-talk";
+export interface VoicePlaybackStartEvent {
+    text: string;
+    segment: SpeechSegmentKind;
+    provider: SpeechProviderKind;
+    cached: boolean;
+    startedAtMs: number;
+}
+export interface VoiceTranscriptPreviewEvent {
+    mode: Exclude<VoiceCaptureMode, "idle">;
+    isFinal: boolean;
+}
+export interface VoiceChatOptions {
+    /** Called when a final transcript is ready to send */
+    onTranscript: (text: string) => void;
+    /** Called whenever the live transcript buffer changes */
+    onTranscriptPreview?: (text: string, event: VoiceTranscriptPreviewEvent) => void;
+    /** Called when playback of a speech segment starts */
+    onPlaybackStart?: (event: VoicePlaybackStartEvent) => void;
+    /** True when Eliza Cloud-managed voice access is available */
+    cloudConnected?: boolean;
+    /** Whether user speech should immediately interrupt assistant playback */
+    interruptOnSpeech?: boolean;
+    /** Language for speech recognition (default: "en-US") */
+    lang?: string;
+    /** Saved voice configuration — switches TTS provider when set */
+    voiceConfig?: VoiceConfig | null;
+}
+export interface VoiceChatState {
+    /** Whether voice input is currently active */
+    isListening: boolean;
+    /** Current mic capture mode */
+    captureMode: VoiceCaptureMode;
+    /** Whether the agent is currently speaking */
+    isSpeaking: boolean;
+    /** Current mouth openness (0-1) for lip sync */
+    mouthOpen: number;
+    /** Current interim transcript being recognized */
+    interimTranscript: string;
+    /** Whether Web Speech API is supported */
+    supported: boolean;
+    /** True when using real audio analysis (ElevenLabs) for mouth */
+    usingAudioAnalysis: boolean;
+    /** Toggle voice listening on/off */
+    toggleListening: () => void;
+    /** Begin voice capture in compose or push-to-talk mode */
+    startListening: (mode?: Exclude<VoiceCaptureMode, "idle">) => Promise<void>;
+    /** End voice capture and optionally submit the transcript */
+    stopListening: (options?: {
+        submit?: boolean;
+    }) => Promise<void>;
+    /** Speak text aloud with mouth animation */
+    speak: (text: string, options?: {
+        append?: boolean;
+    }) => void;
+    /** Progressively speak an assistant message while it streams */
+    queueAssistantSpeech: (messageId: string, text: string, isFinal: boolean) => void;
+    /** Stop any current speech */
+    stopSpeaking: () => void;
+    /** Increments when AudioContext is unlocked by a user gesture, allowing callers to retry speech that was silently blocked by autoplay policy. */
+    voiceUnlockedGeneration: number;
+    /**
+     * Assistant reply TTS: `enhanced` = ElevenLabs path (own key, cloud proxy, or direct);
+     * `standard` = browser / Edge voices or non-ElevenLabs provider.
+     */
+    assistantTtsQuality: "enhanced" | "standard";
+}
+export interface SpeakTask {
+    text: string;
+    append: boolean;
+    segment: SpeechSegmentKind;
+    cacheKey?: string;
+    /** App-only: sent as `x-elizaos-tts-*` headers on `/api/tts/*` when debug is on (never forwarded to Eliza Cloud). */
+    debugUtteranceContext?: {
+        messageId: string;
+        fullAssistTextPreview: string;
+    };
+}
+export interface AssistantSpeechState {
+    messageId: string;
+    /** Speakable text already submitted to the playback queue (prefix of current stream). */
+    queuedSpeakablePrefix: string;
+    /** Latest speakable from the stream (debounce flush reads this). */
+    latestSpeakable: string;
+    finalQueued: boolean;
+}
+export declare const DEFAULT_ELEVEN_MODEL = "eleven_flash_v2_5";
+export declare const DEFAULT_ELEVEN_VOICE = "EXAVITQu4vr4xnSDxMaL";
+export declare const MAX_SPOKEN_CHARS = 360;
+export declare const MAX_CACHED_SEGMENTS = 128;
+/** First assistant clip: start synthesis after this much speakable text (avoids one-word TTS). */
+export declare const ASSISTANT_TTS_FIRST_FLUSH_CHARS = 24;
+/** Later clips: batch for better prosody (avoid token-thin slices). */
+export declare const ASSISTANT_TTS_MIN_CHUNK_CHARS = 88;
+/** Merge rapid stream deltas into one request after a short pause. */
+export declare const ASSISTANT_TTS_DEBOUNCE_MS = 170;
+/**
+ * Temporary safety switch:
+ * only speak assistant replies once the final text has arrived.
+ *
+ * This avoids garbled overlap when cloud text streaming and speech playback
+ * race each other on partial chunks.
+ */
+export declare const ASSISTANT_TTS_FINAL_ONLY = true;
+export declare const TALKMODE_STOP_SETTLE_MS = 120;
+export declare const REDACTED_SECRET = "[REDACTED]";
+export declare const MOUTH_OPEN_STEP = 0.02;
+export declare const globalAudioCache: Map<string, Uint8Array<ArrayBufferLike>>;
+export declare function resolveVoiceMode(mode: VoiceMode | undefined, _cloudConnected: boolean, _apiKey?: string | null): VoiceMode;
+export declare function resolveVoiceProxyEndpoint(mode: VoiceMode): string;
+/** For ELIZA_TTS_DEBUG: shows whether cloud TTS hits the API or the wrong (page) origin. */
+export declare function describeTtsCloudFetchTargetForDebug(): string;
+export declare function cloneVoiceConfig(config: (VoiceConfig & {
+    provider?: VoiceConfig["provider"] | "openai";
+    openai?: {
+        apiKey?: string;
+        voice?: string;
+        model?: string;
+    };
+}) | null | undefined): (VoiceConfig & {
+    provider?: VoiceConfig["provider"] | "openai";
+    openai?: {
+        apiKey?: string;
+        voice?: string;
+        model?: string;
+    };
+}) | null;
+export declare function resolveEffectiveVoiceConfig(config: (VoiceConfig & {
+    provider?: VoiceConfig["provider"] | "openai";
+    openai?: {
+        apiKey?: string;
+        voice?: string;
+        model?: string;
+    };
+}) | null | undefined, options?: {
+    cloudConnected?: boolean;
+}): (VoiceConfig & {
+    provider?: VoiceConfig["provider"] | "openai";
+    openai?: {
+        apiKey?: string;
+        voice?: string;
+        model?: string;
+    };
+}) | null;
+export declare function isAbortError(error: unknown): boolean;
+/** ELIZA_TTS_DEBUG fields for OS/browser SpeechSynthesis (often Microsoft Edge on Windows). */
+export declare function webSpeechVoiceDebugFields(voice: SpeechSynthesisVoice | undefined): Record<string, string | boolean | undefined>;
+export declare function normalizeSpeechLocale(input: string | undefined): string;
+export declare function localePrefix(locale: string): string;
+export declare function matchesVoiceLocale(voice: SpeechSynthesisVoice, targetLocale: string): boolean;
+export declare function toArrayBuffer(bytes: Uint8Array): ArrayBuffer;
+//# sourceMappingURL=voice-chat-types.d.ts.map

package/voice/voice-chat-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-chat-types.d.ts","sourceRoot":"","sources":["../../../../../src/voice/voice-chat-types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAM5D,MAAM,WAAW,yBAA0B,SAAQ,WAAW;IAC5D,UAAU,EAAE,OAAO,CAAC;IACpB,cAAc,EAAE,OAAO,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7B,KAAK,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IACrD,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IACjE,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,IAAI,CAAC;IACb,KAAK,IAAI,IAAI,CAAC;CACf;AAED,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,2BAA2B,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,KAAK,EAAE,MAAM,GAAG;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,CAAC,EAAE;YAAE,UAAU,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAA;SAAE,CAAC;KAC/C,CAAC;CACH;AAED,MAAM,MAAM,qBAAqB,GAAG,UAAU,yBAAyB,CAAC;AAExE,MAAM,WAAW,2BAA4B,SAAQ,MAAM;IACzD,iBAAiB,CAAC,EAAE,qBAAqB,CAAC;IAC1C,uBAAuB,CAAC,EAAE,qBAAqB,CAAC;CACjD;AAQD,kFAAkF;AAClF,wBAAgB,wBAAwB,IAAI,qBAAqB,GAAG,SAAS,CAa5E;AAID,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,gBAAgB,GAAG,WAAW,CAAC;AACxE,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG,SAAS,CAAC;AAC1D,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,SAAS,GAAG,cAAc,CAAC;AAEnE,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,iBAAiB,CAAC;IAC3B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,sDAAsD;IACtD,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACrC,yDAAyD;IACzD,mBAAmB,CAAC,EAAE,CACpB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,2BAA2B,KAC/B,IAAI,CAAC;IACV,sDAAsD;IACtD,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,uBAAuB,KAAK,IAAI,CAAC;IAC3D,8DAA8D;IAC9D,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,yDAAyD;IACzD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,cAAc;IAC7B,8CAA8C;IAC9C,WAAW,EAAE,OAAO,CAAC;IACrB,+BAA+B;IAC/B,WAAW,EAAE,gBAAgB,CAAC;IAC9B,8CAA8C;IAC9C,UAAU,EAAE,OAAO,CAAC;IACpB,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,SAAS,EAAE,OAAO,CAAC;IACnB,iEAAiE;IACjE,kBAAkB,EAAE,OAAO,CAAC;IAC5B,oCAAoC;IACpC,eAAe,EAAE,MAAM,IAAI,CAAC;IAC5B,0DAA0D;IAC1D,cAAc,EAAE,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,gBAAgB,EAAE,MAAM,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,6DAA6D;IAC7D,aAAa,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACjE,4CAA4C;IAC5C,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAC;IAC9D,gEAAgE;IAChE,oBAAoB,EAAE,CACpB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,KACb,IAAI,CAAC;IACV,8BAA8B;IAC9B,YAAY,EAAE,MAAM,IAAI,CAAC;IACzB,iJAAiJ;IACjJ,uBAAuB,EAAE,MAAM,CAAC;IAChC;;;OAGG;IACH,mBAAmB,EAAE,UAAU,GAAG,UAAU,CAAC;CAC9C;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qHAAqH;IACrH,qBAAqB,CAAC,EAAE;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC;CACH;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,yFAAyF;IACzF,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,OAAO,CAAC;CACtB;AAID,eAAO,MAAM,oBAAoB,sBAAsB,CAAC;AACxD,eAAO,MAAM,oBAAoB,yBAAyB,CAAC;AAC3D,eAAO,MAAM,gBAAgB,MAAM,CAAC;AACpC,eAAO,MAAM,mBAAmB,MAAM,CAAC;AACvC,kGAAkG;AAClG,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAClD,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,KAAK,CAAC;AAChD,sEAAsE;AACtE,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAC7C;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAC7C,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAC3C,eAAO,MAAM,eAAe,eAAe,CAAC;AAC5C,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,eAAO,MAAM,gBAAgB,0CAAgC,CAAC;AAI9D,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,SAAS,GAAG,SAAS,EAC3B,eAAe,EAAE,OAAO,EACxB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,GACtB,SAAS,CAGX;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAIjE;AAED,4FAA4F;AAC5F,wBAAgB,mCAAmC,IAAI,MAAM,CAa5D;AAQD,wBAAgB,gBAAgB,CAC9B,MAAM,EACF,CAAC,WAAW,GAAG;IACb,QAAQ,CAAC,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;IAC9C,MAAM,CAAC,EAAE;QACP,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC,GACF,IAAI,GACJ,SAAS,GAEX,CAAC,WAAW,GAAG;IACb,QAAQ,CAAC,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;IAC9C,MAAM,CAAC,EAAE;QACP,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC,GACF,IAAI,CAQP;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EACF,CAAC,WAAW,GAAG;IACb,QAAQ,CAAC,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;IAC9C,MAAM,CAAC,EAAE;QACP,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC,GACF,IAAI,GACJ,SAAS,EACb,OAAO,CAAC,EAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAEpC,CAAC,WAAW,GAAG;IACb,QAAQ,CAAC,EAAE,WAAW,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;IAC9C,MAAM,CAAC,EAAE;QACP,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,CAAC,GACF,IAAI,CAsEP;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAIpD;AAED,+FAA+F;AAC/F,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,oBAAoB,GAAG,SAAS,GACtC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC,CAiC9C;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAGvE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,oBAAoB,EAC3B,YAAY,EAAE,MAAM,GACnB,OAAO,CAMT;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,CAI5D"}

package/voice/voice-chat-types.js

@@ -0,0 +1,194 @@
+/**
+ * Types, constants, and config interfaces for the voice chat system.
+ */
+import { resolveApiUrl } from "../utils/index.js";
+import { ttsDebug } from "../utils/tts-debug.js";
+function isSpeechRecognitionCtor(value) {
+    return typeof value === "function" && "prototype" in value;
+}
+/** Access browser SpeechRecognition APIs which may live under a vendor prefix. */
+export function getSpeechRecognitionCtor() {
+    const speechRecognition = Reflect.get(window, "SpeechRecognition");
+    if (isSpeechRecognitionCtor(speechRecognition)) {
+        return speechRecognition;
+    }
+    const webkitSpeechRecognition = Reflect.get(window, "webkitSpeechRecognition");
+    return isSpeechRecognitionCtor(webkitSpeechRecognition)
+        ? webkitSpeechRecognition
+        : undefined;
+}
+// ── Constants ─────────────────────────────────────────────────────────
+export const DEFAULT_ELEVEN_MODEL = "eleven_flash_v2_5";
+export const DEFAULT_ELEVEN_VOICE = "EXAVITQu4vr4xnSDxMaL";
+export const MAX_SPOKEN_CHARS = 360;
+export const MAX_CACHED_SEGMENTS = 128;
+/** First assistant clip: start synthesis after this much speakable text (avoids one-word TTS). */
+export const ASSISTANT_TTS_FIRST_FLUSH_CHARS = 24;
+/** Later clips: batch for better prosody (avoid token-thin slices). */
+export const ASSISTANT_TTS_MIN_CHUNK_CHARS = 88;
+/** Merge rapid stream deltas into one request after a short pause. */
+export const ASSISTANT_TTS_DEBOUNCE_MS = 170;
+/**
+ * Temporary safety switch:
+ * only speak assistant replies once the final text has arrived.
+ *
+ * This avoids garbled overlap when cloud text streaming and speech playback
+ * race each other on partial chunks.
+ */
+export const ASSISTANT_TTS_FINAL_ONLY = true;
+export const TALKMODE_STOP_SETTLE_MS = 120;
+export const REDACTED_SECRET = "[REDACTED]";
+export const MOUTH_OPEN_STEP = 0.02;
+export const globalAudioCache = new Map();
+// ── Voice config helpers ──────────────────────────────────────────────
+export function resolveVoiceMode(mode, _cloudConnected, _apiKey) {
+    if (mode)
+        return mode;
+    return "own-key";
+}
+export function resolveVoiceProxyEndpoint(mode) {
+    return resolveApiUrl(mode === "cloud" ? "/api/tts/cloud" : "/api/tts/elevenlabs");
+}
+/** For ELIZA_TTS_DEBUG: shows whether cloud TTS hits the API or the wrong (page) origin. */
+export function describeTtsCloudFetchTargetForDebug() {
+    const target = resolveApiUrl("/api/tts/cloud");
+    if (/^https?:\/\//i.test(target)) {
+        try {
+            return `${new URL(target).origin} (absolute)`;
+        }
+        catch {
+            return target.slice(0, 120);
+        }
+    }
+    const origin = typeof window !== "undefined" ? window.location.origin : "(no-window)";
+    const path = target.startsWith("/") ? target : `/${target}`;
+    return `${origin}${path} — relative URL (TTS fetch goes to the UI host, not the app API). Set __ELIZAOS_API_BASE__ / session elizaos_api_base / boot apiBase to http://127.0.0.1:<apiPort>`;
+}
+function isRedactedSecret(value) {
+    return (typeof value === "string" && value.trim().toUpperCase() === REDACTED_SECRET);
+}
+export function cloneVoiceConfig(config) {
+    if (!config)
+        return null;
+    return {
+        ...config,
+        elevenlabs: config.elevenlabs ? { ...config.elevenlabs } : undefined,
+        edge: config.edge ? { ...config.edge } : undefined,
+        openai: config.openai ? { ...config.openai } : undefined,
+    };
+}
+export function resolveEffectiveVoiceConfig(config, options) {
+    const cloudConnected = options?.cloudConnected === true;
+    const base = cloneVoiceConfig(config) ?? {};
+    const rawProvider = base.provider;
+    const hasLegacyOpenAiProvider = rawProvider === "openai";
+    let provider = (hasLegacyOpenAiProvider ? undefined : rawProvider) ??
+        (base.elevenlabs ? "elevenlabs" : base.edge ? "edge" : undefined) ??
+        (cloudConnected ? "elevenlabs" : undefined);
+    if (cloudConnected &&
+        (provider === "edge" ||
+            hasLegacyOpenAiProvider ||
+            provider === "robot-voice")) {
+        ttsDebug("voiceConfig:upgrade_provider_for_cloud", {
+            fromProvider: hasLegacyOpenAiProvider ? "openai" : provider,
+        });
+        provider = "elevenlabs";
+    }
+    if (!provider)
+        return null;
+    if (provider !== "elevenlabs") {
+        return { ...base, provider };
+    }
+    const currentElevenLabs = base.elevenlabs ?? {};
+    const mode = resolveVoiceMode(base.mode, cloudConnected, currentElevenLabs.apiKey);
+    const elevenlabs = {
+        ...currentElevenLabs,
+        voiceId: currentElevenLabs.voiceId ?? DEFAULT_ELEVEN_VOICE,
+        modelId: currentElevenLabs.modelId ?? DEFAULT_ELEVEN_MODEL,
+        stability: typeof currentElevenLabs.stability === "number"
+            ? currentElevenLabs.stability
+            : 0.5,
+        similarityBoost: typeof currentElevenLabs.similarityBoost === "number"
+            ? currentElevenLabs.similarityBoost
+            : 0.75,
+        speed: typeof currentElevenLabs.speed === "number"
+            ? currentElevenLabs.speed
+            : 1.0,
+    };
+    const apiKey = typeof currentElevenLabs.apiKey === "string"
+        ? currentElevenLabs.apiKey.trim()
+        : "";
+    if (mode === "own-key" && apiKey && !isRedactedSecret(apiKey)) {
+        elevenlabs.apiKey = currentElevenLabs.apiKey;
+    }
+    else {
+        delete elevenlabs.apiKey;
+    }
+    return {
+        ...base,
+        provider,
+        mode,
+        elevenlabs,
+    };
+}
+export function isAbortError(error) {
+    if (error instanceof DOMException && error.name === "AbortError")
+        return true;
+    if (error instanceof Error && error.name === "AbortError")
+        return true;
+    return false;
+}
+/** ELIZA_TTS_DEBUG fields for OS/browser SpeechSynthesis (often Microsoft Edge on Windows). */
+export function webSpeechVoiceDebugFields(voice) {
+    if (!voice) {
+        return {
+            voiceName: "(engine default)",
+            voiceURI: "(none)",
+            engineGuess: "unknown",
+        };
+    }
+    const blob = `${voice.voiceURI} ${voice.name}`.toLowerCase();
+    let engineGuess = "unknown";
+    if (blob.includes("microsoft") ||
+        blob.includes("msedge") ||
+        blob.includes("edge-tts")) {
+        engineGuess = "microsoft-edge-family";
+    }
+    else if (blob.includes("com.apple")) {
+        engineGuess = "apple-webkit";
+    }
+    else if (blob.includes("google")) {
+        engineGuess = "google";
+    }
+    const extended = voice;
+    return {
+        voiceName: voice.name,
+        voiceURI: voice.voiceURI,
+        voiceLang: voice.lang,
+        voiceDefault: voice.default,
+        voiceLocalService: typeof extended.localService === "boolean"
+            ? extended.localService
+            : undefined,
+        engineGuess,
+    };
+}
+export function normalizeSpeechLocale(input) {
+    const trimmed = input?.trim();
+    return trimmed || "en-US";
+}
+export function localePrefix(locale) {
+    return locale.toLowerCase().split("-")[0] || "en";
+}
+export function matchesVoiceLocale(voice, targetLocale) {
+    const target = targetLocale.toLowerCase();
+    const voiceLang = voice.lang.toLowerCase();
+    if (voiceLang === target)
+        return true;
+    const base = localePrefix(targetLocale);
+    return voiceLang.startsWith(`${base}-`) || voiceLang === base;
+}
+export function toArrayBuffer(bytes) {
+    const out = new Uint8Array(bytes.byteLength);
+    out.set(bytes);
+    return out.buffer;
+}