@omote/core 0.3.1 → 0.4.1

package/dist/index.d.ts

@@ -27,11 +27,19 @@ declare class MicrophoneCapture {
     private buffer;
     private _isRecording;
     private _loggedFirstChunk;
+    /** Actual AudioContext sample rate (may differ from target on Firefox) */
+    private _nativeSampleRate;
     constructor(events: EventEmitter<OmoteEvents>, config?: MicrophoneCaptureConfig);
     get isRecording(): boolean;
     get isSupported(): boolean;
     start(): Promise<void>;
     stop(): void;
+    /**
+     * Resample audio using linear interpolation.
+     * Used when the AudioContext runs at the device's native rate (e.g. 48kHz)
+     * and we need to downsample to the target rate (e.g. 16kHz).
+     */
+    private resample;
     private floatToPCM16;
 }
 
@@ -98,12 +106,11 @@ interface AudioSchedulerOptions {
     /** Number of audio channels (default: 1 for mono) */
     channels?: number;
     /**
-     * Delay before first audio chunk plays (seconds).
-     * Gives slow inference backends (WASM) a head start so lip sync
-     * frames are ready by the time audio reaches the listener.
-     * Default: 0.05 (50ms — just enough to enqueue the first node)
+     * Initial lookahead delay in seconds before first audio plays.
+     * Gives LAM inference time to compute blendshapes before audio starts.
+     * Default: 0.05 (50ms) for WebGPU, increase to 0.3-0.5 for WASM on iOS.
      */
-    initialDelayS?: number;
+    initialLookaheadSec?: number;
 }
 declare class AudioScheduler {
     private readonly options;
@@ -373,13 +380,14 @@ declare function isSafari(): boolean;
 /**
  * Recommend using CPU-optimized lip sync model (wav2arkit_cpu)
  *
- * All WebKit browsers (Safari macOS, Safari iOS, Chrome iOS, Firefox iOS)
- * have ONNX Runtime WebGPU JSEP bugs that crash session creation, and the
- * 384MB LAM model stack-overflows in WASM mode.
- * The wav2arkit_cpu model (1.8MB) provides identical 52 ARKit blendshape
- * output at 22x real-time on CPU/WASM.
+ * All iOS browsers use WebKit and have tight memory limits — the 384MB
+ * LAM model causes silent crashes. wav2arkit_cpu uses URL pass-through
+ * (ORT fetches the 402MB weights directly into WASM, no JS heap copy).
+ *
+ * macOS Safari also needs this due to ONNX Runtime JSEP/ASYNCIFY bugs
+ * that crash WebKit's JIT compiler.
  *
- * @returns true if on Safari or any iOS browser (should use CPU lip sync model)
+ * @returns true if iOS (any browser) or Safari (any platform)
  */
 declare function shouldUseCpuLipSync(): boolean;
 /**
@@ -400,7 +408,7 @@ declare function isSpeechRecognitionAvailable(): boolean;
  * - Battery-efficient (no WASM overhead)
  * - No model download needed (saves 30-150MB)
  *
- * @returns true if on iOS with Speech API available
+ * @returns true if on iOS or Safari with Speech API available
  */
 declare function shouldUseNativeASR(): boolean;
 /**
@@ -419,7 +427,7 @@ declare function shouldUseServerLipSync(): boolean;
 /**
  * Common interface for lip sync inference backends
  *
- * Both Wav2Vec2Inference (GPU, 384MB) and Wav2ArkitCpuInference (CPU, 1.8MB)
+ * Both Wav2Vec2Inference (GPU, 384MB) and Wav2ArkitCpuInference (CPU, 404MB)
  * implement this interface, allowing SyncedAudioPipeline and LAMPipeline to
  * work with either model transparently.
  *
@@ -454,19 +462,15 @@ interface LipSyncResult {
  *
  * Implemented by:
  * - Wav2Vec2Inference (WebGPU/WASM, 384MB, ASR + lip sync)
- * - Wav2ArkitCpuInference (WASM-only, 1.8MB, lip sync only)
+ * - Wav2ArkitCpuInference (WASM-only, 404MB, lip sync only)
  */
 interface LipSyncBackend {
+    /** Model identifier for backend-specific tuning (e.g. audio delay) */
+    readonly modelId: 'wav2vec2' | 'wav2arkit_cpu';
     /** Current backend type (webgpu, wasm, or null if not loaded) */
     readonly backend: RuntimeBackend | null;
     /** Whether the model is loaded and ready for inference */
     readonly isLoaded: boolean;
-    /**
-     * Preferred number of audio samples per inference chunk.
-     * Models with variable-length input can use smaller values for lower latency.
-     * Default (if undefined): 16000 (1.0s at 16kHz, required by Wav2Vec2).
-     */
-    readonly chunkSamples?: number;
     /**
      * Load the ONNX model
      * @returns Model loading information
@@ -529,7 +533,7 @@ interface LAMPipelineOptions {
 }
 declare class LAMPipeline {
     private readonly options;
-    private readonly DEFAULT_CHUNK_SAMPLES;
+    private readonly REQUIRED_SAMPLES;
     private readonly FRAME_RATE;
     private buffer;
     private bufferStartTime;
@@ -558,13 +562,15 @@ declare class LAMPipeline {
     /**
      * Get the frame that should be displayed at the current time
      *
-     * Timestamp-synced playback for all backends. Audio playback is delayed
-     * for slow backends (WASM gets 1s head start via AudioScheduler) so
-     * frames are ready by the time their corresponding audio plays.
+     * Automatically removes frames that have already been displayed.
+     * This prevents memory leaks from accumulating old frames.
+     *
+     * Discard Window (prevents premature frame discarding):
+     * - WebGPU: 0.5s (LAM inference 20-100ms + RAF jitter + React stalls)
+     * - WASM: 1.0s (LAM inference 50-500ms + higher variability)
      *
-     * Discard window is generous for WASM to handle inference jitter.
-     * Late frames play at RAF rate (~60fps) until caught up, then settle
-     * to natural 30fps pacing via timestamp gating.
+     * Last-Frame-Hold: Returns last valid frame instead of null to prevent
+     * avatar freezing when between frames (RAF at 60fps vs LAM at 30fps).
      *
      * @param currentTime - Current AudioContext time
      * @param lam - LAM inference engine (optional, for backend detection)
@@ -592,7 +598,7 @@ declare class LAMPipeline {
     /**
      * Flush remaining buffered audio
      *
-     * Processes any remaining audio in the buffer, even if less than the chunk size.
+     * Processes any remaining audio in the buffer, even if less than REQUIRED_SAMPLES.
      * This ensures the final audio chunk generates blendshape frames.
      *
      * Should be called when audio stream ends to prevent losing the last 0-1 seconds.
@@ -645,6 +651,12 @@ interface SyncedAudioPipelineOptions {
     chunkTargetMs?: number;
     /** LAM inference engine */
     lam: LipSyncBackend;
+    /**
+     * Audio playback delay in ms before first audio plays.
+     * Gives LAM inference time to pre-compute blendshapes.
+     * Default: auto-detected from lam.backend (50ms WebGPU, 350ms WASM).
+     */
+    audioDelayMs?: number;
 }
 interface SyncedAudioPipelineEvents {
     /** New frame ready for display */
@@ -838,134 +850,219 @@ declare function getLoadedBackend(): RuntimeBackend | null;
  * Check if ONNX Runtime has been loaded
  */
 declare function isOnnxRuntimeLoaded(): boolean;
+/**
+ * Preload ONNX Runtime and compile the WASM binary early
+ *
+ * Call this before loading heavy resources (Three.js, VRM models) to ensure
+ * WASM memory is allocated in a clean JS heap, reducing iOS memory pressure.
+ * Uses the singleton pattern — subsequent model loading reuses this instance.
+ *
+ * @param preference Backend preference (default: 'auto')
+ * @returns The resolved backend that was loaded
+ */
+declare function preloadOnnxRuntime(preference?: BackendPreference): Promise<RuntimeBackend>;
 
 /**
- * Whisper Automatic Speech Recognition using transformers.js
- * Uses Xenova's proven pipeline API for reliable transcription
- */
-type WhisperModel = 'tiny' | 'base' | 'small' | 'medium';
-type WhisperDtype = 'fp32' | 'fp16' | 'q8' | 'int8' | 'uint8' | 'q4' | 'q4f16' | 'bnb4';
-interface WhisperConfig {
-    /** Model size: tiny (~75MB), base (~150MB), small (~500MB), medium (~1.5GB) */
-    model?: WhisperModel;
-    /** Use multilingual model (default: false, uses .en models) */
-    multilingual?: boolean;
-    /** Language code (e.g., 'en', 'es', 'fr') - for multilingual models */
-    language?: string;
-    /** Task: transcribe or translate (default: transcribe) */
-    task?: 'transcribe' | 'translate';
-    /** Model quantization format (default: 'q8' for balance of speed/quality) */
-    dtype?: WhisperDtype;
-    /** Use WebGPU acceleration if available (default: auto-detect) */
-    device?: 'auto' | 'webgpu' | 'wasm';
-    /** Local model path (e.g., '/models/whisper-tiny.en') - overrides HuggingFace CDN */
-    localModelPath?: string;
-    /** HuggingFace API token to bypass rate limits (get from https://huggingface.co/settings/tokens) */
-    token?: string;
-    /** Suppress non-speech tokens like [LAUGHTER], [CLICKING], etc. (default: true) */
-    suppressNonSpeech?: boolean;
-}
-interface TranscriptionResult {
+ * SenseVoice automatic speech recognition using ONNX Runtime Web
+ *
+ * Non-autoregressive CTC-based ASR that is 5x faster than Whisper-Small.
+ * Runs entirely in browser via WebGPU or WASM. No transformers.js dependency.
+ *
+ * Uses the sherpa-onnx SenseVoice export (model.int8.onnx, 239MB int8 quantized).
+ * Also provides emotion detection, language identification, and audio event detection
+ * from the same forward pass.
+ *
+ * @category Inference
+ *
+ * @example Basic usage
+ * ```typescript
+ * import { SenseVoiceInference } from '@omote/core';
+ *
+ * const asr = new SenseVoiceInference({
+ *   modelUrl: '/models/sensevoice/model.int8.onnx',
+ *   tokensUrl: '/models/sensevoice/tokens.txt',
+ * });
+ * await asr.load();
+ *
+ * const { text, emotion, language } = await asr.transcribe(audioSamples);
+ * console.log(text);       // "Hello world"
+ * console.log(emotion);    // "NEUTRAL"
+ * console.log(language);   // "en"
+ * ```
+ *
+ * @module inference/SenseVoiceInference
+ */
+
+type SenseVoiceLanguage = 'auto' | 'zh' | 'en' | 'ja' | 'ko' | 'yue';
+interface SenseVoiceConfig {
+    /** Path or URL to model.int8.onnx (239MB) */
+    modelUrl: string;
+    /** Path or URL to tokens.txt vocabulary file (default: sibling of modelUrl) */
+    tokensUrl?: string;
+    /** Language hint (default: 'auto' for auto-detection) */
+    language?: SenseVoiceLanguage;
+    /** Text normalization: 'with_itn' applies inverse text normalization (default: 'with_itn') */
+    textNorm?: 'with_itn' | 'without_itn';
+    /** Preferred backend (default: 'auto') */
+    backend?: BackendPreference;
+}
+interface SenseVoiceResult {
     /** Transcribed text */
     text: string;
-    /** Detected/used language */
-    language: string;
-    /** Inference time in ms */
+    /** Detected language (e.g., 'zh', 'en', 'ja', 'ko', 'yue') */
+    language?: string;
+    /** Detected emotion (e.g., 'HAPPY', 'SAD', 'ANGRY', 'NEUTRAL') */
+    emotion?: string;
+    /** Detected audio event (e.g., 'Speech', 'BGM', 'Laughter') */
+    event?: string;
+    /** Inference time in milliseconds (preprocessing + model + decode) */
     inferenceTimeMs: number;
-    /** Full chunks with timestamps (if requested) */
-    chunks?: Array<{
-        text: string;
-        timestamp: [number, number | null];
-    }>;
+    /** Preprocessing time in milliseconds (fbank + LFR + CMVN) */
+    preprocessTimeMs: number;
 }
-/**
- * Whisper ASR inference using transformers.js pipeline API
- *
- * Features:
- * - Automatic WebGPU/WASM backend selection
- * - Streaming support with chunk callbacks
- * - Proven implementation from Xenova's demo
- * - Handles all audio preprocessing automatically
- */
-declare class WhisperInference {
+interface SenseVoiceModelInfo {
+    backend: RuntimeBackend;
+    loadTimeMs: number;
+    inputNames: string[];
+    outputNames: string[];
+    vocabSize: number;
+}
+declare class SenseVoiceInference {
+    private session;
+    private ort;
     private config;
-    private pipeline;
-    private currentModel;
+    private _backend;
     private isLoading;
-    private actualBackend;
-    constructor(config?: WhisperConfig);
-    /**
-     * Check if WebGPU is available in this browser
-     */
-    static isWebGPUAvailable(): Promise<boolean>;
-    /**
-     * Load the Whisper model pipeline
-     */
-    load(onProgress?: (progress: {
-        status: string;
-        progress?: number;
-        file?: string;
-    }) => void): Promise<void>;
-    /**
-     * Transcribe audio to text
-     *
-     * @param audio Audio samples (Float32Array, 16kHz mono)
-     * @param options Transcription options
-     */
-    transcribe(audio: Float32Array, options?: {
-        /** Return timestamps for each chunk */
-        returnTimestamps?: boolean;
-        /** Chunk length in seconds (default: 30) */
-        chunkLengthS?: number;
-        /** Stride length in seconds for overlapping chunks (default: 5) */
-        strideLengthS?: number;
-        /** Language override */
-        language?: string;
-        /** Task override */
-        task?: 'transcribe' | 'translate';
-    }): Promise<TranscriptionResult>;
-    /**
-     * Transcribe with streaming chunks (progressive results)
-     *
-     * @param audio Audio samples
-     * @param onChunk Called when each chunk is finalized
-     * @param onUpdate Called after each generation step (optional)
-     */
-    transcribeStreaming(audio: Float32Array, onChunk: (chunk: {
-        text: string;
-        timestamp: [number, number | null];
-    }) => void, onUpdate?: (text: string) => void, options?: {
-        chunkLengthS?: number;
-        strideLengthS?: number;
-        language?: string;
-        task?: 'transcribe' | 'translate';
-    }): Promise<TranscriptionResult>;
-    /**
-     * Dispose of the model and free resources
-     */
-    dispose(): Promise<void>;
-    /**
-     * Check if model is loaded
-     */
+    private inferenceQueue;
+    private tokenMap;
+    private negMean;
+    private invStddev;
+    private languageId;
+    private textNormId;
+    constructor(config: SenseVoiceConfig);
+    get backend(): RuntimeBackend | null;
     get isLoaded(): boolean;
+    load(onProgress?: (loaded: number, total: number) => void): Promise<SenseVoiceModelInfo>;
     /**
-     * Get the backend being used (webgpu or wasm)
-     */
-    get backend(): string;
-    /**
-     * Get the full model name used by transformers.js
-     */
-    private getModelName;
-    /**
-     * Remove non-speech event tokens from transcription
-     *
-     * Whisper outputs special tokens for non-speech events like:
-     * [LAUGHTER], [APPLAUSE], [MUSIC], [BLANK_AUDIO], [CLICKING], etc.
+     * Transcribe audio samples to text
      *
-     * This method strips these tokens and cleans up extra whitespace.
+     * @param audioSamples Float32Array of audio samples at 16kHz, [-1, 1] range
+     * @returns Transcription result with text, emotion, language, and event
      */
-    private removeNonSpeechTokens;
+    transcribe(audioSamples: Float32Array): Promise<SenseVoiceResult>;
+    private queueInference;
+    dispose(): Promise<void>;
+}
+
+/**
+ * Kaldi-compatible filterbank (fbank) feature extraction
+ *
+ * Pure TypeScript implementation matching kaldi-native-fbank parameters
+ * used by SenseVoice. No external dependencies.
+ *
+ * Pipeline: audio → framing → windowing → FFT → power spectrum → mel filterbank → log
+ *
+ * @module inference/kaldiFbank
+ */
+interface KaldiFbankOptions {
+    /** Frame length in ms (default: 25) */
+    frameLengthMs?: number;
+    /** Frame shift in ms (default: 10) */
+    frameShiftMs?: number;
+    /** Low frequency cutoff in Hz (default: 20) */
+    lowFreq?: number;
+    /** High frequency cutoff in Hz (default: sampleRate / 2) */
+    highFreq?: number;
+    /** Dither amount (default: 0 for deterministic output) */
+    dither?: number;
+    /** Preemphasis coefficient (default: 0.97) */
+    preemphasis?: number;
+}
+/**
+ * Compute Kaldi-compatible log mel filterbank features
+ *
+ * @param audio Raw audio samples (float32, [-1, 1] range)
+ * @param sampleRate Sample rate in Hz (must be 16000 for SenseVoice)
+ * @param numMelBins Number of mel bins (80 for SenseVoice)
+ * @param opts Optional parameters
+ * @returns Flattened Float32Array of shape [numFrames, numMelBins]
+ */
+declare function computeKaldiFbank(audio: Float32Array, sampleRate: number, numMelBins: number, opts?: KaldiFbankOptions): Float32Array;
+/**
+ * Apply Low Frame Rate stacking for SenseVoice
+ *
+ * Concatenates lfrM consecutive frames with stride lfrN.
+ * Left-pads with copies of first frame, right-pads last group.
+ *
+ * @param features Flattened [numFrames, featureDim]
+ * @param featureDim Feature dimension per frame (e.g., 80)
+ * @param lfrM Number of frames to stack (default: 7)
+ * @param lfrN Stride (default: 6)
+ * @returns Flattened [numOutputFrames, featureDim * lfrM]
+ */
+declare function applyLFR(features: Float32Array, featureDim: number, lfrM?: number, lfrN?: number): Float32Array;
+/**
+ * Apply CMVN normalization in-place
+ *
+ * Formula: normalized[i] = (features[i] + negMean[i % dim]) * invStddev[i % dim]
+ *
+ * @param features Flattened feature array (modified in-place)
+ * @param dim Feature dimension (560 for SenseVoice after LFR)
+ * @param negMean Negative mean vector (dim-dimensional)
+ * @param invStddev Inverse standard deviation vector (dim-dimensional)
+ * @returns The same features array (for chaining)
+ */
+declare function applyCMVN(features: Float32Array, dim: number, negMean: Float32Array, invStddev: Float32Array): Float32Array;
+/**
+ * Parse CMVN vectors from comma-separated strings (stored in ONNX metadata)
+ *
+ * The sherpa-onnx SenseVoice export stores neg_mean and inv_stddev
+ * as comma-separated float strings in the model's metadata.
+ */
+declare function parseCMVNFromMetadata(negMeanStr: string, invStddevStr: string): {
+    negMean: Float32Array;
+    invStddev: Float32Array;
+};
+
+/**
+ * CTC greedy decoder for SenseVoice
+ *
+ * Decodes CTC logits into text with structured token parsing
+ * for language, emotion, and audio event detection.
+ *
+ * @module inference/ctcDecoder
+ */
+interface CTCDecodeResult {
+    /** Decoded text (speech content only) */
+    text: string;
+    /** Detected language (e.g., 'zh', 'en', 'ja', 'ko', 'yue') */
+    language?: string;
+    /** Detected emotion (e.g., 'HAPPY', 'SAD', 'ANGRY', 'NEUTRAL') */
+    emotion?: string;
+    /** Detected audio event (e.g., 'Speech', 'BGM', 'Laughter') */
+    event?: string;
 }
+/** Resolve language string to SenseVoice language ID */
+declare function resolveLanguageId(language: string): number;
+/** Resolve text norm string to SenseVoice text norm ID */
+declare function resolveTextNormId(textNorm: string): number;
+/**
+ * Parse tokens.txt into a token ID → string map
+ *
+ * Format: each line is "token_string token_id"
+ * e.g., "<unk> 0", "▁the 3", "s 4"
+ */
+declare function parseTokensFile(content: string): Map<number, string>;
+/**
+ * CTC greedy decode
+ *
+ * @param logits Raw logits from model output, flattened [seqLen, vocabSize]
+ * @param seqLen Sequence length (time steps)
+ * @param vocabSize Vocabulary size
+ * @param tokenMap Token ID → string map from tokens.txt
+ * @returns Decoded text and structured metadata
+ */
+declare function ctcGreedyDecode(logits: Float32Array, seqLen: number, vocabSize: number, tokenMap: Map<number, string>): CTCDecodeResult;
 
 /**
  * Shared blendshape constants and utilities for lip sync inference
@@ -1036,6 +1133,13 @@ type InferenceBackend = BackendPreference;
 interface Wav2Vec2InferenceConfig {
     /** Path or URL to the ONNX model */
     modelUrl: string;
+    /**
+     * Path or URL to external model data file (.onnx.data weights).
+     * Default: `${modelUrl}.data` (e.g., /models/model.onnx.data)
+     *
+     * Set to `false` to skip external data loading (single-file models only).
+     */
+    externalDataUrl?: string | false;
     /** Preferred backend (auto will try WebGPU first, fallback to WASM) */
     backend?: InferenceBackend;
     /** Number of identity classes (default: 12 for streaming model) */
@@ -1066,7 +1170,8 @@ interface Wav2Vec2Result {
     /** Inference time in ms */
     inferenceTimeMs: number;
 }
-declare class Wav2Vec2Inference {
+declare class Wav2Vec2Inference implements LipSyncBackend {
+    readonly modelId: "wav2vec2";
     private session;
     private ort;
     private config;
@@ -1116,12 +1221,16 @@ declare class Wav2Vec2Inference {
 /**
  * CPU-optimized lip sync inference using wav2arkit_cpu model
  *
- * A lightweight (1.8MB) alternative to Wav2Vec2Inference (384MB) designed
- * for Safari/iOS where WebGPU crashes due to ONNX Runtime JSEP bugs.
+ * A Safari/iOS-compatible alternative to Wav2Vec2Inference (384MB) designed
+ * for platforms where WebGPU crashes due to ONNX Runtime JSEP bugs.
+ *
+ * The model uses ONNX external data format:
+ * - wav2arkit_cpu.onnx (1.86MB graph structure)
+ * - wav2arkit_cpu.onnx.data (402MB weights)
+ * Both files are fetched and cached automatically.
  *
  * Key differences from Wav2Vec2Inference:
- * - WASM-only backend (CPU-optimized, no WebGPU)
- * - 1.8MB model vs 384MB
+ * - WASM-only backend (CPU-optimized, single-threaded, no WebGPU)
  * - No identity input (baked to identity 11)
  * - No ASR output (lip sync only)
  * - Dynamic input length (not fixed to 16000 samples)
@@ -1146,12 +1255,18 @@ declare class Wav2Vec2Inference {
 interface Wav2ArkitCpuConfig {
     /** Path or URL to the wav2arkit_cpu ONNX model (.onnx graph file) */
     modelUrl: string;
-    /** Path or URL to the external data file (.onnx.data weights file) */
-    modelDataUrl?: string;
+    /**
+     * Path or URL to external model data file (.onnx.data weights).
+     * Default: `${modelUrl}.data` (e.g., /models/wav2arkit_cpu.onnx.data)
+     *
+     * Set to `false` to skip external data loading (single-file models only).
+     */
+    externalDataUrl?: string | false;
     /** Preferred backend (default: 'wasm' — this model is CPU-optimized) */
     backend?: BackendPreference;
 }
 declare class Wav2ArkitCpuInference implements LipSyncBackend {
+    readonly modelId: "wav2arkit_cpu";
     private session;
     private ort;
     private config;
@@ -1161,12 +1276,6 @@ declare class Wav2ArkitCpuInference implements LipSyncBackend {
     constructor(config: Wav2ArkitCpuConfig);
     get backend(): RuntimeBackend | null;
     get isLoaded(): boolean;
-    /**
-     * Preferred chunk size: 4000 samples (250ms at 16kHz).
-     * wav2arkit_cpu accepts variable-length input, so we use smaller chunks
-     * for lower latency on WASM (vs 16000 for Wav2Vec2's fixed requirement).
-     */
-    readonly chunkSamples = 4000;
     /**
      * Load the ONNX model
      */
@@ -1195,10 +1304,20 @@ declare class Wav2ArkitCpuInference implements LipSyncBackend {
  * Factory function for lip sync with automatic GPU/CPU model selection
  *
  * Provides a unified API that automatically selects the optimal model:
- * - Safari (macOS + iOS): Uses Wav2ArkitCpuInference (1.8MB, WASM)
+ * - Safari (macOS + iOS): Uses Wav2ArkitCpuInference (404MB, WASM)
  * - Chrome/Firefox/Edge: Uses Wav2Vec2Inference (384MB, WebGPU)
  * - Fallback: Gracefully falls back to CPU model if GPU model fails to load
  *
+ * Why two separate models?
+ * Wav2Vec2 (LAM) cannot run on Safari/iOS for two reasons:
+ * 1. Its dual-head transformer graph needs ~750-950MB peak during ORT session
+ *    creation (graph optimization), exceeding iOS WebKit's ~1-1.5GB tab limit.
+ * 2. It ships as a single 384MB .onnx file that must load into JS heap before
+ *    ORT can consume it. iOS WebKit OOMs on this allocation.
+ * wav2arkit_cpu solves both: external data format (1.86MB graph + 402MB weights)
+ * lets ORT load only the tiny graph, then stream weights via URL pass-through
+ * directly into WASM memory. JS heap stays at ~2MB.
+ *
  * @category Inference
  *
  * @example Auto-detect (recommended)
@@ -1230,10 +1349,15 @@ declare class Wav2ArkitCpuInference implements LipSyncBackend {
 interface CreateLipSyncConfig {
     /** URL for the GPU model (Wav2Vec2, used on Chrome/Firefox/Edge) */
     gpuModelUrl: string;
+    /**
+     * URL for GPU model external data file (.onnx.data weights).
+     * Default: `${gpuModelUrl}.data`
+     *
+     * Set to `false` to skip external data loading (single-file models only).
+     */
+    gpuExternalDataUrl?: string | false;
     /** URL for the CPU model (wav2arkit_cpu, used on Safari/iOS) */
     cpuModelUrl: string;
-    /** URL for the CPU model's external data file (.onnx.data weights) */
-    cpuModelDataUrl?: string;
     /**
      * Model selection mode:
      * - 'auto': Safari/iOS → CPU, everything else → GPU (default)
@@ -1388,6 +1512,7 @@ declare class SileroVADInference {
     private inferenceQueue;
     private preSpeechBuffer;
     private wasSpeaking;
+    private srTensor;
     constructor(config: SileroVADConfig);
     get backend(): RuntimeBackend | null;
     get isLoaded(): boolean;
@@ -2444,7 +2569,7 @@ declare class AgentCoreAdapter extends EventEmitter<AIAdapterEvents> implements
     private _state;
     private _sessionId;
     private _isConnected;
-    private whisper;
+    private asr;
     private vad;
     private lam;
     private emotionController;
@@ -2488,7 +2613,7 @@ declare class AgentCoreAdapter extends EventEmitter<AIAdapterEvents> implements
     healthCheck(): Promise<boolean>;
     private setState;
     private getAuthToken;
-    private initWhisper;
+    private initASR;
     private initLAM;
     private initPipeline;
     private connectWebSocket;
@@ -3182,148 +3307,6 @@ declare function preloadModels(urls: string[], onProgress?: (current: number, to
  */
 declare function formatBytes(bytes: number): string;
 
-/**
- * HuggingFace CDN Utilities
- *
- * Helper functions for working with HuggingFace CDN URLs.
- * Used by transformers.js models (Whisper, etc.) for model downloads.
- *
- * @category Cache
- */
-/**
- * Test URL for HuggingFace CDN reachability check.
- * Uses a small, stable file from a well-known public model.
- */
-declare const HF_CDN_TEST_URL = "https://huggingface.co/Xenova/whisper-tiny/resolve/main/config.json";
-/**
- * Parsed HuggingFace URL components
- */
-interface HuggingFaceUrlInfo {
-    /** Organization or username */
-    org: string;
-    /** Model name */
-    model: string;
-    /** Branch, tag, or commit */
-    branch: string;
-    /** File path within the repository */
-    file: string;
-}
-/**
- * Parse a HuggingFace CDN URL into its components
- *
- * @param url - The HuggingFace URL to parse
- * @returns Parsed URL info or null if not a valid HF URL
- *
- * @example
- * ```typescript
- * const info = parseHuggingFaceUrl(
- *   'https://huggingface.co/openai/whisper-tiny/resolve/main/model.onnx'
- * );
- * // Returns: { org: 'openai', model: 'whisper-tiny', branch: 'main', file: 'model.onnx' }
- * ```
- */
-declare function parseHuggingFaceUrl(url: string): HuggingFaceUrlInfo | null;
-/**
- * Check if HuggingFace CDN is reachable
- *
- * Performs a HEAD request to a known HuggingFace model file to verify
- * connectivity. Useful for offline detection or network diagnostics.
- *
- * @param testUrl - Optional custom URL to test (defaults to HF_CDN_TEST_URL)
- * @returns True if CDN is reachable, false otherwise
- *
- * @example
- * ```typescript
- * import { isHuggingFaceCDNReachable } from '@omote/core';
- *
- * const reachable = await isHuggingFaceCDNReachable();
- * if (!reachable) {
- *   console.log('HuggingFace CDN unreachable - running offline?');
- *   // Fall back to cached models or show error
- * }
- * ```
- */
-declare function isHuggingFaceCDNReachable(testUrl?: string): Promise<boolean>;
-
-/**
- * Utility to clear transformers.js Cache API storage
- *
- * Problem: transformers.js v4 uses Browser Cache API which persists across hard refreshes.
- * If an HTML error page gets cached (due to network errors, CDN issues, or dev server restarts),
- * it will be served instead of JSON files, causing JSON.parse() errors.
- *
- * Solution: Manually clear Cache API storage before loading models.
- *
- * @module utils/transformersCacheClear
- */
-/**
- * Clear all transformers.js and HuggingFace caches from Browser Cache API
- *
- * This clears:
- * - transformers-cache (default cache key)
- * - Any caches with 'transformers' or 'huggingface' in the name
- *
- * @param options Configuration options
- * @returns Promise resolving to array of deleted cache names
- */
-declare function clearTransformersCache(options?: {
-    /** Whether to log deletion details (default: true) */
-    verbose?: boolean;
-    /** Additional cache name patterns to clear (e.g., ['my-custom-cache']) */
-    additionalPatterns?: string[];
-}): Promise<string[]>;
-/**
- * Clear a specific cache by exact name
- *
- * @param cacheName Exact cache name to delete
- * @returns Promise resolving to true if deleted, false otherwise
- */
-declare function clearSpecificCache(cacheName: string): Promise<boolean>;
-/**
- * List all cache names currently stored
- *
- * @returns Promise resolving to array of cache names
- */
-declare function listCaches(): Promise<string[]>;
-/**
- * Check if a specific cached response is valid JSON/binary (not HTML error page)
- *
- * @param cacheName Cache name to check
- * @param requestUrl URL/key to check
- * @returns Promise resolving to validation result
- */
-declare function validateCachedResponse(cacheName: string, requestUrl: string): Promise<{
-    exists: boolean;
-    valid: boolean;
-    contentType: string | null;
-    isHtml: boolean;
-    reason?: string;
-}>;
-/**
- * Scan all caches for potentially invalid cached responses
- *
- * @returns Promise resolving to report of invalid entries
- */
-declare function scanForInvalidCaches(): Promise<{
-    totalCaches: number;
-    scannedEntries: number;
-    invalidEntries: Array<{
-        cacheName: string;
-        url: string;
-        reason: string;
-    }>;
-}>;
-/**
- * Clear all caches and optionally prevent re-creation (development mode)
- *
- * WARNING: This is aggressive and should only be used in development.
- * It clears ALL browser caches, not just transformers.js.
- *
- * @param preventRecreation If true, sets env.useBrowserCache = false
- * @returns Promise resolving to number of deleted caches
- */
-declare function nukeBrowserCaches(preventRecreation?: boolean): Promise<number>;
-
 /**
  * Telemetry Types
  *
@@ -4086,4 +4069,4 @@ declare class EmphasisDetector {
     reset(): void;
 }
 
-export { type AIAdapter, type AIAdapterEvents, type AISessionState, ARKIT_BLENDSHAPES, type ActiveSpan, AgentCoreAdapter, type AgentCoreConfig, type AnimationClip, AnimationEvent, AnimationGraph, type AnimationGraphConfig, type AnimationGraphEvents, type AnimationLayer, type AnimationOutput, type AnimationState, type AnimationStateName, type AnimationTrigger, AudioChunkCoalescer, type AudioChunkCoalescerOptions, AudioEnergyAnalyzer, AudioScheduler, type AudioSchedulerOptions, type AudioSyncConfig, type AudioSyncEvents, AudioSyncManager, type BackendPreference, type BlendWeight, CTC_VOCAB, type CacheConfig, type CacheSpanAttributes, ConsoleExporter, type ConversationMessage, ConversationOrchestrator, type ConversationSession, type CreateLipSyncConfig, DEFAULT_ANIMATION_CONFIG, EMOTION_NAMES, EMOTION_VECTOR_SIZE, type EmotionAnimationMap, EmotionController, type EmotionLabel, type EmotionName, type EmotionPresetName, EmotionPresets, type EmotionWeights, EmphasisDetector, EventEmitter, type FetchWithCacheOptions, HF_CDN_TEST_URL, type HuggingFaceUrlInfo, INFERENCE_LATENCY_BUCKETS, type InferenceSpanAttributes, type InterruptionConfig, type InterruptionEvents, InterruptionHandler, type LAMFrame, LAMPipeline, type LAMPipelineOptions, LAM_BLENDSHAPES, type LipSyncBackend, type LipSyncModelInfo, type LipSyncResult, MODEL_LOAD_TIME_BUCKETS, type MessageRole, type MetricData, MetricNames, MicrophoneCapture, type MicrophoneCaptureConfig, ModelCache, type ModelSpanAttributes, OTLPExporter, type OTLPExporterConfig, OmoteEvents, OmoteTelemetry, type OrchestratorConfig, type OrchestratorEvents, type QuotaInfo, RingBuffer, type RuntimeBackend, type SafariSpeechConfig, SafariSpeechRecognition, type SamplingConfig, type SessionConfig, type SessionOptions, type SessionSnapshot, type SileroVADBackend, type SileroVADConfig, type SileroVADFactoryConfig, SileroVADInference, SileroVADWorker, type SpanAttributes, type SpanData, type SpeechErrorCallback, type SpeechRecognitionResult, type SpeechResultCallback, type SpeechSegment, SyncedAudioPipeline, type SyncedAudioPipelineEvents, type SyncedAudioPipelineOptions, type TelemetryConfig, type TelemetryExporter, type TelemetryExporterInterface, type TenantConfig, TenantManager, type TenantQuota, type TenantUsage, type TokenRefreshCallback, type TranscriptionResult, type Transition, type VADBackend, type VADModelInfo, type VADResult$1 as VADResult, type VADWorkerConfig, type VADWorkerModelInfo, type ValidationResult, type VoiceConfig, WAV2ARKIT_BLENDSHAPES, type Wav2ArkitCpuConfig, Wav2ArkitCpuInference, Wav2Vec2Inference, type Wav2Vec2InferenceConfig, type Wav2Vec2Result, type WhisperConfig, WhisperInference, type WhisperModel, blendEmotions, calculatePeak, calculateRMS, clearSpecificCache, clearTransformersCache, configureCacheLimit, configureTelemetry, createEmotionVector, createLipSync, createSessionWithFallback, createSileroVAD, fetchWithCache, formatBytes, getCacheConfig, getCacheKey, getEmotionPreset, getLoadedBackend, getModelCache, getOnnxRuntime, getOnnxRuntimeForPreference, getOptimalWasmThreads, getRecommendedBackend, getSessionOptions, getTelemetry, hasWebGPUApi, isAndroid, isHuggingFaceCDNReachable, isIOS, isIOSSafari, isMobile, isOnnxRuntimeLoaded, isSafari, isSpeechRecognitionAvailable, isWebGPUAvailable, lerpEmotion, listCaches, nukeBrowserCaches, parseHuggingFaceUrl, preloadModels, remapWav2ArkitToLam, resolveBackend, scanForInvalidCaches, shouldEnableWasmProxy, shouldUseCpuLipSync, shouldUseNativeASR, shouldUseServerLipSync, supportsVADWorker, symmetrizeBlendshapes, validateCachedResponse };
+export { type AIAdapter, type AIAdapterEvents, type AISessionState, ARKIT_BLENDSHAPES, type ActiveSpan, AgentCoreAdapter, type AgentCoreConfig, type AnimationClip, AnimationEvent, AnimationGraph, type AnimationGraphConfig, type AnimationGraphEvents, type AnimationLayer, type AnimationOutput, type AnimationState, type AnimationStateName, type AnimationTrigger, AudioChunkCoalescer, type AudioChunkCoalescerOptions, AudioEnergyAnalyzer, AudioScheduler, type AudioSchedulerOptions, type AudioSyncConfig, type AudioSyncEvents, AudioSyncManager, type BackendPreference, type BlendWeight, type CTCDecodeResult, CTC_VOCAB, type CacheConfig, type CacheSpanAttributes, ConsoleExporter, type ConversationMessage, ConversationOrchestrator, type ConversationSession, type CreateLipSyncConfig, DEFAULT_ANIMATION_CONFIG, EMOTION_NAMES, EMOTION_VECTOR_SIZE, type EmotionAnimationMap, EmotionController, type EmotionLabel, type EmotionName, type EmotionPresetName, EmotionPresets, type EmotionWeights, EmphasisDetector, EventEmitter, type FetchWithCacheOptions, INFERENCE_LATENCY_BUCKETS, type InferenceSpanAttributes, type InterruptionConfig, type InterruptionEvents, InterruptionHandler, type KaldiFbankOptions, type LAMFrame, LAMPipeline, type LAMPipelineOptions, LAM_BLENDSHAPES, type LipSyncBackend, type LipSyncModelInfo, type LipSyncResult, MODEL_LOAD_TIME_BUCKETS, type MessageRole, type MetricData, MetricNames, MicrophoneCapture, type MicrophoneCaptureConfig, ModelCache, type ModelSpanAttributes, OTLPExporter, type OTLPExporterConfig, OmoteEvents, OmoteTelemetry, type OrchestratorConfig, type OrchestratorEvents, type QuotaInfo, RingBuffer, type RuntimeBackend, type SafariSpeechConfig, SafariSpeechRecognition, type SamplingConfig, type SenseVoiceConfig, SenseVoiceInference, type SenseVoiceLanguage, type SenseVoiceModelInfo, type SenseVoiceResult, type SessionConfig, type SessionOptions, type SessionSnapshot, type SileroVADBackend, type SileroVADConfig, type SileroVADFactoryConfig, SileroVADInference, SileroVADWorker, type SpanAttributes, type SpanData, type SpeechErrorCallback, type SpeechRecognitionResult, type SpeechResultCallback, type SpeechSegment, SyncedAudioPipeline, type SyncedAudioPipelineEvents, type SyncedAudioPipelineOptions, type TelemetryConfig, type TelemetryExporter, type TelemetryExporterInterface, type TenantConfig, TenantManager, type TenantQuota, type TenantUsage, type TokenRefreshCallback, type Transition, type VADBackend, type VADModelInfo, type VADResult$1 as VADResult, type VADWorkerConfig, type VADWorkerModelInfo, type ValidationResult, type VoiceConfig, WAV2ARKIT_BLENDSHAPES, type Wav2ArkitCpuConfig, Wav2ArkitCpuInference, Wav2Vec2Inference, type Wav2Vec2InferenceConfig, type Wav2Vec2Result, applyCMVN, applyLFR, blendEmotions, calculatePeak, calculateRMS, computeKaldiFbank, configureCacheLimit, configureTelemetry, createEmotionVector, createLipSync, createSessionWithFallback, createSileroVAD, ctcGreedyDecode, fetchWithCache, formatBytes, getCacheConfig, getCacheKey, getEmotionPreset, getLoadedBackend, getModelCache, getOnnxRuntime, getOnnxRuntimeForPreference, getOptimalWasmThreads, getRecommendedBackend, getSessionOptions, getTelemetry, hasWebGPUApi, isAndroid, isIOS, isIOSSafari, isMobile, isOnnxRuntimeLoaded, isSafari, isSpeechRecognitionAvailable, isWebGPUAvailable, lerpEmotion, parseCMVNFromMetadata, parseTokensFile, preloadModels, preloadOnnxRuntime, remapWav2ArkitToLam, resolveBackend, resolveLanguageId, resolveTextNormId, shouldEnableWasmProxy, shouldUseCpuLipSync, shouldUseNativeASR, shouldUseServerLipSync, supportsVADWorker, symmetrizeBlendshapes };