@omote/core 0.6.2 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -35,11 +35,7 @@ The most common use case: feed TTS audio chunks and get back 52 ARKit blendshape
35
35
  import { FullFacePipeline, createA2E } from '@omote/core';
36
36
 
37
37
  // 1. Create A2E backend (auto-detects GPU vs CPU)
38
- const lam = createA2E({
39
- gpuModelUrl: '/models/lam-wav2vec2.onnx',
40
- cpuModelUrl: '/models/wav2arkit_cpu.onnx',
41
- mode: 'auto',
42
- });
38
+ const lam = createA2E(); // auto-detects GPU vs CPU, fetches from HF CDN (192MB fp16)
43
39
  await lam.load();
44
40
 
45
41
  // 2. Create pipeline with expression profile
@@ -72,12 +68,7 @@ Auto-detects platform: Chrome/Edge/Android use WebGPU, Safari/iOS use WASM CPU f
72
68
  ```typescript
73
69
  import { createA2E } from '@omote/core';
74
70
 
75
- const a2e = createA2E({
76
- gpuModelUrl: '/models/lam-wav2vec2.onnx', // 384MB, WebGPU
77
- cpuModelUrl: '/models/wav2arkit_cpu.onnx', // 404MB, WASM
78
- mode: 'auto', // 'auto' | 'gpu' | 'cpu'
79
- fallbackOnError: true, // GPU failure → auto-switch to CPU
80
- });
71
+ const a2e = createA2E(); // auto-detects: GPU (192MB fp16) or CPU (404MB WASM)
81
72
  await a2e.load();
82
73
 
83
74
  const { blendshapes } = await a2e.infer(audioSamples); // Float32Array (16kHz)
@@ -89,7 +80,7 @@ const { blendshapes } = await a2e.infer(audioSamples); // Float32Array (16kHz)
89
80
  ```typescript
90
81
  import { Wav2Vec2Inference, LAM_BLENDSHAPES } from '@omote/core';
91
82
 
92
- const lam = new Wav2Vec2Inference({ modelUrl: '/models/lam-wav2vec2.onnx' });
83
+ const lam = new Wav2Vec2Inference({ modelUrl: '/models/model_fp16.onnx' });
93
84
  await lam.load();
94
85
 
95
86
  const { blendshapes } = await lam.infer(audioSamples);
@@ -317,7 +308,7 @@ Place models in your public assets directory:
317
308
 
318
309
  ```
319
310
  public/models/
320
- lam-wav2vec2.onnx # A2E lip sync — WebGPU (384MB)
311
+ model_fp16.onnx # A2E lip sync — WebGPU (192MB fp16, from omote-ai/lam-a2e)
321
312
  wav2arkit_cpu.onnx # A2E lip sync — WASM fallback (1.86MB graph)
322
313
  wav2arkit_cpu.onnx.data # A2E lip sync — WASM fallback (402MB weights)
323
314
  sensevoice/model.int8.onnx # SenseVoice ASR (239MB)
package/dist/index.d.mts CHANGED
@@ -380,7 +380,7 @@ declare function isSafari(): boolean;
380
380
  /**
381
381
  * Recommend using CPU-optimized A2E model (wav2arkit_cpu)
382
382
  *
383
- * All iOS browsers use WebKit and have tight memory limits — the 384MB
383
+ * All iOS browsers use WebKit and have tight memory limits — the 192MB fp16
384
384
  * LAM model causes silent crashes. wav2arkit_cpu uses URL pass-through
385
385
  * (ORT fetches the 402MB weights directly into WASM, no JS heap copy).
386
386
  *
@@ -427,8 +427,8 @@ declare function shouldUseServerA2E(): boolean;
427
427
  /**
428
428
  * Common interface for audio-to-expression (A2E) inference backends
429
429
  *
430
- * Both Wav2Vec2Inference (GPU, 384MB) and Wav2ArkitCpuInference (CPU, 404MB)
431
- * implement this interface, allowing SyncedAudioPipeline and LAMPipeline to
430
+ * Both Wav2Vec2Inference (GPU, 192MB fp16) and Wav2ArkitCpuInference (CPU, 404MB)
431
+ * implement this interface, allowing FullFacePipeline and A2EProcessor to
432
432
  * work with either model transparently.
433
433
  *
434
434
  * @category Inference
@@ -461,7 +461,7 @@ interface A2EResult {
461
461
  * Common interface for A2E (audio-to-expression) inference engines
462
462
  *
463
463
  * Implemented by:
464
- * - Wav2Vec2Inference (WebGPU/WASM, 384MB, ASR + A2E)
464
+ * - Wav2Vec2Inference (WebGPU/WASM, 192MB fp16, A2E)
465
465
  * - Wav2ArkitCpuInference (WASM-only, 404MB, A2E only)
466
466
  */
467
467
  interface A2EBackend {
@@ -1616,7 +1616,9 @@ interface SileroVADBackend {
1616
1616
  *
1617
1617
  * Extends SileroVADConfig with worker-specific options.
1618
1618
  */
1619
- interface SileroVADFactoryConfig extends SileroVADConfig {
1619
+ interface SileroVADFactoryConfig extends Omit<SileroVADConfig, 'modelUrl'> {
1620
+ /** Path or URL to the ONNX model. Default: HuggingFace CDN */
1621
+ modelUrl?: string;
1620
1622
  /**
1621
1623
  * Force worker usage (true), main thread (false), or auto-detect (undefined).
1622
1624
  *
@@ -1689,7 +1691,7 @@ declare function supportsVADWorker(): boolean;
1689
1691
  * const vadMain = createSileroVAD({ modelUrl: '/models/silero-vad.onnx', useWorker: false });
1690
1692
  * ```
1691
1693
  */
1692
- declare function createSileroVAD(config: SileroVADFactoryConfig): SileroVADBackend;
1694
+ declare function createSileroVAD(config?: SileroVADFactoryConfig): SileroVADBackend;
1693
1695
 
1694
1696
  /**
1695
1697
  * Web Worker-based wav2arkit_cpu lip sync inference
@@ -2012,8 +2014,8 @@ interface SenseVoiceBackend {
2012
2014
  * Configuration for the SenseVoice factory
2013
2015
  */
2014
2016
  interface CreateSenseVoiceConfig {
2015
- /** Path or URL to model.int8.onnx (239MB) */
2016
- modelUrl: string;
2017
+ /** Path or URL to model.int8.onnx (239MB). Default: HuggingFace CDN */
2018
+ modelUrl?: string;
2017
2019
  /** Path or URL to tokens.txt vocabulary file (default: sibling of modelUrl) */
2018
2020
  tokensUrl?: string;
2019
2021
  /** Language hint (default: 'auto') */
@@ -2040,7 +2042,7 @@ interface CreateSenseVoiceConfig {
2040
2042
  * @param config - Factory configuration
2041
2043
  * @returns A SenseVoiceBackend instance (either Worker or main thread)
2042
2044
  */
2043
- declare function createSenseVoice(config: CreateSenseVoiceConfig): SenseVoiceBackend;
2045
+ declare function createSenseVoice(config?: CreateSenseVoiceConfig): SenseVoiceBackend;
2044
2046
 
2045
2047
  /**
2046
2048
  * Shared blendshape constants and utilities for lip sync inference
@@ -2075,12 +2077,10 @@ declare const ARKIT_BLENDSHAPES: readonly ["browDownLeft", "browDownRight", "bro
2075
2077
  declare function lerpBlendshapes(current: Float32Array | number[], target: Float32Array | number[], factor?: number): number[];
2076
2078
 
2077
2079
  /**
2078
- * Unified Wav2Vec2 inference engine for Audio-to-Expression + ASR
2080
+ * Wav2Vec2 inference engine for Audio-to-Expression (A2E)
2079
2081
  *
2080
2082
  * Runs entirely in the browser using WebGPU or WASM.
2081
- * Takes raw 16kHz audio and outputs:
2082
- * - 52 ARKit blendshapes (lip sync)
2083
- * - 32-token CTC logits (speech recognition)
2083
+ * Takes raw 16kHz audio and outputs 52 ARKit blendshapes for lip sync.
2084
2084
  *
2085
2085
  * @category Inference
2086
2086
  *
@@ -2088,14 +2088,12 @@ declare function lerpBlendshapes(current: Float32Array | number[], target: Float
2088
2088
  * ```typescript
2089
2089
  * import { Wav2Vec2Inference } from '@omote/core';
2090
2090
  *
2091
- * const wav2vec = new Wav2Vec2Inference({ modelUrl: '/models/unified_wav2vec2_asr_a2e.onnx' });
2091
+ * const wav2vec = new Wav2Vec2Inference({ modelUrl: '/models/model.onnx' });
2092
2092
  * await wav2vec.load();
2093
2093
  *
2094
2094
  * // Process 1 second of audio (16kHz = 16000 samples)
2095
2095
  * const result = await wav2vec.infer(audioSamples);
2096
- *
2097
2096
  * console.log('Blendshapes:', result.blendshapes); // [30, 52] for 30fps
2098
- * console.log('ASR text:', result.text); // Decoded transcription
2099
2097
  * ```
2100
2098
  */
2101
2099
 
@@ -2128,21 +2126,16 @@ interface ModelInfo {
2128
2126
  outputNames: string[];
2129
2127
  }
2130
2128
 
2131
- /** CTC vocabulary (32 tokens from wav2vec2-base-960h) */
2129
+ /**
2130
+ * CTC vocabulary (32 tokens from wav2vec2-base-960h)
2131
+ * @deprecated ASR is handled by SenseVoice. This will be removed in a future release.
2132
+ */
2132
2133
  declare const CTC_VOCAB: string[];
2133
2134
  interface Wav2Vec2Result {
2134
2135
  /** Blendshape weights [frames, 52] - 30fps */
2135
2136
  blendshapes: Float32Array[];
2136
- /** Raw CTC logits [frames, 32] - 50fps */
2137
- asrLogits: Float32Array[];
2138
- /** Decoded text from CTC */
2139
- text: string;
2140
- /** Number of blendshape frames (30fps) — alias for numA2EFrames */
2137
+ /** Number of blendshape frames (30fps) */
2141
2138
  numFrames: number;
2142
- /** Number of A2E frames (30fps) */
2143
- numA2EFrames: number;
2144
- /** Number of ASR frames (50fps) */
2145
- numASRFrames: number;
2146
2139
  /** Inference time in ms */
2147
2140
  inferenceTimeMs: number;
2148
2141
  }
@@ -2180,10 +2173,6 @@ declare class Wav2Vec2Inference implements A2EBackend {
2180
2173
  * Audio will be zero-padded or truncated to chunkSize samples.
2181
2174
  */
2182
2175
  infer(audioSamples: Float32Array, identityIndex?: number): Promise<Wav2Vec2Result>;
2183
- /**
2184
- * Decode CTC logits to text using greedy decoding
2185
- */
2186
- private decodeCTC;
2187
2176
  /**
2188
2177
  * Queue inference to serialize ONNX session calls
2189
2178
  */
@@ -2198,10 +2187,85 @@ declare class Wav2Vec2Inference implements A2EBackend {
2198
2187
  dispose(): Promise<void>;
2199
2188
  }
2200
2189
 
2190
+ /**
2191
+ * Default and user-configurable model URLs for all ONNX models
2192
+ *
2193
+ * Out of the box, models are served from HuggingFace CDN (`/resolve/main/`
2194
+ * endpoint with `Access-Control-Allow-Origin: *`). For production apps that
2195
+ * need faster or more reliable delivery, call {@link configureModelUrls} once
2196
+ * at startup to point any or all models at your own CDN.
2197
+ *
2198
+ * @category Inference
2199
+ *
2200
+ * @example Use HuggingFace defaults (zero-config)
2201
+ * ```typescript
2202
+ * import { createA2E } from '@omote/core';
2203
+ * const a2e = createA2E(); // fetches from HuggingFace CDN
2204
+ * ```
2205
+ *
2206
+ * @example Self-host on your own CDN
2207
+ * ```typescript
2208
+ * import { configureModelUrls, createA2E } from '@omote/core';
2209
+ *
2210
+ * configureModelUrls({
2211
+ * lam: 'https://cdn.example.com/models/model_fp16.onnx',
2212
+ * senseVoice: 'https://cdn.example.com/models/sensevoice.int8.onnx',
2213
+ * // omitted keys keep HuggingFace defaults
2214
+ * });
2215
+ *
2216
+ * const a2e = createA2E(); // now fetches from your CDN
2217
+ * ```
2218
+ */
2219
+ /** Model URL keys that can be configured */
2220
+ type ModelUrlKey = 'lam' | 'wav2arkitCpu' | 'senseVoice' | 'sileroVad';
2221
+ /**
2222
+ * Resolved model URLs — user overrides take priority, HuggingFace CDN is fallback.
2223
+ *
2224
+ * All SDK factories (`createA2E`, `createSenseVoice`, `createSileroVAD`) and
2225
+ * orchestrators (`VoicePipeline`) read from this object. Call
2226
+ * {@link configureModelUrls} before constructing any pipelines to point
2227
+ * models at your own CDN.
2228
+ */
2229
+ declare const DEFAULT_MODEL_URLS: Readonly<Record<ModelUrlKey, string>>;
2230
+ /**
2231
+ * Configure custom model URLs. Overrides persist for the lifetime of the page.
2232
+ * Omitted keys keep their HuggingFace CDN defaults.
2233
+ *
2234
+ * Call this **once** at app startup, before constructing any pipelines.
2235
+ *
2236
+ * @example Self-host all models
2237
+ * ```typescript
2238
+ * configureModelUrls({
2239
+ * lam: 'https://cdn.example.com/models/model_fp16.onnx',
2240
+ * wav2arkitCpu: 'https://cdn.example.com/models/wav2arkit_cpu.onnx',
2241
+ * senseVoice: 'https://cdn.example.com/models/sensevoice.int8.onnx',
2242
+ * sileroVad: 'https://cdn.example.com/models/silero-vad.onnx',
2243
+ * });
2244
+ * ```
2245
+ *
2246
+ * @example Override only one model
2247
+ * ```typescript
2248
+ * configureModelUrls({
2249
+ * lam: '/models/model_fp16.onnx', // self-hosted, same origin
2250
+ * });
2251
+ * ```
2252
+ */
2253
+ declare function configureModelUrls(urls: Partial<Record<ModelUrlKey, string>>): void;
2254
+ /**
2255
+ * Reset all model URL overrides back to HuggingFace CDN defaults.
2256
+ * Mainly useful for testing.
2257
+ */
2258
+ declare function resetModelUrls(): void;
2259
+ /**
2260
+ * Get the immutable HuggingFace CDN URLs (ignoring any overrides).
2261
+ * Useful for documentation or fallback logic.
2262
+ */
2263
+ declare const HF_CDN_URLS: Readonly<Record<ModelUrlKey, string>>;
2264
+
2201
2265
  /**
2202
2266
  * CPU-optimized lip sync inference using wav2arkit_cpu model
2203
2267
  *
2204
- * A Safari/iOS-compatible alternative to Wav2Vec2Inference (384MB) designed
2268
+ * A Safari/iOS-compatible alternative to Wav2Vec2Inference (192MB fp16) designed
2205
2269
  * for platforms where WebGPU crashes due to ONNX Runtime JSEP bugs.
2206
2270
  *
2207
2271
  * The model uses ONNX external data format:
@@ -2288,41 +2352,33 @@ declare class Wav2ArkitCpuInference implements A2EBackend {
2288
2352
  *
2289
2353
  * Provides a unified API that automatically selects the optimal model:
2290
2354
  * - Safari (macOS + iOS): Uses Wav2ArkitCpuInference (404MB, WASM)
2291
- * - Chrome/Firefox/Edge: Uses Wav2Vec2Inference (384MB, WebGPU)
2355
+ * - Chrome/Firefox/Edge: Uses Wav2Vec2Inference (192MB fp16, WebGPU)
2292
2356
  * - Fallback: Gracefully falls back to CPU model if GPU model fails to load
2293
2357
  *
2294
2358
  * Why two separate models?
2295
2359
  * Wav2Vec2 (LAM) cannot run on Safari/iOS for two reasons:
2296
2360
  * 1. Its dual-head transformer graph needs ~750-950MB peak during ORT session
2297
2361
  * creation (graph optimization), exceeding iOS WebKit's ~1-1.5GB tab limit.
2298
- * 2. It ships as a single 384MB .onnx file that must load into JS heap before
2299
- * ORT can consume it. iOS WebKit OOMs on this allocation.
2362
+ * 2. It ships as a single 192MB .onnx file (fp16) that must load into JS heap
2363
+ * before ORT can consume it. iOS WebKit OOMs on this allocation.
2300
2364
  * wav2arkit_cpu solves both: external data format (1.86MB graph + 402MB weights)
2301
2365
  * lets ORT load only the tiny graph, then stream weights via URL pass-through
2302
2366
  * directly into WASM memory. JS heap stays at ~2MB.
2303
2367
  *
2304
2368
  * @category Inference
2305
2369
  *
2306
- * @example Auto-detect (recommended)
2370
+ * @example Auto-detect (recommended, zero-config)
2307
2371
  * ```typescript
2308
2372
  * import { createA2E } from '@omote/core';
2309
2373
  *
2310
- * const a2e = createA2E({
2311
- * gpuModelUrl: '/models/unified_wav2vec2_asr_a2e.onnx',
2312
- * cpuModelUrl: '/models/wav2arkit_cpu.onnx',
2313
- * });
2314
- *
2374
+ * const a2e = createA2E(); // uses HF CDN defaults (192MB fp16 GPU, 404MB CPU fallback)
2315
2375
  * await a2e.load();
2316
2376
  * const { blendshapes } = await a2e.infer(audioSamples);
2317
2377
  * ```
2318
2378
  *
2319
2379
  * @example Force CPU model
2320
2380
  * ```typescript
2321
- * const a2e = createA2E({
2322
- * gpuModelUrl: '/models/unified_wav2vec2_asr_a2e.onnx',
2323
- * cpuModelUrl: '/models/wav2arkit_cpu.onnx',
2324
- * mode: 'cpu',
2325
- * });
2381
+ * const a2e = createA2E({ mode: 'cpu' });
2326
2382
  * ```
2327
2383
  */
2328
2384
 
@@ -2330,8 +2386,8 @@ declare class Wav2ArkitCpuInference implements A2EBackend {
2330
2386
  * Configuration for the A2E factory
2331
2387
  */
2332
2388
  interface CreateA2EConfig {
2333
- /** URL for the GPU model (Wav2Vec2, used on Chrome/Firefox/Edge) */
2334
- gpuModelUrl: string;
2389
+ /** URL for the GPU model (Wav2Vec2, used on Chrome/Firefox/Edge). Default: HuggingFace CDN */
2390
+ gpuModelUrl?: string;
2335
2391
  /**
2336
2392
  * URL for GPU model external data file (.onnx.data weights).
2337
2393
  * Default: `${gpuModelUrl}.data`
@@ -2339,8 +2395,8 @@ interface CreateA2EConfig {
2339
2395
  * Set to `false` to skip external data loading (single-file models only).
2340
2396
  */
2341
2397
  gpuExternalDataUrl?: string | false;
2342
- /** URL for the CPU model (wav2arkit_cpu, used on Safari/iOS) */
2343
- cpuModelUrl: string;
2398
+ /** URL for the CPU model (wav2arkit_cpu, used on Safari/iOS). Default: HuggingFace CDN */
2399
+ cpuModelUrl?: string;
2344
2400
  /**
2345
2401
  * Model selection mode:
2346
2402
  * - 'auto': Safari/iOS -> CPU, everything else -> GPU (default)
@@ -2382,7 +2438,7 @@ interface CreateA2EConfig {
2382
2438
  * @param config - Factory configuration
2383
2439
  * @returns An A2EBackend instance (either GPU or CPU model)
2384
2440
  */
2385
- declare function createA2E(config: CreateA2EConfig): A2EBackend;
2441
+ declare function createA2E(config?: CreateA2EConfig): A2EBackend;
2386
2442
 
2387
2443
  /**
2388
2444
  * A2EProcessor — Engine-agnostic audio-to-expression processor
@@ -4111,10 +4167,12 @@ declare class EmphasisDetector {
4111
4167
  * breathing/postural sway, and simplex noise-driven brow drift.
4112
4168
  *
4113
4169
  * Research sources:
4114
- * - Blink frequency: 15-20/min (every 3-4s), PMC4043155
4170
+ * - Blink frequency: log-normal IBI (mean=5.97s, SD(log)=0.89), PMC3565584
4171
+ * - Blink shape: asymmetric (92ms close, 242ms open, 3:1 ratio), PMC4043155
4115
4172
  * - Saccade latency: ~200ms, duration 20-200ms
4116
4173
  * - Microsaccades: ~1/second, amplitude 0.02-0.05, Scholarpedia
4117
4174
  * - Fixation duration: 200-350ms, Nature Scientific Reports
4175
+ * - Conversational gaze: Kendon (1967), Argyle & Cook (1976)
4118
4176
  * - Brow noise: NVIDIA Audio2Face, Unreal MetaHuman layered procedural animation
4119
4177
  *
4120
4178
  * @category Animation
@@ -4131,6 +4189,7 @@ declare class EmphasisDetector {
4131
4189
  * eyeTargetY: normalizedY,
4132
4190
  * audioEnergy: energy, // 0-1 from AudioEnergyAnalyzer
4133
4191
  * isSpeaking: true,
4192
+ * state: 'speaking', // conversational state for gaze behavior
4134
4193
  * });
4135
4194
  *
4136
4195
  * // Apply blendshapes to mesh
@@ -4169,6 +4228,8 @@ interface LifeLayerConfig {
4169
4228
  /** Eye smoothing factor (higher = faster response). Default: 15 */
4170
4229
  eyeSmoothing?: number;
4171
4230
  }
4231
+ /** Conversational state for state-dependent gaze behavior */
4232
+ type ConversationalState = 'idle' | 'listening' | 'thinking' | 'speaking';
4172
4233
  /**
4173
4234
  * Per-frame input to the life layer
4174
4235
  */
@@ -4181,6 +4242,8 @@ interface LifeLayerInput {
4181
4242
  audioEnergy?: number;
4182
4243
  /** Whether avatar is speaking. Multiplies brow noise amplitude. */
4183
4244
  isSpeaking?: boolean;
4245
+ /** Conversational state for gaze behavior (idle/listening/thinking/speaking) */
4246
+ state?: ConversationalState;
4184
4247
  }
4185
4248
  /**
4186
4249
  * Per-frame output from the life layer
@@ -4202,6 +4265,7 @@ interface LifeLayerOutput {
4202
4265
  */
4203
4266
  declare class ProceduralLifeLayer {
4204
4267
  private blinkIntervalRange;
4268
+ private useLogNormalBlinks;
4205
4269
  private gazeBreakIntervalRange;
4206
4270
  private gazeBreakAmplitudeRange;
4207
4271
  private eyeNoiseAmplitude;
@@ -4229,6 +4293,7 @@ declare class ProceduralLifeLayer {
4229
4293
  private gazeBreakTargetY;
4230
4294
  private gazeBreakCurrentX;
4231
4295
  private gazeBreakCurrentY;
4296
+ private currentState;
4232
4297
  private microMotionTime;
4233
4298
  private breathingPhase;
4234
4299
  private noiseTime;
@@ -4243,17 +4308,230 @@ declare class ProceduralLifeLayer {
4243
4308
  * @returns Blendshape values and head rotation deltas
4244
4309
  */
4245
4310
  update(delta: number, input?: LifeLayerInput): LifeLayerOutput;
4311
+ /**
4312
+ * Write life layer output directly to a Float32Array[52] in LAM_BLENDSHAPES order.
4313
+ *
4314
+ * Includes micro-jitter (0.4% amplitude simplex noise on all channels) to
4315
+ * break uncanny stillness on undriven channels.
4316
+ *
4317
+ * @param delta - Time since last frame in seconds
4318
+ * @param input - Per-frame input
4319
+ * @param out - Pre-allocated Float32Array(52) to write into
4320
+ */
4321
+ updateToArray(delta: number, input: LifeLayerInput, out: Float32Array): void;
4246
4322
  /**
4247
4323
  * Reset all internal state to initial values.
4248
4324
  */
4249
4325
  reset(): void;
4326
+ /**
4327
+ * Sample next blink interval.
4328
+ * Uses log-normal distribution (PMC3565584) when using default config,
4329
+ * or uniform random when custom blinkIntervalRange is provided.
4330
+ */
4331
+ private nextBlinkInterval;
4250
4332
  private updateBlinks;
4251
4333
  private getBlinkValues;
4252
4334
  private getEyeMicroMotion;
4335
+ /**
4336
+ * Get active gaze parameters — uses state-dependent params when
4337
+ * conversational state is provided, otherwise falls back to config ranges.
4338
+ */
4339
+ private getActiveGazeParams;
4253
4340
  private updateGazeBreaks;
4254
4341
  private updateBrowNoise;
4255
4342
  }
4256
4343
 
4344
+ /**
4345
+ * FACS (Facial Action Coding System) to ARKit Blendshape Mapping
4346
+ *
4347
+ * Two static lookup tables that decompose emotions into FACS Action Units,
4348
+ * then map AUs to ARKit blendshapes. Based on Ekman's FACS research.
4349
+ *
4350
+ * @category Face
4351
+ */
4352
+
4353
+ /**
4354
+ * A single FACS Action Unit activation within an emotion
4355
+ */
4356
+ interface AUActivation {
4357
+ /** FACS Action Unit identifier (e.g. 'AU6', 'AU12') */
4358
+ au: string;
4359
+ /** Activation intensity 0-1 */
4360
+ intensity: number;
4361
+ /** Facial region: upper (brows/eyes/cheeks) or lower (mouth/jaw) */
4362
+ region: 'upper' | 'lower';
4363
+ }
4364
+ /**
4365
+ * Table 1: Emotion → FACS Action Units
4366
+ *
4367
+ * Maps each of the 10 SDK emotion channels to their FACS AU combinations
4368
+ * with intensity and upper/lower face region tags.
4369
+ *
4370
+ * Sources:
4371
+ * - Ekman & Friesen (1978) FACS Manual
4372
+ * - Ekman (2003) Emotions Revealed
4373
+ * - Lucey et al. (2010) Extended Cohn-Kanade dataset
4374
+ */
4375
+ declare const EMOTION_TO_AU: Record<EmotionName, AUActivation[]>;
4376
+ /**
4377
+ * Table 2: FACS Action Unit → ARKit Blendshapes
4378
+ *
4379
+ * Maps each AU to one or more ARKit blendshape channels with weight.
4380
+ *
4381
+ * Sources:
4382
+ * - Apple ARKit face tracking documentation
4383
+ * - Melinda Ozel's ARKit-to-FACS cheat sheet
4384
+ */
4385
+ declare const AU_TO_ARKIT: Record<string, {
4386
+ blendshape: string;
4387
+ weight: number;
4388
+ }[]>;
4389
+ /**
4390
+ * All AU identifiers referenced by EMOTION_TO_AU (for validation)
4391
+ */
4392
+ declare const ALL_AUS: string[];
4393
+
4394
+ /**
4395
+ * EmotionResolver — Resolves EmotionWeights → split upper/lower face Float32Array[52]
4396
+ *
4397
+ * Uses FACS decomposition (EMOTION_TO_AU → AU_TO_ARKIT) to produce
4398
+ * anatomically correct blendshape contributions, split by facial region
4399
+ * for the FaceCompositor's modulation strategy:
4400
+ * - Upper face: additive overlay (independent of speech)
4401
+ * - Lower face: modulates speech output
4402
+ *
4403
+ * @category Face
4404
+ */
4405
+
4406
+ /**
4407
+ * Resolved emotion split into upper and lower face contributions
4408
+ */
4409
+ interface ResolvedEmotion {
4410
+ /** 52 channels — only upper face (brows, eyes, cheeks, nose) non-zero */
4411
+ upper: Float32Array;
4412
+ /** 52 channels — only lower face (mouth, jaw) non-zero */
4413
+ lower: Float32Array;
4414
+ }
4415
+ /**
4416
+ * Resolves EmotionWeights into upper/lower face blendshape arrays
4417
+ * using FACS Action Unit decomposition.
4418
+ */
4419
+ declare class EmotionResolver {
4420
+ private readonly upperBuffer;
4421
+ private readonly lowerBuffer;
4422
+ /**
4423
+ * Resolve emotion weights to upper/lower face blendshape contributions.
4424
+ *
4425
+ * @param weights - Emotion channel weights from EmotionController
4426
+ * @param intensity - Global intensity multiplier (0-2). Default: 1.0
4427
+ * @returns Upper and lower face blendshape arrays (52 channels each)
4428
+ */
4429
+ resolve(weights: EmotionWeights, intensity?: number): ResolvedEmotion;
4430
+ }
4431
+
4432
+ /**
4433
+ * FaceCompositor — 5-stage signal processing chain for facial animation
4434
+ *
4435
+ * Composes A2E lip sync, emotion modulation, procedural life, and character
4436
+ * profile into a single Float32Array[52] per frame.
4437
+ *
4438
+ * ```
4439
+ * BASE (A2E) → EMOTION MODULATION → PROCEDURAL LIFE → CHARACTER PROFILE → OUTPUT [0,1]
4440
+ * ```
4441
+ *
4442
+ * Replaces manual blendshape merging in consumer code with a single `compose()` call.
4443
+ *
4444
+ * @category Face
4445
+ */
4446
+
4447
+ /**
4448
+ * Per-blendshape character profile (multiplier + offset)
4449
+ *
4450
+ * Superset of ExpressionProfile — gives per-channel control instead of per-group.
4451
+ */
4452
+ interface CharacterProfile {
4453
+ /** Per-blendshape multiplier (default: all 1.0) */
4454
+ multiplier?: Partial<Record<string, number>>;
4455
+ /** Per-blendshape offset (default: all 0.0) */
4456
+ offset?: Partial<Record<string, number>>;
4457
+ }
4458
+ /**
4459
+ * Configuration for FaceCompositor
4460
+ */
4461
+ interface FaceCompositorConfig {
4462
+ /** ProceduralLifeLayer instance (compositor creates default if omitted) */
4463
+ lifeLayer?: ProceduralLifeLayer;
4464
+ /** Character profile: per-BS multiplier + offset */
4465
+ profile?: CharacterProfile;
4466
+ /** Emotion smoothing factor per frame (0-1). Default: 0.12 */
4467
+ emotionSmoothing?: number;
4468
+ }
4469
+ /**
4470
+ * Per-frame input to the compositor
4471
+ */
4472
+ interface FaceCompositorInput extends LifeLayerInput {
4473
+ /** Delta time in seconds */
4474
+ deltaTime: number;
4475
+ /** Current emotion weights (from EmotionController.emotion or manual) */
4476
+ emotion?: EmotionWeights;
4477
+ /** Emotion intensity multiplier (0-2). Default: 1.0 */
4478
+ emotionIntensity?: number;
4479
+ }
4480
+ /**
4481
+ * FaceCompositor — 5-stage facial animation signal chain.
4482
+ *
4483
+ * @example
4484
+ * ```typescript
4485
+ * import { FaceCompositor, createA2E } from '@omote/core';
4486
+ *
4487
+ * const compositor = new FaceCompositor();
4488
+ *
4489
+ * // In animation loop:
4490
+ * const output = compositor.compose(a2eFrame, {
4491
+ * deltaTime: 0.016,
4492
+ * emotion: { joy: 0.8 },
4493
+ * isSpeaking: true,
4494
+ * audioEnergy: 0.5,
4495
+ * });
4496
+ *
4497
+ * // Apply output[0..51] to avatar morphTargetInfluences
4498
+ * ```
4499
+ */
4500
+ declare class FaceCompositor {
4501
+ private readonly emotionResolver;
4502
+ private readonly lifeLayer;
4503
+ private readonly emotionSmoothing;
4504
+ private readonly smoothedUpper;
4505
+ private readonly smoothedLower;
4506
+ private readonly lifeBuffer;
4507
+ private readonly multiplier;
4508
+ private readonly offset;
4509
+ private stickyEmotion;
4510
+ constructor(config?: FaceCompositorConfig);
4511
+ /**
4512
+ * Compose a single output frame from the 5-stage signal chain.
4513
+ *
4514
+ * @param base - A2E raw output (Float32Array[52], LAM_BLENDSHAPES order)
4515
+ * @param input - Per-frame input (deltaTime, emotion, life layer params)
4516
+ * @returns Float32Array[52] with all values clamped to [0, 1]
4517
+ */
4518
+ compose(base: Float32Array, input: FaceCompositorInput): Float32Array;
4519
+ /**
4520
+ * Set sticky emotion (used when input.emotion is not provided).
4521
+ */
4522
+ setEmotion(weights: EmotionWeights): void;
4523
+ /**
4524
+ * Update character profile at runtime.
4525
+ */
4526
+ setProfile(profile: CharacterProfile): void;
4527
+ /**
4528
+ * Reset all smoothing state and life layer.
4529
+ */
4530
+ reset(): void;
4531
+ /** Expand partial profile maps into dense Float32Arrays */
4532
+ private applyProfileArrays;
4533
+ }
4534
+
4257
4535
  /**
4258
4536
  * MicLipSync - Microphone → VAD → A2E → blendshapes
4259
4537
  *
@@ -4539,4 +4817,4 @@ declare class VoicePipeline extends EventEmitter<VoicePipelineEvents> {
4539
4817
  private clearSilenceTimer;
4540
4818
  }
4541
4819
 
4542
- export { type A2EBackend, type A2EModelInfo, A2EOrchestrator, type A2EOrchestratorConfig, A2EProcessor, type A2EProcessorConfig, type A2EProgressEvent, type A2EResult, ARKIT_BLENDSHAPES, type ActiveSpan, type AnimationClip, AnimationGraph, type AnimationGraphConfig, type AnimationGraphEvents, type AnimationLayer, type AnimationOutput, type AnimationState, type AnimationStateName, type AnimationTrigger, AudioChunkCoalescer, type AudioChunkCoalescerOptions, AudioEnergyAnalyzer, AudioScheduler, type AudioSchedulerOptions, BLENDSHAPE_TO_GROUP, type BackendPreference, type BlendWeight, type BlendshapeGroup, BlendshapeSmoother, type BlendshapeSmootherConfig, CTC_VOCAB, type CacheConfig, type CacheSpanAttributes, ConsoleExporter, type CreateA2EConfig, type CreateSenseVoiceConfig, DEFAULT_ANIMATION_CONFIG, EMOTION_NAMES, EMOTION_VECTOR_SIZE, type EmotionAnimationMap, EmotionController, type EmotionLabel, type EmotionName, type EmotionPresetName, EmotionPresets, type EmotionWeights, EmphasisDetector, EventEmitter, type ExpressionProfile, type FetchWithCacheOptions, type FullFaceFrame, FullFacePipeline, type FullFacePipelineEvents, type FullFacePipelineOptions, INFERENCE_LATENCY_BUCKETS, type InferenceSpanAttributes, type InterruptionConfig, type InterruptionEvents, InterruptionHandler, LAM_BLENDSHAPES, type LifeLayerConfig, type LifeLayerInput, type LifeLayerOutput, type LoadingProgress, MODEL_LOAD_TIME_BUCKETS, type MetricData, MetricNames, MicLipSync, type MicLipSyncConfig, type MicLipSyncEvents, type MicLipSyncFrame, type MicLipSyncState, MicrophoneCapture, type MicrophoneCaptureConfig, ModelCache, type ModelSpanAttributes, OTLPExporter, type OTLPExporterConfig, OmoteEvents, OmoteTelemetry, PlaybackPipeline, type PlaybackPipelineConfig, type PlaybackPipelineEvents, type PlaybackState, ProceduralLifeLayer, type QuotaInfo, type ResponseHandler, RingBuffer, type RuntimeBackend, type SafariSpeechConfig, SafariSpeechRecognition, type SamplingConfig, type SenseVoiceBackend, type SenseVoiceConfig, SenseVoiceInference, type SenseVoiceLanguage, type SenseVoiceModelInfo, type SenseVoiceResult, SenseVoiceUnifiedAdapter, SenseVoiceWorker, type SenseVoiceWorkerConfig, type SileroVADBackend, type SileroVADConfig, type SileroVADFactoryConfig, SileroVADInference, SileroVADUnifiedAdapter, SileroVADWorker, type SpanAttributes, type SpanData, type SpeechErrorCallback, type SpeechRecognitionResult, type SpeechResultCallback, type SpeechSegment, type TelemetryConfig, type TelemetryExporter, type TelemetryExporterInterface, type TranscriptResult, type Transition, UnifiedInferenceWorker, type VADBackend, type VADModelInfo, type VADResult, type VADWorkerConfig, type VADWorkerModelInfo, type ValidationResult, VoicePipeline, type VoicePipelineConfig, type VoicePipelineEvents, type VoicePipelineState, type Wav2ArkitCpuConfig, Wav2ArkitCpuInference, Wav2ArkitCpuUnifiedAdapter, Wav2ArkitCpuWorker, type Wav2ArkitCpuWorkerConfig, Wav2Vec2Inference, type Wav2Vec2InferenceConfig, type Wav2Vec2Result, applyProfile, blendEmotions, calculatePeak, calculateRMS, configureCacheLimit, configureTelemetry, createA2E, createEmotionVector, createSenseVoice, createSileroVAD, fetchWithCache, formatBytes, getCacheConfig, getCacheKey, getEmotionPreset, getModelCache, getOptimalWasmThreads, getRecommendedBackend, getTelemetry, hasWebGPUApi, isAndroid, isIOS, isIOSSafari, isMobile, isSafari, isSpeechRecognitionAvailable, isWebGPUAvailable, lerpBlendshapes, lerpEmotion, preloadModels, resolveBackend, shouldEnableWasmProxy, shouldUseCpuA2E, shouldUseNativeASR, shouldUseServerA2E, supportsVADWorker };
4820
+ export { type A2EBackend, type A2EModelInfo, A2EOrchestrator, type A2EOrchestratorConfig, A2EProcessor, type A2EProcessorConfig, type A2EProgressEvent, type A2EResult, ALL_AUS, ARKIT_BLENDSHAPES, type AUActivation, AU_TO_ARKIT, type ActiveSpan, type AnimationClip, AnimationGraph, type AnimationGraphConfig, type AnimationGraphEvents, type AnimationLayer, type AnimationOutput, type AnimationState, type AnimationStateName, type AnimationTrigger, AudioChunkCoalescer, type AudioChunkCoalescerOptions, AudioEnergyAnalyzer, AudioScheduler, type AudioSchedulerOptions, BLENDSHAPE_TO_GROUP, type BackendPreference, type BlendWeight, type BlendshapeGroup, BlendshapeSmoother, type BlendshapeSmootherConfig, CTC_VOCAB, type CacheConfig, type CacheSpanAttributes, type CharacterProfile, ConsoleExporter, type ConversationalState, type CreateA2EConfig, type CreateSenseVoiceConfig, DEFAULT_ANIMATION_CONFIG, DEFAULT_MODEL_URLS, EMOTION_NAMES, EMOTION_TO_AU, EMOTION_VECTOR_SIZE, type EmotionAnimationMap, EmotionController, type EmotionLabel, type EmotionName, type EmotionPresetName, EmotionPresets, EmotionResolver, type EmotionWeights, EmphasisDetector, EventEmitter, type ExpressionProfile, FaceCompositor, type FaceCompositorConfig, type FaceCompositorInput, type FetchWithCacheOptions, type FullFaceFrame, FullFacePipeline, type FullFacePipelineEvents, type FullFacePipelineOptions, HF_CDN_URLS, INFERENCE_LATENCY_BUCKETS, type InferenceSpanAttributes, type InterruptionConfig, type InterruptionEvents, InterruptionHandler, LAM_BLENDSHAPES, type LifeLayerConfig, type LifeLayerInput, type LifeLayerOutput, type LoadingProgress, MODEL_LOAD_TIME_BUCKETS, type MetricData, MetricNames, MicLipSync, type MicLipSyncConfig, type MicLipSyncEvents, type MicLipSyncFrame, type MicLipSyncState, MicrophoneCapture, type MicrophoneCaptureConfig, ModelCache, type ModelSpanAttributes, type ModelUrlKey, OTLPExporter, type OTLPExporterConfig, OmoteEvents, OmoteTelemetry, PlaybackPipeline, type PlaybackPipelineConfig, type PlaybackPipelineEvents, type PlaybackState, ProceduralLifeLayer, type QuotaInfo, type ResolvedEmotion, type ResponseHandler, RingBuffer, type RuntimeBackend, type SafariSpeechConfig, SafariSpeechRecognition, type SamplingConfig, type SenseVoiceBackend, type SenseVoiceConfig, SenseVoiceInference, type SenseVoiceLanguage, type SenseVoiceModelInfo, type SenseVoiceResult, SenseVoiceUnifiedAdapter, SenseVoiceWorker, type SenseVoiceWorkerConfig, type SileroVADBackend, type SileroVADConfig, type SileroVADFactoryConfig, SileroVADInference, SileroVADUnifiedAdapter, SileroVADWorker, type SpanAttributes, type SpanData, type SpeechErrorCallback, type SpeechRecognitionResult, type SpeechResultCallback, type SpeechSegment, type TelemetryConfig, type TelemetryExporter, type TelemetryExporterInterface, type TranscriptResult, type Transition, UnifiedInferenceWorker, type VADBackend, type VADModelInfo, type VADResult, type VADWorkerConfig, type VADWorkerModelInfo, type ValidationResult, VoicePipeline, type VoicePipelineConfig, type VoicePipelineEvents, type VoicePipelineState, type Wav2ArkitCpuConfig, Wav2ArkitCpuInference, Wav2ArkitCpuUnifiedAdapter, Wav2ArkitCpuWorker, type Wav2ArkitCpuWorkerConfig, Wav2Vec2Inference, type Wav2Vec2InferenceConfig, type Wav2Vec2Result, applyProfile, blendEmotions, calculatePeak, calculateRMS, configureCacheLimit, configureModelUrls, configureTelemetry, createA2E, createEmotionVector, createSenseVoice, createSileroVAD, fetchWithCache, formatBytes, getCacheConfig, getCacheKey, getEmotionPreset, getModelCache, getOptimalWasmThreads, getRecommendedBackend, getTelemetry, hasWebGPUApi, isAndroid, isIOS, isIOSSafari, isMobile, isSafari, isSpeechRecognitionAvailable, isWebGPUAvailable, lerpBlendshapes, lerpEmotion, preloadModels, resetModelUrls, resolveBackend, shouldEnableWasmProxy, shouldUseCpuA2E, shouldUseNativeASR, shouldUseServerA2E, supportsVADWorker };