@omote/core 0.6.6 → 0.9.1

package/README.md

@@ -4,11 +4,18 @@
 
 ## Features
 
-- **Lip Sync (A2E)** — Audio to 52 ARKit blendshapes via Wav2Vec2, with automatic GPU/CPU platform detection
-- **Full-Face Pipeline** — TTS audio playback to lip sync with ExpressionProfile scaling, gapless scheduling
+- **Lip Sync (A2E)** — Audio to 52 ARKit blendshapes via LAM, with automatic WebGPU/WASM platform detection
+- **PlaybackPipeline** — TTS audio playback to lip sync with ExpressionProfile scaling, gapless scheduling
 - **Speech Recognition** — SenseVoice ASR (ONNX), 15x faster than Whisper, progressive transcription
 - **Voice Activity Detection** — Silero VAD with Worker and main-thread modes
-- **Text-to-Speech** — ChatterboxTurbo (experimental, use server-side TTS for production)
+- **Text-to-Speech** — Kokoro TTS (82M q8, offline) with TTSBackend interface for custom engines
+- **CharacterController** — Renderer-agnostic avatar composition (compositor + gaze + life layer)
+- **TTSPlayback** — Composes TTSBackend + PlaybackPipeline for text → lip sync
+- **TTSSpeaker** — High-level speak(text) with abort, queueing, and LLM streaming
+- **SpeechListener** — Mic → VAD → ASR orchestration with adaptive silence detection
+- **createTTSPlayer()** — Factory composing Kokoro TTS + TTSSpeaker for zero-config playback
+- **VoicePipeline** — Full conversational agent loop with local TTS support (cloud or offline)
+- **configureOrtCdn()** — Enterprise CDN override for ORT WASM/WebGPU binaries
 - **Animation Graph** — State machine (idle/listening/thinking/speaking) with emotion blending
 - **Emotion Controller** — Preset-based emotion system with smooth transitions
 - **Model Caching** — IndexedDB with versioning, LRU eviction, and quota monitoring
@@ -27,34 +34,33 @@ Peer dependency: `onnxruntime-web` is included — no additional installs needed
 
 ## Quick Start
 
-### FullFacePipeline (TTS Lip Sync)
+### PlaybackPipeline (TTS Lip Sync)
 
 The most common use case: feed TTS audio chunks and get back 52 ARKit blendshape frames at render rate.
 
 ```typescript
-import { FullFacePipeline, createA2E } from '@omote/core';
+import { PlaybackPipeline, createA2E } from '@omote/core';
 
 // 1. Create A2E backend (auto-detects GPU vs CPU)
 const lam = createA2E(); // auto-detects GPU vs CPU, fetches from HF CDN (192MB fp16)
 await lam.load();
 
 // 2. Create pipeline with expression profile
-const pipeline = new FullFacePipeline({
+const pipeline = new PlaybackPipeline({
   lam,
   sampleRate: 16000,
   profile: { mouth: 1.0, jaw: 1.0, brows: 0.6, eyes: 0.0, cheeks: 0.5, nose: 0.3, tongue: 0.5 },
 });
-await pipeline.initialize();
 
 // 3. Listen for blendshape frames
-pipeline.on('full_frame_ready', (frame) => {
+pipeline.on('frame', (frame) => {
   applyToAvatar(frame.blendshapes); // ExpressionProfile-scaled, 52 ARKit weights
 });
 
 // 4. Feed TTS audio and play
 pipeline.start();
-await pipeline.onAudioChunk(ttsAudioChunk); // Uint8Array PCM16
-await pipeline.end(); // Flush remaining audio
+pipeline.feedBuffer(ttsAudioChunk); // Uint8Array PCM16
+pipeline.end(); // Flush remaining audio
 ```
 
 ## API Reference
@@ -68,7 +74,7 @@ Auto-detects platform: Chrome/Edge/Android use WebGPU, Safari/iOS use WASM CPU f
 ```typescript
 import { createA2E } from '@omote/core';
 
-const a2e = createA2E(); // auto-detects: GPU (192MB fp16) or CPU (404MB WASM)
+const a2e = createA2E(); // auto-detects: WebGPU on Chrome/Edge, WASM on Safari/iOS/Firefox
 await a2e.load();
 
 const { blendshapes } = await a2e.infer(audioSamples); // Float32Array (16kHz)
@@ -78,38 +84,37 @@ const { blendshapes } = await a2e.infer(audioSamples); // Float32Array (16kHz)
 #### Direct API
 
 ```typescript
-import { Wav2Vec2Inference, LAM_BLENDSHAPES } from '@omote/core';
+import { A2EInference, ARKIT_BLENDSHAPES } from '@omote/core';
 
-const lam = new Wav2Vec2Inference({ modelUrl: '/models/model_fp16.onnx' });
+const lam = new A2EInference({ modelUrl: '/models/model_fp16.onnx' });
 await lam.load();
 
 const { blendshapes } = await lam.infer(audioSamples);
-const jawOpen = blendshapes[LAM_BLENDSHAPES.indexOf('jawOpen')];
+const jawOpen = blendshapes[ARKIT_BLENDSHAPES.indexOf('jawOpen')];
 ```
 
-### FullFacePipeline
+### PlaybackPipeline
 
 End-to-end TTS playback with lip sync inference, audio scheduling, and ExpressionProfile scaling.
 
 ```typescript
-import { FullFacePipeline } from '@omote/core';
+import { PlaybackPipeline } from '@omote/core';
 
-const pipeline = new FullFacePipeline({
+const pipeline = new PlaybackPipeline({
   lam,                // A2E backend from createA2E()
   sampleRate: 16000,
   profile: { mouth: 1.0, jaw: 1.0, brows: 0.6, eyes: 0.0, cheeks: 0.5, nose: 0.3, tongue: 0.5 },
 });
-await pipeline.initialize();
 
-pipeline.on('full_frame_ready', (frame) => {
+pipeline.on('frame', (frame) => {
   // frame.blendshapes    — ExpressionProfile-scaled
   // frame.rawBlendshapes — unscaled original values
   applyToAvatar(frame.blendshapes);
 });
 
 pipeline.start();
-await pipeline.onAudioChunk(chunk); // feed TTS audio (Uint8Array PCM16)
-await pipeline.end();               // flush final partial chunk
+pipeline.feedBuffer(chunk); // feed TTS audio (Uint8Array PCM16)
+pipeline.end();             // flush final partial chunk
 ```
 
 ### A2EProcessor
@@ -309,8 +314,6 @@ Place models in your public assets directory:
 ```
 public/models/
   model_fp16.onnx                 # A2E lip sync — WebGPU (192MB fp16, from omote-ai/lam-a2e)
-  wav2arkit_cpu.onnx              # A2E lip sync — WASM fallback (1.86MB graph)
-  wav2arkit_cpu.onnx.data         # A2E lip sync — WASM fallback (402MB weights)
   sensevoice/model.int8.onnx      # SenseVoice ASR (239MB)
   silero-vad.onnx                 # Voice activity detection (~2MB)
 ```
@@ -338,7 +341,7 @@ const webgpu = await isWebGPUAvailable();
 All iOS browsers use WebKit under the hood. The SDK handles three platform constraints automatically:
 
 1. **WASM binary selection** — iOS crashes with the default JSEP/ASYNCIFY WASM binary. The SDK imports `onnxruntime-web/wasm` (non-JSEP) on iOS/Safari.
-2. **A2E model fallback** — The Wav2Vec2 GPU model exceeds iOS memory limits. `createA2E({ mode: 'auto' })` automatically selects the `wav2arkit_cpu` model on iOS.
+2. **A2E model routing** — `createA2E()` routes all platforms through `A2EInference` via `UnifiedInferenceWorker`. WebGPU on Chrome/Edge, WASM on Safari/iOS/Firefox.
 3. **Worker memory** — Multiple Workers each load their own ORT WASM runtime, exceeding iOS tab memory (~1.5GB). The SDK defaults to main-thread inference on iOS.
 
 **Consumer requirement:** COEP/COOP headers must be skipped for iOS to avoid triggering SharedArrayBuffer (which forces threaded WASM with 4GB shared memory — crashes iOS). Desktop should keep COEP/COOP for multi-threaded performance.
@@ -347,7 +350,7 @@ All iOS browsers use WebKit under the hood. The SDK handles three platform const
 |---------|------------|-------|
 | Silero VAD | Works | 0.9ms latency |
 | SenseVoice ASR | Works | WASM, ~200ms |
-| A2E Lip Sync | Works | wav2arkit_cpu via createA2E auto-detect, ~45ms |
+| A2E Lip Sync | Works | A2EInference (WASM) via createA2E(), ~45ms |
 
 ## License
 

package/dist/Logger-BeUI6jG7.d.mts

@@ -0,0 +1,145 @@
+/**
+ * Logging types for Omote SDK
+ *
+ * 6-level logging system with structured output:
+ * - error: Critical failures that prevent operation
+ * - warn: Recoverable issues or degraded performance
+ * - info: Key lifecycle events (model loaded, inference complete)
+ * - debug: Detailed operational info for development
+ * - trace: Fine-grained tracing for performance analysis
+ * - verbose: Extremely detailed output (tensor shapes, intermediate values)
+ */
+type LogLevel = 'error' | 'warn' | 'info' | 'debug' | 'trace' | 'verbose';
+/**
+ * Numeric priority for log levels (lower = more severe)
+ */
+declare const LOG_LEVEL_PRIORITY: Record<LogLevel, number>;
+/**
+ * Structured log entry
+ */
+interface LogEntry {
+    /** Unix timestamp in milliseconds */
+    timestamp: number;
+    /** Log level */
+    level: LogLevel;
+    /** Module name (e.g., 'LocalInference', 'ModelCache') */
+    module: string;
+    /** Human-readable message */
+    message: string;
+    /** Optional structured data */
+    data?: Record<string, unknown>;
+    /** Optional error object */
+    error?: Error;
+    /** Trace ID from active telemetry span (log-to-span correlation) */
+    traceId?: string;
+    /** Span ID from active telemetry span (log-to-span correlation) */
+    spanId?: string;
+}
+/**
+ * Log output sink interface
+ */
+interface LogSink {
+    (entry: LogEntry): void;
+}
+/**
+ * Log formatter interface
+ */
+interface LogFormatter {
+    (entry: LogEntry): string;
+}
+/**
+ * Global logging configuration
+ */
+interface LoggingConfig {
+    /** Minimum log level to output (default: 'info') */
+    level: LogLevel;
+    /** Enable/disable logging globally (default: true) */
+    enabled: boolean;
+    /** Output format: 'json' for structured, 'pretty' for human-readable */
+    format: 'json' | 'pretty';
+    /** Custom output sink (default: console) */
+    sink?: LogSink;
+    /** Include timestamps in output (default: true) */
+    timestamps?: boolean;
+    /** Include module name in output (default: true) */
+    includeModule?: boolean;
+}
+/**
+ * Logger interface for module-specific logging
+ */
+interface ILogger {
+    error(message: string, data?: Record<string, unknown>): void;
+    warn(message: string, data?: Record<string, unknown>): void;
+    info(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>): void;
+    trace(message: string, data?: Record<string, unknown>): void;
+    verbose(message: string, data?: Record<string, unknown>): void;
+    /** Create a child logger with a sub-module name */
+    child(subModule: string): ILogger;
+    /** Get the module name for this logger */
+    readonly module: string;
+}
+/**
+ * Default configuration
+ */
+declare const DEFAULT_LOGGING_CONFIG: LoggingConfig;
+
+/**
+ * Omote SDK Logger
+ *
+ * Unified logging system with:
+ * - 6 log levels (error, warn, info, debug, trace, verbose)
+ * - Structured JSON output for machine parsing
+ * - Pretty output for human readability
+ * - Module-based child loggers
+ * - Runtime configuration
+ * - Browser and Node.js compatible
+ */
+
+/**
+ * Configure global logging settings
+ */
+declare function configureLogging(config: Partial<LoggingConfig>): void;
+/**
+ * Get current logging configuration
+ */
+declare function getLoggingConfig(): LoggingConfig;
+/**
+ * Reset logging configuration to defaults
+ */
+declare function resetLoggingConfig(): void;
+/**
+ * Set log level at runtime
+ */
+declare function setLogLevel(level: LogLevel): void;
+/**
+ * Enable or disable logging
+ */
+declare function setLoggingEnabled(enabled: boolean): void;
+/**
+ * Create a logger for a specific module
+ *
+ * @param module - Module name (e.g., 'LocalInference', 'ModelCache')
+ * @returns Logger instance
+ *
+ * @example
+ * ```typescript
+ * const logger = createLogger('LocalInference');
+ * logger.info('Model loaded', { backend: 'webgpu', loadTimeMs: 1234 });
+ * ```
+ */
+declare function createLogger(module: string): ILogger;
+/**
+ * Clear logger cache (useful for testing)
+ */
+declare function clearLoggerCache(): void;
+/**
+ * No-op logger for when logging is completely disabled
+ */
+declare const noopLogger: ILogger;
+/**
+ * Get a no-op logger (for production builds that tree-shake logging)
+ */
+declare function getNoopLogger(): ILogger;
+
+export { DEFAULT_LOGGING_CONFIG as D, type ILogger as I, type LogFormatter as L, LOG_LEVEL_PRIORITY as a, type LogEntry as b, type LogLevel as c, type LogSink as d, type LoggingConfig as e, clearLoggerCache as f, configureLogging as g, createLogger as h, getLoggingConfig as i, getNoopLogger as j, setLoggingEnabled as k, noopLogger as n, resetLoggingConfig as r, setLogLevel as s };

package/dist/Logger-BeUI6jG7.d.ts

@@ -0,0 +1,145 @@
+/**
+ * Logging types for Omote SDK
+ *
+ * 6-level logging system with structured output:
+ * - error: Critical failures that prevent operation
+ * - warn: Recoverable issues or degraded performance
+ * - info: Key lifecycle events (model loaded, inference complete)
+ * - debug: Detailed operational info for development
+ * - trace: Fine-grained tracing for performance analysis
+ * - verbose: Extremely detailed output (tensor shapes, intermediate values)
+ */
+type LogLevel = 'error' | 'warn' | 'info' | 'debug' | 'trace' | 'verbose';
+/**
+ * Numeric priority for log levels (lower = more severe)
+ */
+declare const LOG_LEVEL_PRIORITY: Record<LogLevel, number>;
+/**
+ * Structured log entry
+ */
+interface LogEntry {
+    /** Unix timestamp in milliseconds */
+    timestamp: number;
+    /** Log level */
+    level: LogLevel;
+    /** Module name (e.g., 'LocalInference', 'ModelCache') */
+    module: string;
+    /** Human-readable message */
+    message: string;
+    /** Optional structured data */
+    data?: Record<string, unknown>;
+    /** Optional error object */
+    error?: Error;
+    /** Trace ID from active telemetry span (log-to-span correlation) */
+    traceId?: string;
+    /** Span ID from active telemetry span (log-to-span correlation) */
+    spanId?: string;
+}
+/**
+ * Log output sink interface
+ */
+interface LogSink {
+    (entry: LogEntry): void;
+}
+/**
+ * Log formatter interface
+ */
+interface LogFormatter {
+    (entry: LogEntry): string;
+}
+/**
+ * Global logging configuration
+ */
+interface LoggingConfig {
+    /** Minimum log level to output (default: 'info') */
+    level: LogLevel;
+    /** Enable/disable logging globally (default: true) */
+    enabled: boolean;
+    /** Output format: 'json' for structured, 'pretty' for human-readable */
+    format: 'json' | 'pretty';
+    /** Custom output sink (default: console) */
+    sink?: LogSink;
+    /** Include timestamps in output (default: true) */
+    timestamps?: boolean;
+    /** Include module name in output (default: true) */
+    includeModule?: boolean;
+}
+/**
+ * Logger interface for module-specific logging
+ */
+interface ILogger {
+    error(message: string, data?: Record<string, unknown>): void;
+    warn(message: string, data?: Record<string, unknown>): void;
+    info(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>): void;
+    trace(message: string, data?: Record<string, unknown>): void;
+    verbose(message: string, data?: Record<string, unknown>): void;
+    /** Create a child logger with a sub-module name */
+    child(subModule: string): ILogger;
+    /** Get the module name for this logger */
+    readonly module: string;
+}
+/**
+ * Default configuration
+ */
+declare const DEFAULT_LOGGING_CONFIG: LoggingConfig;
+
+/**
+ * Omote SDK Logger
+ *
+ * Unified logging system with:
+ * - 6 log levels (error, warn, info, debug, trace, verbose)
+ * - Structured JSON output for machine parsing
+ * - Pretty output for human readability
+ * - Module-based child loggers
+ * - Runtime configuration
+ * - Browser and Node.js compatible
+ */
+
+/**
+ * Configure global logging settings
+ */
+declare function configureLogging(config: Partial<LoggingConfig>): void;
+/**
+ * Get current logging configuration
+ */
+declare function getLoggingConfig(): LoggingConfig;
+/**
+ * Reset logging configuration to defaults
+ */
+declare function resetLoggingConfig(): void;
+/**
+ * Set log level at runtime
+ */
+declare function setLogLevel(level: LogLevel): void;
+/**
+ * Enable or disable logging
+ */
+declare function setLoggingEnabled(enabled: boolean): void;
+/**
+ * Create a logger for a specific module
+ *
+ * @param module - Module name (e.g., 'LocalInference', 'ModelCache')
+ * @returns Logger instance
+ *
+ * @example
+ * ```typescript
+ * const logger = createLogger('LocalInference');
+ * logger.info('Model loaded', { backend: 'webgpu', loadTimeMs: 1234 });
+ * ```
+ */
+declare function createLogger(module: string): ILogger;
+/**
+ * Clear logger cache (useful for testing)
+ */
+declare function clearLoggerCache(): void;
+/**
+ * No-op logger for when logging is completely disabled
+ */
+declare const noopLogger: ILogger;
+/**
+ * Get a no-op logger (for production builds that tree-shake logging)
+ */
+declare function getNoopLogger(): ILogger;
+
+export { DEFAULT_LOGGING_CONFIG as D, type ILogger as I, type LogFormatter as L, LOG_LEVEL_PRIORITY as a, type LogEntry as b, type LogLevel as c, type LogSink as d, type LoggingConfig as e, clearLoggerCache as f, configureLogging as g, createLogger as h, getLoggingConfig as i, getNoopLogger as j, setLoggingEnabled as k, noopLogger as n, resetLoggingConfig as r, setLogLevel as s };

package/dist/{Logger-I_k4sGhM.d.mts → Logger-DSoGAYJu.d.mts}

@@ -138,4 +138,4 @@ declare const noopLogger: ILogger;
  */
 declare function getNoopLogger(): ILogger;
 
-export { DEFAULT_LOGGING_CONFIG as D, type ILogger as I, type LogFormatter as L, type LogLevel as a, type LogEntry as b, type LogSink as c, type LoggingConfig as d, LOG_LEVEL_PRIORITY as e, configureLogging as f, getLoggingConfig as g, setLoggingEnabled as h, createLogger as i, clearLoggerCache as j, getNoopLogger as k, noopLogger as n, resetLoggingConfig as r, setLogLevel as s };
+export { DEFAULT_LOGGING_CONFIG as D, type ILogger as I, type LogFormatter as L, LOG_LEVEL_PRIORITY as a, type LogEntry as b, type LogLevel as c, type LogSink as d, type LoggingConfig as e, clearLoggerCache as f, configureLogging as g, createLogger as h, getLoggingConfig as i, getNoopLogger as j, setLoggingEnabled as k, noopLogger as n, resetLoggingConfig as r, setLogLevel as s };

package/dist/{Logger-I_k4sGhM.d.ts → Logger-DSoGAYJu.d.ts}

@@ -138,4 +138,4 @@ declare const noopLogger: ILogger;
  */
 declare function getNoopLogger(): ILogger;
 
-export { DEFAULT_LOGGING_CONFIG as D, type ILogger as I, type LogFormatter as L, type LogLevel as a, type LogEntry as b, type LogSink as c, type LoggingConfig as d, LOG_LEVEL_PRIORITY as e, configureLogging as f, getLoggingConfig as g, setLoggingEnabled as h, createLogger as i, clearLoggerCache as j, getNoopLogger as k, noopLogger as n, resetLoggingConfig as r, setLogLevel as s };
+export { DEFAULT_LOGGING_CONFIG as D, type ILogger as I, type LogFormatter as L, LOG_LEVEL_PRIORITY as a, type LogEntry as b, type LogLevel as c, type LogSink as d, type LoggingConfig as e, clearLoggerCache as f, configureLogging as g, createLogger as h, getLoggingConfig as i, getNoopLogger as j, setLoggingEnabled as k, noopLogger as n, resetLoggingConfig as r, setLogLevel as s };