@goodganglabs/lipsync-wasm-v2 0.3.0

package/README.md

@@ -0,0 +1,565 @@
+# @goodganglabs/lipsync-wasm-v2
+
+WebAssembly-based real-time audio-to-blendshape lip sync engine.
+Converts 16kHz PCM audio into 52-dimensional ARKit-compatible blendshape frames at 30fps using a student distillation model.
+
+## Which Version?
+
+| | V2 (this package) | V1 |
+|---|---|---|
+| **Dimensions** | 52-dim ARKit | 111-dim ARKit |
+| **Model** | Student distillation | Phoneme classification |
+| **Post-processing** | Built-in (crisp mouth, fade, blinks) | Manual |
+| **Idle expression** | Not included | Built-in `IdleExpressionGenerator` |
+| **VAD** | Not included | Built-in `VoiceActivityDetector` |
+| **ONNX fallback** | None (ONNX required) | Heuristic fallback |
+| **Recommendation** | Most use cases | Full expression control needed |
+
+## Features
+
+- 52-dim ARKit blendshape output (direct prediction, no intermediate phoneme step)
+- Batch and real-time streaming processing
+- Built-in post-processing: mouth articulation enhancement, fade in/out, automatic blink injection
+- Built-in expression preset blending
+- Embedded VRMA bone animation data
+- Runs entirely in the browser via WebAssembly
+
+## Requirements
+
+- **onnxruntime-web** `>=1.17.0` (peer dependency, **required** — V2 has no heuristic fallback)
+
+```html
+<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web@1.17.0/dist/ort.min.js"></script>
+```
+
+## Installation
+
+```bash
+npm install @goodganglabs/lipsync-wasm-v2
+```
+
+## Quick Start
+
+### Batch Processing
+
+```js
+import { LipSyncWasmWrapper } from '@goodganglabs/lipsync-wasm-v2';
+
+const lipsync = new LipSyncWasmWrapper();
+await lipsync.init();
+
+const result = await lipsync.processFile(audioFile);
+for (let i = 0; i < result.frame_count; i++) {
+  const frame = lipsync.getFrame(result, i); // number[52]
+  applyToAvatar(frame);
+}
+
+lipsync.dispose();
+```
+
+### Three.js VRM Complete Example
+
+Full integration with a VRM avatar: init, load VRM, apply blendshapes, render loop.
+
+```html
+<script type="importmap">
+{ "imports": {
+    "three": "https://cdn.jsdelivr.net/npm/three@0.179.1/build/three.module.js",
+    "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.179.1/examples/jsm/",
+    "@pixiv/three-vrm": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm@3.4.5/lib/three-vrm.module.min.js",
+    "@pixiv/three-vrm-animation": "https://cdn.jsdelivr.net/npm/@pixiv/three-vrm-animation@3.4.5/lib/three-vrm-animation.module.min.js"
+}}
+</script>
+<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web@1.17.0/dist/ort.min.js"></script>
+
+<canvas id="avatar-canvas" style="width:100%; height:500px;"></canvas>
+
+<script type="module">
+import * as THREE from 'three';
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';
+import { LipSyncWasmWrapper } from '@goodganglabs/lipsync-wasm-v2';
+
+// --- Index-to-name mapping (52-dim ARKit) ---
+const SYSTEM_INDEX_TO_BLENDSHAPE = {
+  0: ['browDownLeft'], 1: ['browDownRight'], 2: ['browInnerUp'],
+  3: ['browOuterUpLeft'], 4: ['browOuterUpRight'],
+  5: ['cheekPuff'], 6: ['cheekSquintLeft'], 7: ['cheekSquintRight'],
+  8: ['eyeBlinkLeft'], 9: ['eyeBlinkRight'],
+  10: ['eyeLookDownLeft'], 11: ['eyeLookDownRight'],
+  12: ['eyeLookInLeft'], 13: ['eyeLookInRight'],
+  14: ['eyeLookOutLeft'], 15: ['eyeLookOutRight'],
+  16: ['eyeLookUpLeft'], 17: ['eyeLookUpRight'],
+  18: ['eyeSquintLeft'], 19: ['eyeSquintRight'],
+  20: ['eyeWideLeft'], 21: ['eyeWideRight'],
+  22: ['jawForward'], 23: ['jawLeft'], 24: ['jawOpen'], 25: ['jawRight'],
+  26: ['mouthClose'], 27: ['mouthDimpleLeft'], 28: ['mouthDimpleRight'],
+  29: ['mouthFrownLeft'], 30: ['mouthFrownRight'], 31: ['mouthFunnel'],
+  32: ['mouthLeft'], 33: ['mouthLowerDownLeft'], 34: ['mouthLowerDownRight'],
+  35: ['mouthPressLeft'], 36: ['mouthPressRight'], 37: ['mouthPucker'],
+  38: ['mouthRight'], 39: ['mouthRollLower'], 40: ['mouthRollUpper'],
+  41: ['mouthShrugLower'], 42: ['mouthShrugUpper'],
+  43: ['mouthSmileLeft'], 44: ['mouthSmileRight'],
+  45: ['mouthStretchLeft'], 46: ['mouthStretchRight'],
+  47: ['mouthUpperUpLeft'], 48: ['mouthUpperUpRight'],
+  49: ['noseSneerLeft'], 50: ['noseSneerRight'],
+  51: ['tongueOut']
+};
+
+// --- Apply blendshape frame to VRM ---
+function applyBlendshapes(vrm, frame) {
+  if (!vrm) return;
+
+  // VRM 1.0 (expressionManager)
+  if (vrm.expressionManager) {
+    for (const [idx, names] of Object.entries(SYSTEM_INDEX_TO_BLENDSHAPE)) {
+      const value = frame[idx] || 0;
+      for (const name of names) {
+        vrm.expressionManager.setValue(name, value);
+      }
+    }
+    return;
+  }
+
+  // VRM 0.x (blendShapeProxy)
+  if (vrm.blendShapeProxy) {
+    for (const [idx, names] of Object.entries(SYSTEM_INDEX_TO_BLENDSHAPE)) {
+      const value = frame[idx] || 0;
+      for (const name of names) {
+        vrm.blendShapeProxy.setValue(name, value);
+      }
+    }
+    vrm.blendShapeProxy.update();
+    return;
+  }
+
+  // Fallback: direct morph target manipulation
+  vrm.scene.traverse((child) => {
+    if (!child.isMesh || !child.morphTargetDictionary || !child.morphTargetInfluences) return;
+    for (const [idx, names] of Object.entries(SYSTEM_INDEX_TO_BLENDSHAPE)) {
+      const value = frame[idx] || 0;
+      for (const name of names) {
+        const morphIdx = child.morphTargetDictionary[name];
+        if (morphIdx !== undefined) {
+          child.morphTargetInfluences[morphIdx] = value;
+        }
+      }
+    }
+  });
+}
+
+// --- Setup ---
+const canvas = document.getElementById('avatar-canvas');
+const scene = new THREE.Scene();
+scene.background = new THREE.Color(0x1a1a2e);
+
+const camera = new THREE.PerspectiveCamera(30, canvas.clientWidth / canvas.clientHeight, 0.1, 100);
+camera.position.set(0, 1.25, 0.5);
+
+const renderer = new THREE.WebGLRenderer({ canvas, antialias: true });
+renderer.setSize(canvas.clientWidth, canvas.clientHeight);
+renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
+
+const controls = new OrbitControls(camera, canvas);
+controls.target.set(0, 1.25, 0);
+controls.enableDamping = true;
+
+scene.add(new THREE.AmbientLight(0xffffff, 2.0));
+const dirLight = new THREE.DirectionalLight(0xffffff, 1.1);
+dirLight.position.set(1, 3, 2);
+scene.add(dirLight);
+
+// --- Load VRM ---
+const loader = new GLTFLoader();
+loader.register((parser) => new VRMLoaderPlugin(parser));
+
+const gltf = await new Promise((resolve, reject) =>
+  loader.load('your-avatar.vrm', resolve, undefined, reject)
+);
+const vrm = gltf.userData.vrm;
+VRMUtils.removeUnnecessaryVertices(gltf.scene);
+VRMUtils.removeUnnecessaryJoints(gltf.scene);
+scene.add(vrm.scene);
+
+// --- Init LipSync ---
+const lipsync = new LipSyncWasmWrapper();
+await lipsync.init();
+
+// --- Process audio & animate ---
+const result = await lipsync.processFile(audioFile);
+let frameIndex = 0;
+const clock = new THREE.Clock();
+
+function animate() {
+  requestAnimationFrame(animate);
+  const delta = clock.getDelta();
+  controls.update();
+
+  if (frameIndex < result.frame_count) {
+    const frame = lipsync.getFrame(result, frameIndex);
+    applyBlendshapes(vrm, frame);
+    frameIndex++;
+  }
+
+  vrm.update(delta);
+  renderer.render(scene, camera);
+}
+animate();
+</script>
+```
+
+## ARKit Blendshape Index
+
+Full 52-element index mapping:
+
+| Index | Name | Index | Name |
+|-------|------|-------|------|
+| 0 | `browDownLeft` | 26 | `mouthClose` |
+| 1 | `browDownRight` | 27 | `mouthDimpleLeft` |
+| 2 | `browInnerUp` | 28 | `mouthDimpleRight` |
+| 3 | `browOuterUpLeft` | 29 | `mouthFrownLeft` |
+| 4 | `browOuterUpRight` | 30 | `mouthFrownRight` |
+| 5 | `cheekPuff` | 31 | `mouthFunnel` |
+| 6 | `cheekSquintLeft` | 32 | `mouthLeft` |
+| 7 | `cheekSquintRight` | 33 | `mouthLowerDownLeft` |
+| 8 | `eyeBlinkLeft` | 34 | `mouthLowerDownRight` |
+| 9 | `eyeBlinkRight` | 35 | `mouthPressLeft` |
+| 10 | `eyeLookDownLeft` | 36 | `mouthPressRight` |
+| 11 | `eyeLookDownRight` | 37 | `mouthPucker` |
+| 12 | `eyeLookInLeft` | 38 | `mouthRight` |
+| 13 | `eyeLookInRight` | 39 | `mouthRollLower` |
+| 14 | `eyeLookOutLeft` | 40 | `mouthRollUpper` |
+| 15 | `eyeLookOutRight` | 41 | `mouthShrugLower` |
+| 16 | `eyeLookUpLeft` | 42 | `mouthShrugUpper` |
+| 17 | `eyeLookUpRight` | 43 | `mouthSmileLeft` |
+| 18 | `eyeSquintLeft` | 44 | `mouthSmileRight` |
+| 19 | `eyeSquintRight` | 45 | `mouthStretchLeft` |
+| 20 | `eyeWideLeft` | 46 | `mouthStretchRight` |
+| 21 | `eyeWideRight` | 47 | `mouthUpperUpLeft` |
+| 22 | `jawForward` | 48 | `mouthUpperUpRight` |
+| 23 | `jawLeft` | 49 | `noseSneerLeft` |
+| 24 | `jawOpen` | 50 | `noseSneerRight` |
+| 25 | `jawRight` | 51 | `tongueOut` |
+
+## VRMA Bone Animation
+
+The package includes embedded VRMA bone animation data for idle and speaking poses. Use these with Three.js `AnimationMixer` for natural body motion during lip sync.
+
+```js
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+import { VRMAnimationLoaderPlugin, createVRMAnimationClip } from '@pixiv/three-vrm-animation';
+
+// 1. Get embedded VRMA bytes from the wrapper
+const vrmaData = lipsync.getVrmaBytes();
+
+// 2. Load VRMA from bytes
+async function loadVRMAFromBytes(bytes) {
+  const blob = new Blob([bytes], { type: 'application/octet-stream' });
+  const url = URL.createObjectURL(blob);
+  const loader = new GLTFLoader();
+  loader.register((parser) => new VRMAnimationLoaderPlugin(parser));
+  const gltf = await new Promise((resolve, reject) =>
+    loader.load(url, resolve, undefined, reject)
+  );
+  URL.revokeObjectURL(url);
+  return gltf.userData.vrmAnimations[0];
+}
+
+const idleAnim = await loadVRMAFromBytes(vrmaData.idle);
+const speakingAnim = await loadVRMAFromBytes(vrmaData.speaking);
+
+// 3. Setup AnimationMixer with crossfade
+const mixer = new THREE.AnimationMixer(vrm.scene);
+
+const idleClip = createVRMAnimationClip(idleAnim, vrm);
+const speakingClip = createVRMAnimationClip(speakingAnim, vrm);
+
+const idleAction = mixer.clipAction(idleClip);
+const speakingAction = mixer.clipAction(speakingClip);
+
+idleAction.setLoop(THREE.LoopRepeat);
+speakingAction.setLoop(THREE.LoopRepeat);
+
+idleAction.setEffectiveWeight(1);
+idleAction.play();
+speakingAction.setEffectiveWeight(0);
+speakingAction.play();
+
+// 4. Smoothstep crossfade between idle and speaking
+let crossFadeProgress = 0;
+let isSpeaking = false;
+
+function updateBoneWeights(delta) {
+  const target = isSpeaking ? 1 : 0;
+  const speed = 1.0 / 0.4; // 0.4s transition duration
+  if (target > crossFadeProgress) {
+    crossFadeProgress = Math.min(crossFadeProgress + delta * speed, 1);
+  } else {
+    crossFadeProgress = Math.max(crossFadeProgress - delta * speed, 0);
+  }
+  // Smoothstep interpolation
+  const t = crossFadeProgress;
+  const w = t * t * (3 - 2 * t);
+  speakingAction.setEffectiveWeight(w);
+  idleAction.setEffectiveWeight(1 - w);
+}
+
+// In your render loop:
+// updateBoneWeights(delta);
+// mixer.update(delta);
+```
+
+## Real-time Streaming
+
+### Microphone Input with AudioWorklet
+
+```js
+// 1. Get microphone stream
+const stream = await navigator.mediaDevices.getUserMedia({
+  audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true }
+});
+const audioCtx = new AudioContext({ sampleRate: 16000 });
+const source = audioCtx.createMediaStreamSource(stream);
+
+// 2. AudioWorklet: batch 1600 samples (100ms @ 16kHz)
+const workletCode = `
+class MicProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.buffer = [];
+    this.bufferLen = 0;
+    this.TARGET = 1600; // 100ms @ 16kHz
+  }
+  process(inputs) {
+    const input = inputs[0];
+    if (input.length > 0 && input[0].length > 0) {
+      this.buffer.push(new Float32Array(input[0]));
+      this.bufferLen += input[0].length;
+      if (this.bufferLen >= this.TARGET) {
+        const merged = new Float32Array(this.bufferLen);
+        let off = 0;
+        for (const buf of this.buffer) { merged.set(buf, off); off += buf.length; }
+        this.port.postMessage(merged);
+        this.buffer = [];
+        this.bufferLen = 0;
+      }
+    }
+    return true;
+  }
+}
+registerProcessor('mic-processor', MicProcessor);
+`;
+const blob = new Blob([workletCode], { type: 'application/javascript' });
+const url = URL.createObjectURL(blob);
+await audioCtx.audioWorklet.addModule(url);
+URL.revokeObjectURL(url);
+
+const workletNode = new AudioWorkletNode(audioCtx, 'mic-processor');
+source.connect(workletNode);
+workletNode.connect(audioCtx.destination);
+
+// 3. Frame queue + processing
+const streamQueue = [];
+let micProcessing = false;
+const micBuffer = [];
+
+workletNode.port.onmessage = (e) => {
+  micBuffer.push(e.data);
+  if (!micProcessing) processMicBuffer();
+};
+
+async function processMicBuffer() {
+  if (micBuffer.length === 0) return;
+  micProcessing = true;
+  try {
+    const chunks = micBuffer.splice(0);
+    let totalLen = 0;
+    for (const c of chunks) totalLen += c.length;
+    const audio = new Float32Array(totalLen);
+    let offset = 0;
+    for (const c of chunks) { audio.set(c, offset); offset += c.length; }
+
+    const result = await lipsync.processAudioChunk(audio);
+    if (result && result.frame_count > 0) {
+      for (let i = 0; i < result.frame_count; i++) {
+        streamQueue.push(lipsync.getFrame(result, i));
+      }
+    }
+  } finally {
+    micProcessing = false;
+    if (micBuffer.length > 0) processMicBuffer();
+  }
+}
+
+// 4. Consume at 30fps in render loop
+let streamTimeAccum = 0;
+const frameInterval = 1.0 / 30.0;
+
+function renderLoop() {
+  requestAnimationFrame(renderLoop);
+  const delta = clock.getDelta();
+
+  streamTimeAccum += delta;
+  while (streamTimeAccum >= frameInterval) {
+    streamTimeAccum -= frameInterval;
+    if (streamQueue.length > 0) {
+      const frame = streamQueue.shift();
+      applyBlendshapes(vrm, frame);
+    }
+  }
+
+  vrm.update(delta);
+  renderer.render(scene, camera);
+}
+```
+
+### TTS Streaming Integration
+
+When processing TTS audio chunks, yield to the main thread periodically to prevent render freezes:
+
+```js
+async function processTTSChunks(chunks) {
+  for (let i = 0; i < chunks.length; i++) {
+    const result = await lipsync.processAudioChunk(
+      chunks[i],
+      i === chunks.length - 1  // isLast on final chunk
+    );
+    if (result && result.frame_count > 0) {
+      for (let j = 0; j < result.frame_count; j++) {
+        streamQueue.push(lipsync.getFrame(result, j));
+      }
+    }
+    // Yield to main thread every 3 chunks (~300ms) to keep rAF rendering smooth
+    if ((i + 1) % 3 === 0) {
+      await new Promise((resolve) => setTimeout(resolve, 0));
+    }
+  }
+}
+```
+
+## API Reference
+
+### Constructor
+
+```ts
+new LipSyncWasmWrapper(options?: { wasmPath?: string })
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `wasmPath` | `string` | `'./lipsync_wasm_v2.js'` | Path to the WASM glue module |
+
+### Properties
+
+| Property | Type | Description |
+|----------|------|-------------|
+| `ready` | `boolean` | `true` after `init()` completes |
+| `modelVersion` | `string` | `'v2'` |
+| `blendshapeDim` | `number` | `52` |
+
+### `init(options?): Promise<{ mode: string }>`
+
+Initializes the WASM runtime, loads the ONNX model, and applies the expression preset.
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `onProgress` | `(stage: string, percent: number) => void` | — | Progress callback. Stages: `'wasm'`, `'decrypt'`, `'onnx'` |
+| `preset` | `boolean \| string` | `true` | `true` loads the built-in preset. Pass a URL string to load a custom preset JSON. `false` disables preset loading. |
+
+Returns `{ mode: 'v2-onnx' }`. Throws if ONNX Runtime is not available.
+
+### `processAudio(audio: Float32Array): Promise<ProcessResult>`
+
+Processes a complete 16kHz mono PCM audio buffer.
+
+### `processAudioBuffer(audioBuffer: AudioBuffer): Promise<ProcessResult>`
+
+Processes a Web Audio API `AudioBuffer` (automatically resampled to 16kHz).
+
+### `processFile(file: File): Promise<ProcessResult>`
+
+Decodes and processes an audio `File` object.
+
+### `processAudioChunk(chunk: Float32Array, isLast?: boolean): Promise<ProcessResult | null>`
+
+Feeds an audio chunk for real-time streaming. Streaming sessions are managed internally — the first call starts a session, passing `isLast = true` ends it. Returns `null` if the internal buffer has not accumulated enough data.
+
+### `getFrame(result: ProcessResult, frameIndex: number): number[]`
+
+Extracts a single blendshape frame from a `ProcessResult`. Returns `number[52]`.
+
+### `getVrmaBytes(): { idle: Uint8Array, speaking: Uint8Array }`
+
+Returns embedded VRMA bone animation data for idle and speaking states.
+
+### `reset(): void`
+
+Resets internal state and ends any active streaming session.
+
+### `dispose(): void`
+
+Releases all WASM and ONNX resources.
+
+### ProcessResult
+
+```ts
+{
+  blendshapes: number[];   // Flat array: frame_count * 52 values
+  frame_count: number;     // Number of output frames (30fps)
+  mode: string;            // 'v2-onnx' | 'v2-streaming-onnx'
+}
+```
+
+## Bundler Setup
+
+### Vite
+
+Works out of the box. No additional configuration needed.
+
+### Webpack
+
+Enable async WebAssembly support:
+
+```js
+// webpack.config.js
+module.exports = {
+  experiments: {
+    asyncWebAssembly: true,
+  },
+};
+```
+
+### CDN (no bundler)
+
+Use `<script type="module">` with an import map:
+
+```html
+<script type="importmap">
+{ "imports": {
+    "@goodganglabs/lipsync-wasm-v2": "https://your-cdn.com/lipsync-wasm-v2/lipsync-wasm-wrapper.js"
+}}
+</script>
+<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web@1.17.0/dist/ort.min.js"></script>
+<script type="module">
+  import { LipSyncWasmWrapper } from '@goodganglabs/lipsync-wasm-v2';
+  // ... your code
+</script>
+```
+
+When hosting WASM files on a different path than the wrapper JS, use the `wasmPath` option:
+
+```js
+const lipsync = new LipSyncWasmWrapper({
+  wasmPath: '/static/wasm/lipsync_wasm_v2.js'
+});
+```
+
+## Deployment
+
+`.wasm` files must be served with the `application/wasm` MIME type.
+CORS headers are required for cross-origin usage.
+
+## License
+
+Proprietary — GoodGang Labs

package/lipsync-wasm-wrapper.d.ts

@@ -0,0 +1,54 @@
+/**
+ * @goodganglabs/lipsync-wasm-v2
+ * Audio-to-blendshape lip sync engine (52-dim ARKit, student model)
+ */
+
+export interface ProcessResult {
+    blendshapes: number[];
+    frame_count: number;
+    fps: number;
+    mode?: string;
+}
+
+export interface InitResult {
+    mode: 'v2-onnx';
+}
+
+export interface VrmaBytes {
+    idle: Uint8Array;
+    speaking: Uint8Array;
+}
+
+export interface InitOptions {
+    /** GoodGangLabs license key (e.g. "ggl_xxx"). Omit for 30-day free trial. */
+    licenseKey?: string;
+    onProgress?: (stage: string, percent: number) => void;
+    preset?: boolean | string;
+}
+
+export interface ConstructorOptions {
+    wasmPath?: string;
+}
+
+export class LipSyncWasmWrapper {
+    constructor(options?: ConstructorOptions);
+
+    readonly ready: boolean;
+    readonly modelVersion: 'v2';
+    readonly blendshapeDim: 52;
+    readonly wasmModule: any;
+
+    init(options?: InitOptions): Promise<InitResult>;
+
+    processAudio(audio: Float32Array): Promise<ProcessResult>;
+    processAudioBuffer(audioBuffer: AudioBuffer): Promise<ProcessResult>;
+    processFile(file: File): Promise<ProcessResult>;
+
+    processAudioChunk(audioChunk: Float32Array, isLast?: boolean): Promise<ProcessResult | null>;
+
+    getFrame(result: ProcessResult, frameIndex: number): number[];
+
+    getVrmaBytes(): VrmaBytes;
+    reset(): void;
+    dispose(): void;
+}