@omote/core 0.9.2 → 0.9.4

package/dist/index.d.mts

@@ -2517,6 +2517,12 @@ interface A2EProcessorConfig {
      * different expression intensity across face regions (brows, eyes, cheeks).
      */
     identityIndex?: number;
+    /**
+     * Spring halflife for blendshape smoothing (seconds).
+     * - 0.06 (default): Sweet spot for lip sync
+     * - 0: Bypass (raw frames, no smoothing)
+     */
+    smoothingHalflife?: number;
     /** Callback fired with each blendshape frame (push mode) */
     onFrame?: (frame: Float32Array) => void;
     /** Error callback */
@@ -2538,9 +2544,11 @@ declare class A2EProcessor {
     private plainQueue;
     private _latestFrame;
     private dripInterval;
-    private lastPulledFrame;
+    private smoother;
     private lastDequeuedTime;
-    private decayBuffer;
+    private lastUpdateTime;
+    private neutralTriggered;
+    private outputBuffer;
     private inferenceRunning;
     private pendingChunks;
     private getFrameCallCount;

package/dist/index.d.ts

@@ -2517,6 +2517,12 @@ interface A2EProcessorConfig {
      * different expression intensity across face regions (brows, eyes, cheeks).
      */
     identityIndex?: number;
+    /**
+     * Spring halflife for blendshape smoothing (seconds).
+     * - 0.06 (default): Sweet spot for lip sync
+     * - 0: Bypass (raw frames, no smoothing)
+     */
+    smoothingHalflife?: number;
     /** Callback fired with each blendshape frame (push mode) */
     onFrame?: (frame: Float32Array) => void;
     /** Error callback */
@@ -2538,9 +2544,11 @@ declare class A2EProcessor {
     private plainQueue;
     private _latestFrame;
     private dripInterval;
-    private lastPulledFrame;
+    private smoother;
     private lastDequeuedTime;
-    private decayBuffer;
+    private lastUpdateTime;
+    private neutralTriggered;
+    private outputBuffer;
     private inferenceRunning;
     private pendingChunks;
     private getFrameCallCount;

package/dist/index.js

@@ -1687,12 +1687,85 @@ var AudioChunkCoalescer = class {
   }
 };
 
+// src/inference/BlendshapeSmoother.ts
+var NUM_BLENDSHAPES = 52;
+var BlendshapeSmoother = class {
+  constructor(config) {
+    /** Whether any target has been set */
+    this._hasTarget = false;
+    this.halflife = config?.halflife ?? 0.06;
+    this.values = new Float32Array(NUM_BLENDSHAPES);
+    this.velocities = new Float32Array(NUM_BLENDSHAPES);
+    this.targets = new Float32Array(NUM_BLENDSHAPES);
+  }
+  /** Whether a target frame has been set (false until first setTarget call) */
+  get hasTarget() {
+    return this._hasTarget;
+  }
+  /**
+   * Set new target frame from inference output.
+   * Springs will converge toward these values on subsequent update() calls.
+   */
+  setTarget(frame) {
+    this.targets.set(frame);
+    this._hasTarget = true;
+  }
+  /**
+   * Advance all 52 springs by `dt` seconds and return the smoothed frame.
+   *
+   * Call this every render frame (e.g., inside requestAnimationFrame).
+   * Returns the internal values buffer — do NOT mutate the returned array.
+   *
+   * @param dt - Time step in seconds (e.g., 1/60 for 60fps)
+   * @returns Smoothed blendshape values (Float32Array of 52)
+   */
+  update(dt) {
+    if (!this._hasTarget) {
+      return this.values;
+    }
+    if (this.halflife <= 0) {
+      this.values.set(this.targets);
+      this.velocities.fill(0);
+      return this.values;
+    }
+    const damping = Math.LN2 / this.halflife;
+    const eydt = Math.exp(-damping * dt);
+    for (let i = 0; i < NUM_BLENDSHAPES; i++) {
+      const j0 = this.values[i] - this.targets[i];
+      const j1 = this.velocities[i] + j0 * damping;
+      this.values[i] = eydt * (j0 + j1 * dt) + this.targets[i];
+      this.velocities[i] = eydt * (this.velocities[i] - j1 * damping * dt);
+      this.values[i] = Math.max(0, Math.min(1, this.values[i]));
+    }
+    return this.values;
+  }
+  /**
+   * Decay all spring targets to neutral (0).
+   *
+   * Call when inference stalls (no new frames for threshold duration).
+   * The springs will smoothly close the mouth / relax the face over
+   * the halflife period rather than freezing.
+   */
+  decayToNeutral() {
+    this.targets.fill(0);
+  }
+  /**
+   * Reset all state (values, velocities, targets).
+   * Call when starting a new playback session.
+   */
+  reset() {
+    this.values.fill(0);
+    this.velocities.fill(0);
+    this.targets.fill(0);
+    this._hasTarget = false;
+  }
+};
+
 // src/inference/A2EProcessor.ts
 var logger4 = createLogger("A2EProcessor");
 var FRAME_RATE = 30;
 var DRIP_INTERVAL_MS = 33;
-var HOLD_DURATION_MS = 100;
-var DECAY_DURATION_MS = 200;
+var NEUTRAL_THRESHOLD_MS = 1500;
 var _A2EProcessor = class _A2EProcessor {
   constructor(config) {
     this.writeOffset = 0;
@@ -1703,10 +1776,10 @@ var _A2EProcessor = class _A2EProcessor {
     // Push mode state
     this._latestFrame = null;
     this.dripInterval = null;
-    // Last-frame-hold for pull mode (prevents avatar freezing between frames)
-    this.lastPulledFrame = null;
     this.lastDequeuedTime = 0;
-    this.decayBuffer = null;
+    this.lastUpdateTime = 0;
+    this.neutralTriggered = false;
+    this.outputBuffer = null;
     // Inference serialization
     this.inferenceRunning = false;
     this.pendingChunks = [];
@@ -1719,6 +1792,7 @@ var _A2EProcessor = class _A2EProcessor {
     this.identityIndex = config.identityIndex ?? 0;
     this.onFrame = config.onFrame;
     this.onError = config.onError;
+    this.smoother = new BlendshapeSmoother({ halflife: config.smoothingHalflife ?? 0.06 });
     this.bufferCapacity = this.chunkSize * 2;
     this.buffer = new Float32Array(this.bufferCapacity);
   }
@@ -1818,8 +1892,11 @@ var _A2EProcessor = class _A2EProcessor {
     this.timestampedQueue = [];
     this.plainQueue = [];
     this._latestFrame = null;
-    this.lastPulledFrame = null;
+    this.smoother.reset();
     this.lastDequeuedTime = 0;
+    this.lastUpdateTime = 0;
+    this.neutralTriggered = false;
+    this.outputBuffer = null;
     this.pendingChunks = [];
     this.inferenceRunning = false;
     this.getFrameCallCount = 0;
@@ -1838,6 +1915,7 @@ var _A2EProcessor = class _A2EProcessor {
    */
   getFrameForTime(currentTime) {
     this.getFrameCallCount++;
+    const now = getClock().now();
     const discardWindow = this.backend.backend === "wasm" ? 1.5 : 0.5;
     let discardCount = 0;
     while (this.timestampedQueue.length > 0 && this.timestampedQueue[0].timestamp < currentTime - discardWindow) {
@@ -1853,13 +1931,20 @@ var _A2EProcessor = class _A2EProcessor {
         nextFrameTs: this.timestampedQueue.length > 0 ? this.timestampedQueue[0].timestamp.toFixed(3) : "none"
       });
     }
+    let newDequeue = false;
     if (this.timestampedQueue.length > 0 && this.timestampedQueue[0].timestamp <= currentTime) {
       const { frame } = this.timestampedQueue.shift();
-      this.lastPulledFrame = frame;
-      this.lastDequeuedTime = getClock().now();
-      return frame;
+      const firstTarget = !this.smoother.hasTarget;
+      this.smoother.setTarget(frame);
+      if (firstTarget) {
+        this.smoother.update(1);
+        this.lastUpdateTime = now;
+      }
+      newDequeue = true;
+      this.neutralTriggered = false;
+      this.lastDequeuedTime = now;
     }
-    if (this.timestampedQueue.length > 0 && this.getFrameCallCount % 60 === 0) {
+    if (!newDequeue && this.timestampedQueue.length > 0 && this.getFrameCallCount % 60 === 0) {
       logger4.debug("getFrameForTime: waiting for playback time to reach queued frames", {
         queueLen: this.timestampedQueue.length,
         frontTimestamp: this.timestampedQueue[0].timestamp.toFixed(4),
@@ -1867,25 +1952,24 @@ var _A2EProcessor = class _A2EProcessor {
         delta: (this.timestampedQueue[0].timestamp - currentTime).toFixed(4)
       });
     }
-    if (this.lastPulledFrame) {
-      const elapsed = getClock().now() - this.lastDequeuedTime;
-      if (elapsed < HOLD_DURATION_MS) {
-        return this.lastPulledFrame;
-      }
-      const decayElapsed = elapsed - HOLD_DURATION_MS;
-      if (decayElapsed >= DECAY_DURATION_MS) {
-        this.lastPulledFrame = null;
-        return null;
-      }
-      const t = decayElapsed / DECAY_DURATION_MS;
-      const factor = (1 - t) * (1 - t);
-      if (!this.decayBuffer) this.decayBuffer = new Float32Array(52);
-      for (let i = 0; i < 52; i++) {
-        this.decayBuffer[i] = this.lastPulledFrame[i] * factor;
+    if (!newDequeue && this.smoother.hasTarget && !this.neutralTriggered) {
+      const elapsed = now - this.lastDequeuedTime;
+      if (elapsed >= NEUTRAL_THRESHOLD_MS) {
+        this.smoother.decayToNeutral();
+        this.neutralTriggered = true;
       }
-      return this.decayBuffer;
     }
-    return null;
+    if (!this.smoother.hasTarget) {
+      return null;
+    }
+    const dt = Math.min((now - this.lastUpdateTime) / 1e3, 0.1);
+    const smoothed = this.smoother.update(dt);
+    this.lastUpdateTime = now;
+    if (newDequeue || !this.outputBuffer) {
+      this.outputBuffer = new Float32Array(52);
+    }
+    this.outputBuffer.set(smoothed);
+    return this.outputBuffer;
   }
   // ═══════════════════════════════════════════════════════════════════════
   // Frame Output — Push Mode (live mic, game loop)
@@ -11216,80 +11300,6 @@ var InterruptionHandler = class extends EventEmitter {
   }
 };
 
-// src/inference/BlendshapeSmoother.ts
-var NUM_BLENDSHAPES = 52;
-var BlendshapeSmoother = class {
-  constructor(config) {
-    /** Whether any target has been set */
-    this._hasTarget = false;
-    this.halflife = config?.halflife ?? 0.06;
-    this.values = new Float32Array(NUM_BLENDSHAPES);
-    this.velocities = new Float32Array(NUM_BLENDSHAPES);
-    this.targets = new Float32Array(NUM_BLENDSHAPES);
-  }
-  /** Whether a target frame has been set (false until first setTarget call) */
-  get hasTarget() {
-    return this._hasTarget;
-  }
-  /**
-   * Set new target frame from inference output.
-   * Springs will converge toward these values on subsequent update() calls.
-   */
-  setTarget(frame) {
-    this.targets.set(frame);
-    this._hasTarget = true;
-  }
-  /**
-   * Advance all 52 springs by `dt` seconds and return the smoothed frame.
-   *
-   * Call this every render frame (e.g., inside requestAnimationFrame).
-   * Returns the internal values buffer — do NOT mutate the returned array.
-   *
-   * @param dt - Time step in seconds (e.g., 1/60 for 60fps)
-   * @returns Smoothed blendshape values (Float32Array of 52)
-   */
-  update(dt) {
-    if (!this._hasTarget) {
-      return this.values;
-    }
-    if (this.halflife <= 0) {
-      this.values.set(this.targets);
-      this.velocities.fill(0);
-      return this.values;
-    }
-    const damping = Math.LN2 / this.halflife;
-    const eydt = Math.exp(-damping * dt);
-    for (let i = 0; i < NUM_BLENDSHAPES; i++) {
-      const j0 = this.values[i] - this.targets[i];
-      const j1 = this.velocities[i] + j0 * damping;
-      this.values[i] = eydt * (j0 + j1 * dt) + this.targets[i];
-      this.velocities[i] = eydt * (this.velocities[i] - j1 * damping * dt);
-      this.values[i] = Math.max(0, Math.min(1, this.values[i]));
-    }
-    return this.values;
-  }
-  /**
-   * Decay all spring targets to neutral (0).
-   *
-   * Call when inference stalls (no new frames for threshold duration).
-   * The springs will smoothly close the mouth / relax the face over
-   * the halflife period rather than freezing.
-   */
-  decayToNeutral() {
-    this.targets.fill(0);
-  }
-  /**
-   * Reset all state (values, velocities, targets).
-   * Call when starting a new playback session.
-   */
-  reset() {
-    this.values.fill(0);
-    this.velocities.fill(0);
-    this.targets.fill(0);
-    this._hasTarget = false;
-  }
-};
-
 // src/inference/SafariSpeechRecognition.ts
 var logger33 = createLogger("SafariSpeech");
 var SafariSpeechRecognition = class _SafariSpeechRecognition {