@convai/web-sdk 0.2.3 → 0.3.0-beta.0

package/dist/core/types.d.ts

@@ -1,5 +1,6 @@
 import { Room } from 'livekit-client';
 import { LatencyMonitor } from '../utils/LatencyMonitor';
+import { BlendshapeQueue } from './BlendshapeQueue';
 /**
  * Audio processing settings for the microphone input.
  * These settings help optimize the audio quality and reduce interruptions.
@@ -70,6 +71,13 @@ export interface ConvaiConfig {
     startWithAudioOn?: boolean;
     /** Enable text-to-speech audio generation (default: true) */
     ttsEnabled?: boolean;
+    /** Blendshape provider for facial animation (optional, default: "neurosync") */
+    blendshapeProvider?: "neurosync";
+    /** Blendshape configuration for facial animation format */
+    blendshapeConfig?: {
+        /** Format of blendshapes: "arkit" or "mha" (Meta Human Animation, default: "mha") */
+        format?: "arkit" | "mha";
+    };
     /** Configuration for character actions and environmental context */
     actionConfig?: {
         /** List of action names the character can perform */
@@ -273,6 +281,8 @@ export interface IConvaiClient {
     readonly screenShareControls: ScreenShareControls;
     /** Latency monitoring (dev tool) */
     readonly latencyMonitor: LatencyMonitor;
+    /** Blendshape queue for time-based lipsync synchronization */
+    readonly blendshapeQueue: BlendshapeQueue;
     /**
      * Connect to a Convai character.
      */
@@ -315,6 +325,15 @@ export interface IConvaiClient {
      * When disabled, character responses won't be spoken aloud.
      */
     toggleTts: (enabled: boolean) => void;
+    /**
+     * Add a post-processor to modify blendshape frames during playback
+     * @param processor - Function that processes blendshape frames
+     */
+    addLipsyncPostProcessor: (processor: import('./BlendshapeQueue').BlendshapePostProcessor) => void;
+    /** Remove a post-processor */
+    removeLipsyncPostProcessor: (processor: import('./BlendshapeQueue').BlendshapePostProcessor) => void;
+    /** Clear all post-processors */
+    clearLipsyncPostProcessors: () => void;
     /**
      * Subscribe to an event
      * @param event Event name ('stateChange', 'message', 'connect', 'disconnect', 'error')

package/dist/core/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,yEAAyE;IACzE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0EAA0E;IAC1E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,YAAY;IAC3B,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6DAA6D;IAC7D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oEAAoE;IACpE,YAAY,CAAC,EAAE;QACb,qDAAqD;QACrD,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,4DAA4D;QAC5D,UAAU,EAAE,KAAK,CAAC;YAChB,qBAAqB;YACrB,IAAI,EAAE,MAAM,CAAC;YACb,yCAAyC;YACzC,GAAG,EAAE,MAAM,CAAC;SACb,CAAC,CAAC;QACH,oDAAoD;QACpD,OAAO,EAAE,KAAK,CAAC;YACb,kBAAkB;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,uCAAuC;YACvC,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;QACH,+DAA+D;QAC/D,sBAAsB,CAAC,EAAE,MAAM,CAAC;KACjC,CAAC;CACH;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,wCAAwC;IACxC,EAAE,EAAE,MAAM,CAAC;IACX;;;;;;;;;;;;OAYG;IACH,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,eAAe,GAAG,QAAQ,GAAG,oBAAoB,GAAG,cAAc,GAAG,aAAa,GAAG,eAAe,GAAG,eAAe,CAAC;IAC7J,sCAAsC;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,iBAAiB;IAChC,0DAA0D;IAC1D,WAAW,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,YAAY,EAAE,OAAO,CAAC;IACtB,6DAA6D;IAC7D,WAAW,EAAE,OAAO,CAAC;IACrB,oEAAoE;IACpE,UAAU,EAAE,OAAO,CAAC;IACpB,kDAAkD;IAClD,UAAU,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,UAAU,EAAE,cAAc,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,OAAO,CAAC;IACxB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,eAAe,EAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAClD,yBAAyB,EAAE,MAAM,IAAI,CAAC;IACtC,wBAAwB,EAAE,MAAM,IAAI,CAAC;IAErC,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,eAAe,EAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAClD,eAAe,EAAE,CAAC,OAAO,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iBAAiB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,kBAAkB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,iBAAiB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,0BAA0B,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,oBAAoB,EAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAE3C,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,WAAW,aAAa;IAC5B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;IAElC;;;OAGG;IACH,QAAQ,CAAC,cAAc,EAAE,OAAO,GAAG,OAAO,GAAG,IAAI,CAAC;IAElD,sEAAsE;IACtE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B,2EAA2E;IAC3E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAEpB,6DAA6D;IAC7D,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC;IAErC,qDAAqD;IACrD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAEnC,+DAA+D;IAC/D,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAE3C,kFAAkF;IAClF,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAE7B,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IAEtC,0DAA0D;IAC1D,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IAEtC,qCAAqC;IACrC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAElD,oCAAoC;IACpC,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IAExC;;OAEG;IACH,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAElD,oDAAoD;IACpD,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC,mEAAmE;IACnE,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B,wEAAwE;IACxE,YAAY,EAAE,MAAM,IAAI,CAAC;IAEzB,2CAA2C;IAC3C,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAE5C;;;;OAIG;IACH,kBAAkB,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAE5E;;;OAGG;IACH,oBAAoB,EAAE,MAAM,IAAI,CAAC;IAEjC;;;OAGG;IACH,kBAAkB,EAAE,CAAC,YAAY,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAEtE;;;OAGG;IACH,iBAAiB,EAAE,CAAC,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAE3D;;;OAGG;IACH,SAAS,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IAEtC;;;;;OAKG;IACH,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IAEtE;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,yEAAyE;IACzE,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0EAA0E;IAC1E,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,YAAY;IAC3B,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,6DAA6D;IAC7D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gFAAgF;IAChF,kBAAkB,CAAC,EAAE,WAAW,CAAC;IACjC,2DAA2D;IAC3D,gBAAgB,CAAC,EAAE;QACjB,qFAAqF;QACrF,MAAM,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;KAC1B,CAAC;IACF,oEAAoE;IACpE,YAAY,CAAC,EAAE;QACb,qDAAqD;QACrD,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,4DAA4D;QAC5D,UAAU,EAAE,KAAK,CAAC;YAChB,qBAAqB;YACrB,IAAI,EAAE,MAAM,CAAC;YACb,yCAAyC;YACzC,GAAG,EAAE,MAAM,CAAC;SACb,CAAC,CAAC;QACH,oDAAoD;QACpD,OAAO,EAAE,KAAK,CAAC;YACb,kBAAkB;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,uCAAuC;YACvC,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;QACH,+DAA+D;QAC/D,sBAAsB,CAAC,EAAE,MAAM,CAAC;KACjC,CAAC;CACH;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,wCAAwC;IACxC,EAAE,EAAE,MAAM,CAAC;IACX;;;;;;;;;;;;OAYG;IACH,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,eAAe,GAAG,QAAQ,GAAG,oBAAoB,GAAG,cAAc,GAAG,aAAa,GAAG,eAAe,GAAG,eAAe,CAAC;IAC7J,sCAAsC;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,iBAAiB;IAChC,0DAA0D;IAC1D,WAAW,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,YAAY,EAAE,OAAO,CAAC;IACtB,6DAA6D;IAC7D,WAAW,EAAE,OAAO,CAAC;IACrB,oEAAoE;IACpE,UAAU,EAAE,OAAO,CAAC;IACpB,kDAAkD;IAClD,UAAU,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,UAAU,EAAE,cAAc,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;CAClF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,OAAO,CAAC;IACxB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,eAAe,EAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAClD,yBAAyB,EAAE,MAAM,IAAI,CAAC;IACtC,wBAAwB,EAAE,MAAM,IAAI,CAAC;IAErC,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,WAAW,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,eAAe,EAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAClD,eAAe,EAAE,CAAC,OAAO,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iBAAiB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,kBAAkB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,iBAAiB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,0BAA0B,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,oBAAoB,EAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAE3C,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IACtE,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,WAAW,aAAa;IAC5B,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;IAElC;;;OAGG;IACH,QAAQ,CAAC,cAAc,EAAE,OAAO,GAAG,OAAO,GAAG,IAAI,CAAC;IAElD,sEAAsE;IACtE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B,2EAA2E;IAC3E,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAEpC;;;OAGG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAEpB,6DAA6D;IAC7D,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE,CAAC;IAErC,qDAAqD;IACrD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAEnC,+DAA+D;IAC/D,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAE3C,kFAAkF;IAClF,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAE7B,gEAAgE;IAChE,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IAEtC,0DAA0D;IAC1D,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IAEtC,qCAAqC;IACrC,QAAQ,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAElD,oCAAoC;IACpC,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IAExC,8DAA8D;IAC9D,QAAQ,CAAC,eAAe,EAAE,eAAe,CAAC;IAE1C;;OAEG;IACH,OAAO,EAAE,CAAC,MAAM,CAAC,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAElD,oDAAoD;IACpD,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC,mEAAmE;IACnE,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B,wEAAwE;IACxE,YAAY,EAAE,MAAM,IAAI,CAAC;IAEzB,2CAA2C;IAC3C,mBAAmB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAE5C;;;;OAIG;IACH,kBAAkB,EAAE,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAE5E;;;OAGG;IACH,oBAAoB,EAAE,MAAM,IAAI,CAAC;IAEjC;;;OAGG;IACH,kBAAkB,EAAE,CAAC,YAAY,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAEtE;;;OAGG;IACH,iBAAiB,EAAE,CAAC,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAE3D;;;OAGG;IACH,SAAS,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IAItC;;;OAGG;IACH,uBAAuB,EAAE,CAAC,SAAS,EAAE,OAAO,mBAAmB,EAAE,uBAAuB,KAAK,IAAI,CAAC;IAElG,8BAA8B;IAC9B,0BAA0B,EAAE,CAAC,SAAS,EAAE,OAAO,mBAAmB,EAAE,uBAAuB,KAAK,IAAI,CAAC;IAErG,gCAAgC;IAChC,0BAA0B,EAAE,MAAM,IAAI,CAAC;IAEvC;;;;;OAKG;IACH,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,MAAM,IAAI,CAAC;IAEtE;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAClE"}

package/dist/index.d.ts

@@ -1,2 +1,4 @@
 export * from './react';
+export type { LatencyMonitor, LatencyMeasurement, LatencyStats } from "./utils/LatencyMonitor";
+export * from './lipsync-helpers';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,cAAc,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,cAAc,SAAS,CAAC;AAGxB,YAAY,EAAE,cAAc,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAG/F,cAAc,mBAAmB,CAAC"}

package/dist/index.js

@@ -1,4 +1,6 @@
 // Default export: React version for backward compatibility
 // Existing users can continue using: import { useConvaiClient } from '@convai/web-sdk';
 export * from './react';
+// Export lipsync helpers
+export * from './lipsync-helpers';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,wFAAwF;AACxF,cAAc,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,wFAAwF;AACxF,cAAc,SAAS,CAAC;AAKxB,yBAAyB;AACzB,cAAc,mBAAmB,CAAC"}

package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts

@@ -0,0 +1,80 @@
+/**
+ * ARKit Blendshape Helpers
+ * Utilities for working with ARKit blendshape arrays (52-60 elements)
+ *
+ * ARKit Standard: 52 blendshapes (indices 0-51)
+ * ARKit Extended: Up to 60 blendshapes with additional facial features
+ */
+/**
+ * ARKit Standard Blendshape Names (indices 0-51)
+ */
+export declare const ARKIT_BLENDSHAPE_NAMES: readonly string[];
+/**
+ * Extended blendshapes (indices 52-59) - if provided
+ * These are optional extensions beyond standard ARKit
+ */
+export declare const ARKIT_EXTENDED_NAMES: readonly string[];
+/**
+ * Map ARKit blendshape array to named object (ARKit standard only: 0-51)
+ * @param values - Array of blendshape values (length 52-60)
+ * @returns Object with blendshape names as keys
+ */
+export declare function mapARKitBlendshapesToNames(values: number[]): Record<string, number>;
+/**
+ * Get only active blendshapes (above threshold)
+ * @param values - Array of blendshape values
+ * @param threshold - Minimum value to consider "active" (default: 0.01)
+ * @returns Sorted array of active blendshapes
+ */
+export declare function getActiveARKitBlendshapes(values: number[], threshold?: number): Array<{
+    name: string;
+    value: number;
+    index: number;
+}>;
+/**
+ * Get specific blendshape value by name
+ * @param values - Array of blendshape values
+ * @param name - Name of the blendshape
+ * @returns The value, or 0 if not found
+ */
+export declare function getARKitBlendshapeValue(values: number[], name: string): number;
+/**
+ * Format ARKit blendshapes for logging
+ * @param values - Array of blendshape values
+ * @param threshold - Only show values above this (default: 0.01)
+ * @param maxEntries - Maximum entries to show (default: 20)
+ */
+export declare function formatARKitBlendshapesForLogging(values: number[], threshold?: number, maxEntries?: number): {
+    active: Record<string, number>;
+    total: number;
+    activeCount: number;
+};
+/**
+ * Get the most important facial features for speech
+ * These are the key blendshapes for lip-sync
+ */
+export declare const KEY_LIPSYNC_BLENDSHAPES: {
+    readonly jaw: readonly [24];
+    readonly lips: readonly [26, 31, 37, 39, 40];
+    readonly smile: readonly [43, 44, 45, 46];
+    readonly tongue: readonly [51];
+};
+/**
+ * Extract only lip-sync relevant blendshapes
+ * @param values - Full blendshape array
+ * @returns Object with categorized lip-sync blendshapes
+ */
+export declare function getLipSyncBlendshapes(values: number[]): {
+    jaw: number;
+    mouthClose: number;
+    mouthFunnel: number;
+    mouthPucker: number;
+    mouthRollLower: number;
+    mouthRollUpper: number;
+    mouthSmileLeft: number;
+    mouthSmileRight: number;
+    mouthStretchLeft: number;
+    mouthStretchRight: number;
+    tongueOut: number;
+};
+//# sourceMappingURL=arkitBlendshapeHelpers.d.ts.map

package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arkitBlendshapeHelpers.d.ts","sourceRoot":"","sources":["../../src/lipsync-helpers/arkitBlendshapeHelpers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,eAAO,MAAM,sBAAsB,EAAE,SAAS,MAAM,EAqD1C,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EASxC,CAAC;AAEX;;;;GAIG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,MAAM,EAAE,GACf,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAqBxB;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,GAAE,MAAa,GACvB,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAiCvD;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EAAE,EAChB,IAAI,EAAE,MAAM,GACX,MAAM,CAeR;AAED;;;;;GAKG;AACH,wBAAgB,gCAAgC,CAC9C,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,GAAE,MAAa,EACxB,UAAU,GAAE,MAAW,GACtB;IACD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;CACrB,CAgBA;AAED;;;GAGG;AACH,eAAO,MAAM,uBAAuB;;;;;CAK1B,CAAC;AAEX;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;IACvD,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB,CAcA"}

package/dist/lipsync-helpers/arkitBlendshapeHelpers.js

@@ -0,0 +1,201 @@
+/**
+ * ARKit Blendshape Helpers
+ * Utilities for working with ARKit blendshape arrays (52-60 elements)
+ *
+ * ARKit Standard: 52 blendshapes (indices 0-51)
+ * ARKit Extended: Up to 60 blendshapes with additional facial features
+ */
+/**
+ * ARKit Standard Blendshape Names (indices 0-51)
+ */
+export const ARKIT_BLENDSHAPE_NAMES = [
+    "browDownLeft", // 0
+    "browDownRight", // 1
+    "browInnerUp", // 2
+    "browOuterUpLeft", // 3
+    "browOuterUpRight", // 4
+    "cheekPuff", // 5
+    "cheekSquintLeft", // 6
+    "cheekSquintRight", // 7
+    "eyeBlinkLeft", // 8
+    "eyeBlinkRight", // 9
+    "eyeLookDownLeft", // 10
+    "eyeLookDownRight", // 11
+    "eyeLookInLeft", // 12
+    "eyeLookInRight", // 13
+    "eyeLookOutLeft", // 14
+    "eyeLookOutRight", // 15
+    "eyeLookUpLeft", // 16
+    "eyeLookUpRight", // 17
+    "eyeSquintLeft", // 18
+    "eyeSquintRight", // 19
+    "eyeWideLeft", // 20
+    "eyeWideRight", // 21
+    "jawForward", // 22
+    "jawLeft", // 23
+    "jawOpen", // 24 🔑 KEY for mouth opening
+    "jawRight", // 25
+    "mouthClose", // 26 🔑 KEY for mouth closing
+    "mouthDimpleLeft", // 27
+    "mouthDimpleRight", // 28
+    "mouthFrownLeft", // 29
+    "mouthFrownRight", // 30
+    "mouthFunnel", // 31 🔑 KEY for "O" sounds
+    "mouthLeft", // 32
+    "mouthLowerDownLeft", // 33
+    "mouthLowerDownRight", // 34
+    "mouthPressLeft", // 35
+    "mouthPressRight", // 36
+    "mouthPucker", // 37 🔑 KEY for "U" sounds
+    "mouthRight", // 38
+    "mouthRollLower", // 39
+    "mouthRollUpper", // 40
+    "mouthShrugLower", // 41
+    "mouthShrugUpper", // 42
+    "mouthSmileLeft", // 43 🔑 KEY for "ee" sounds
+    "mouthSmileRight", // 44 🔑 KEY for "ee" sounds
+    "mouthStretchLeft", // 45 🔑 KEY for wide mouth
+    "mouthStretchRight", // 46 🔑 KEY for wide mouth
+    "mouthUpperUpLeft", // 47
+    "mouthUpperUpRight", // 48
+    "noseSneerLeft", // 49
+    "noseSneerRight", // 50
+    "tongueOut", // 51 🔑 KEY for "th" sounds
+];
+/**
+ * Extended blendshapes (indices 52-59) - if provided
+ * These are optional extensions beyond standard ARKit
+ */
+export const ARKIT_EXTENDED_NAMES = [
+    "eyeDilation", // 52 - Pupil dilation
+    "eyeConstriction", // 53 - Pupil constriction
+    "mouthShrug", // 54 - Combined mouth shrug
+    "lipSuckUpper", // 55 - Upper lip suck
+    "lipSuckLower", // 56 - Lower lip suck
+    "lipFunnel", // 57 - Lip funnel position
+    "lipPress", // 58 - Lip press together
+    "reserved", // 59 - Reserved for future use
+];
+/**
+ * Map ARKit blendshape array to named object (ARKit standard only: 0-51)
+ * @param values - Array of blendshape values (length 52-60)
+ * @returns Object with blendshape names as keys
+ */
+export function mapARKitBlendshapesToNames(values) {
+    const mapped = {};
+    // Map standard ARKit blendshapes (0-51)
+    for (let i = 0; i < Math.min(values.length, ARKIT_BLENDSHAPE_NAMES.length); i++) {
+        mapped[ARKIT_BLENDSHAPE_NAMES[i]] = values[i];
+    }
+    // Map extended blendshapes if available (52+)
+    if (values.length > 52) {
+        const extendedStart = 52;
+        for (let i = 0; i < Math.min(values.length - extendedStart, ARKIT_EXTENDED_NAMES.length); i++) {
+            mapped[ARKIT_EXTENDED_NAMES[i]] = values[extendedStart + i];
+        }
+    }
+    return mapped;
+}
+/**
+ * Get only active blendshapes (above threshold)
+ * @param values - Array of blendshape values
+ * @param threshold - Minimum value to consider "active" (default: 0.01)
+ * @returns Sorted array of active blendshapes
+ */
+export function getActiveARKitBlendshapes(values, threshold = 0.01) {
+    const active = [];
+    // Check standard ARKit blendshapes
+    for (let i = 0; i < Math.min(values.length, ARKIT_BLENDSHAPE_NAMES.length); i++) {
+        if (Math.abs(values[i]) > threshold) {
+            active.push({
+                name: ARKIT_BLENDSHAPE_NAMES[i],
+                value: values[i],
+                index: i,
+            });
+        }
+    }
+    // Check extended blendshapes
+    if (values.length > 52) {
+        for (let i = 52; i < Math.min(values.length, 52 + ARKIT_EXTENDED_NAMES.length); i++) {
+            if (Math.abs(values[i]) > threshold) {
+                active.push({
+                    name: ARKIT_EXTENDED_NAMES[i - 52],
+                    value: values[i],
+                    index: i,
+                });
+            }
+        }
+    }
+    // Sort by absolute value (descending)
+    return active.sort((a, b) => Math.abs(b.value) - Math.abs(a.value));
+}
+/**
+ * Get specific blendshape value by name
+ * @param values - Array of blendshape values
+ * @param name - Name of the blendshape
+ * @returns The value, or 0 if not found
+ */
+export function getARKitBlendshapeValue(values, name) {
+    const index = ARKIT_BLENDSHAPE_NAMES.indexOf(name);
+    if (index !== -1 && index < values.length) {
+        return values[index];
+    }
+    const extendedIndex = ARKIT_EXTENDED_NAMES.indexOf(name);
+    if (extendedIndex !== -1) {
+        const actualIndex = 52 + extendedIndex;
+        if (actualIndex < values.length) {
+            return values[actualIndex];
+        }
+    }
+    return 0;
+}
+/**
+ * Format ARKit blendshapes for logging
+ * @param values - Array of blendshape values
+ * @param threshold - Only show values above this (default: 0.01)
+ * @param maxEntries - Maximum entries to show (default: 20)
+ */
+export function formatARKitBlendshapesForLogging(values, threshold = 0.01, maxEntries = 20) {
+    const active = getActiveARKitBlendshapes(values, threshold)
+        .slice(0, maxEntries)
+        .reduce((acc, item) => {
+        acc[item.name] = Number(item.value.toFixed(4));
+        return acc;
+    }, {});
+    return {
+        active,
+        total: values.length,
+        activeCount: Object.keys(active).length,
+    };
+}
+/**
+ * Get the most important facial features for speech
+ * These are the key blendshapes for lip-sync
+ */
+export const KEY_LIPSYNC_BLENDSHAPES = {
+    jaw: [24], // jawOpen
+    lips: [26, 31, 37, 39, 40], // mouthClose, mouthFunnel, mouthPucker, mouthRollLower, mouthRollUpper
+    smile: [43, 44, 45, 46], // mouthSmile*, mouthStretch*
+    tongue: [51], // tongueOut
+};
+/**
+ * Extract only lip-sync relevant blendshapes
+ * @param values - Full blendshape array
+ * @returns Object with categorized lip-sync blendshapes
+ */
+export function getLipSyncBlendshapes(values) {
+    return {
+        jaw: values[24] || 0,
+        mouthClose: values[26] || 0,
+        mouthFunnel: values[31] || 0,
+        mouthPucker: values[37] || 0,
+        mouthRollLower: values[39] || 0,
+        mouthRollUpper: values[40] || 0,
+        mouthSmileLeft: values[43] || 0,
+        mouthSmileRight: values[44] || 0,
+        mouthStretchLeft: values[45] || 0,
+        mouthStretchRight: values[46] || 0,
+        tongueOut: values[51] || 0,
+    };
+}
+//# sourceMappingURL=arkitBlendshapeHelpers.js.map

package/dist/lipsync-helpers/arkitBlendshapeHelpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arkitBlendshapeHelpers.js","sourceRoot":"","sources":["../../src/lipsync-helpers/arkitBlendshapeHelpers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAsB;IACvD,cAAc,EAAe,IAAI;IACjC,eAAe,EAAc,IAAI;IACjC,aAAa,EAAgB,IAAI;IACjC,iBAAiB,EAAY,IAAI;IACjC,kBAAkB,EAAW,IAAI;IACjC,WAAW,EAAkB,IAAI;IACjC,iBAAiB,EAAY,IAAI;IACjC,kBAAkB,EAAW,IAAI;IACjC,cAAc,EAAe,IAAI;IACjC,eAAe,EAAc,IAAI;IACjC,iBAAiB,EAAY,KAAK;IAClC,kBAAkB,EAAW,KAAK;IAClC,eAAe,EAAc,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,iBAAiB,EAAY,KAAK;IAClC,eAAe,EAAc,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,eAAe,EAAc,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,aAAa,EAAgB,KAAK;IAClC,cAAc,EAAe,KAAK;IAClC,YAAY,EAAiB,KAAK;IAClC,SAAS,EAAoB,KAAK;IAClC,SAAS,EAAoB,8BAA8B;IAC3D,UAAU,EAAmB,KAAK;IAClC,YAAY,EAAiB,8BAA8B;IAC3D,iBAAiB,EAAY,KAAK;IAClC,kBAAkB,EAAW,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,iBAAiB,EAAY,KAAK;IAClC,aAAa,EAAgB,2BAA2B;IACxD,WAAW,EAAkB,KAAK;IAClC,oBAAoB,EAAS,KAAK;IAClC,qBAAqB,EAAQ,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,iBAAiB,EAAY,KAAK;IAClC,aAAa,EAAgB,2BAA2B;IACxD,YAAY,EAAiB,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,iBAAiB,EAAY,KAAK;IAClC,iBAAiB,EAAY,KAAK;IAClC,gBAAgB,EAAa,4BAA4B;IACzD,iBAAiB,EAAY,4BAA4B;IACzD,kBAAkB,EAAW,2BAA2B;IACxD,mBAAmB,EAAU,2BAA2B;IACxD,kBAAkB,EAAW,KAAK;IAClC,mBAAmB,EAAU,KAAK;IAClC,eAAe,EAAc,KAAK;IAClC,gBAAgB,EAAa,KAAK;IAClC,WAAW,EAAkB,4BAA4B;CACjD,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAsB;IACrD,aAAa,EAAgB,sBAAsB;IACnD,iBAAiB,EAAY,0BAA0B;IACvD,YAAY,EAAiB,4BAA4B;IACzD,cAAc,EAAe,sBAAsB;IACnD,cAAc,EAAe,sBAAsB;IACnD,WAAW,EAAkB,2BAA2B;IACxD,UAAU,EAAmB,0BAA0B;IACvD,UAAU,EAAmB,+BAA+B;CACpD,CAAC;AAEX;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CACxC,MAAgB;IAEhB,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,wCAAwC;IACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAChF,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,8CAA8C;IAC9C,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACvB,MAAM,aAAa,GAAG,EAAE,CAAC;QACzB,KACE,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,aAAa,EAAE,oBAAoB,CAAC,MAAM,CAAC,EACxE,CAAC,EAAE,EACH,CAAC;YACD,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,yBAAyB,CACvC,MAAgB,EAChB,YAAoB,IAAI;IAExB,MAAM,MAAM,GAA0D,EAAE,CAAC;IAEzE,mCAAmC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAChF,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC;gBAC/B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChB,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACvB,KACE,IAAI,CAAC,GAAG,EAAE,EACV,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,EAC7D,CAAC,EAAE,EACH,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,oBAAoB,CAAC,CAAC,GAAG,EAAE,CAAC;oBAClC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;oBAChB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CACrC,MAAgB,EAChB,IAAY;IAEZ,MAAM,KAAK,GAAG,sBAAsB,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC;IAC1D,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,aAAa,GAAG,oBAAoB,CAAC,OAAO,CAAC,IAAW,CAAC,CAAC;IAChE,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,MAAgB,EAChB,YAAoB,IAAI,EACxB,aAAqB,EAAE;IAMvB,MAAM,MAAM,GAAG,yBAAyB,CAAC,MAAM,EAAE,SAAS,CAAC;SACxD,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC;SACpB,MAAM,CACL,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACZ,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA4B,CAC7B,CAAC;IAEJ,OAAO;QACL,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM;KACxC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU;IACrB,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,uEAAuE;IACnG,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,6BAA6B;IACtD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,YAAY;CAClB,CAAC;AAEX;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAgB;IAapD,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAC5B,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAC5B,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAC/B,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAC/B,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAC/B,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAChC,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QACjC,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;QAClC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;KAC3B,CAAC;AACJ,CAAC"}

package/dist/lipsync-helpers/arkitOrder61.d.ts

@@ -0,0 +1,121 @@
+/**
+ * ARKit 61-Element Custom Order Mapper
+ * Maps the custom 61-element ARKit blendshape array to standard ARKit indices
+ *
+ * This ordering includes:
+ * - 52 standard ARKit facial blendshapes (in custom order)
+ * - 9 additional values for head and eye rotations
+ */
+/**
+ * Custom ARKit 61-element ordering
+ * This is the order in which blendshapes are received from the server
+ */
+export declare const ARKIT_ORDER_61: readonly string[];
+/**
+ * Standard ARKit blendshape order (52 elements)
+ * This is Apple's official ordering
+ */
+export declare const STANDARD_ARKIT_ORDER: readonly string[];
+/**
+ * Convert 61-element custom order array to standard 52-element ARKit array
+ * @param values61 - 61-element array in custom order
+ * @returns 52-element array in standard ARKit order
+ */
+export declare function convertOrder61ToStandard(values61: number[]): number[];
+/**
+ * Map 61-element array to named object (all 61 values)
+ * @param values61 - 61-element array in custom order
+ * @returns Object with blendshape/rotation names as keys
+ */
+export declare function mapOrder61ToNames(values61: number[]): Record<string, number>;
+/**
+ * Get specific blendshape value by name from 61-element array
+ * @param values61 - 61-element array in custom order
+ * @param name - Name of the blendshape (case-insensitive)
+ * @returns The value, or 0 if not found
+ */
+export declare function getBlendshapeValueFromOrder61(values61: number[], name: string): number;
+/**
+ * Extract head rotation values from 61-element array
+ * @param values61 - 61-element array in custom order
+ * @returns Object with head rotation values (yaw, pitch, roll)
+ */
+export declare function getHeadRotation(values61: number[]): {
+    yaw: number;
+    pitch: number;
+    roll: number;
+};
+/**
+ * Extract eye rotation values from 61-element array
+ * @param values61 - 61-element array in custom order
+ * @returns Object with left and right eye rotation values
+ */
+export declare function getEyeRotations(values61: number[]): {
+    left: {
+        yaw: number;
+        pitch: number;
+        roll: number;
+    };
+    right: {
+        yaw: number;
+        pitch: number;
+        roll: number;
+    };
+};
+/**
+ * Get key lip-sync blendshapes from 61-element array
+ * These are the most important values for facial animation
+ */
+export declare function getLipSyncValuesFromOrder61(values61: number[]): {
+    jawOpen: number;
+    mouthClose: number;
+    mouthFunnel: number;
+    mouthPucker: number;
+    mouthSmileLeft: number;
+    mouthSmileRight: number;
+    mouthStretchLeft: number;
+    mouthStretchRight: number;
+    mouthRollLower: number;
+    mouthRollUpper: number;
+    tongueOut: number;
+};
+/**
+ * Get active blendshapes from 61-element array (above threshold)
+ * @param values61 - 61-element array in custom order
+ * @param threshold - Minimum value to consider "active" (default: 0.01)
+ * @returns Sorted array of active blendshapes
+ */
+export declare function getActiveBlendshapesFromOrder61(values61: number[], threshold?: number): Array<{
+    name: string;
+    value: number;
+    index: number;
+}>;
+/**
+ * Format 61-element array for logging
+ * @param values61 - 61-element array in custom order
+ * @param threshold - Only show values above this (default: 0.01)
+ * @param maxEntries - Maximum entries to show (default: 20)
+ */
+export declare function formatOrder61ForLogging(values61: number[], threshold?: number, maxEntries?: number): {
+    active: Record<string, number>;
+    total: number;
+    activeCount: number;
+    headRotation?: {
+        yaw: number;
+        pitch: number;
+        roll: number;
+    };
+    eyeRotations?: {
+        left: {
+            yaw: number;
+            pitch: number;
+            roll: number;
+        };
+        right: {
+            yaw: number;
+            pitch: number;
+            roll: number;
+        };
+    };
+};
+//# sourceMappingURL=arkitOrder61.d.ts.map

package/dist/lipsync-helpers/arkitOrder61.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arkitOrder61.d.ts","sourceRoot":"","sources":["../../src/lipsync-helpers/arkitOrder61.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;;GAGG;AACH,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EA8DlC,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAqDxC,CAAC;AAoBX;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAWrE;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAW5E;AAED;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,EAAE,MAAM,GACX,MAAM,CASR;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACd,CAMA;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG;IACnD,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACnD,KAAK,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CACrD,CAaA;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB,CAcA;AAED;;;;;GAKG;AACH,wBAAgB,+BAA+B,CAC7C,QAAQ,EAAE,MAAM,EAAE,EAClB,SAAS,GAAE,MAAa,GACvB,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAiBvD;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,MAAM,EAAE,EAClB,SAAS,GAAE,MAAa,EACxB,UAAU,GAAE,MAAW,GACtB;IACD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5D,YAAY,CAAC,EAAE;QACb,IAAI,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;QACnD,KAAK,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC;KACrD,CAAC;CACH,CAkBA"}