@convai/web-sdk 1.0.0-beta.1 → 1.0.0

package/README.md

@@ -293,6 +293,49 @@ convaiClient.videoControls.on("videoStateChange", (state) => {
 });
 ```
 
+### Lipsync (Facial Animation for 3D Characters)
+
+Enable lipsync to receive blendshape data for animating 3D character faces in sync with speech:
+
+```typescript
+const client = new ConvaiClient({
+  apiKey: "your-api-key",
+  characterId: "your-character-id",
+  enableLipsync: true,
+  blendshapeConfig: {
+    format: "arkit", // or "mha" for MetaHuman
+  },
+});
+
+await client.connect();
+
+// In your 3D render loop (60 FPS)
+let conversationStartTime = 0;
+
+client.on("speakingChange", (speaking) => {
+  if (speaking) conversationStartTime = Date.now();
+});
+
+function render() {
+  const elapsedSeconds = (Date.now() - conversationStartTime) / 1000;
+  const result = client.blendshapeQueue.getFrameAtTime(elapsedSeconds);
+
+  if (result) {
+    // Apply blendshape values to your 3D character
+    myCharacter.morphTargets["jawOpen"] = result.frame[0];
+    myCharacter.morphTargets["mouthSmile"] = result.frame[1];
+    // ... apply remaining blendshapes
+  }
+
+  requestAnimationFrame(render);
+}
+```
+
+**Blendshape Formats:**
+
+- `arkit` - 61 blendshapes (iOS ARKit standard)
+- `mha` - 251 blendshapes (MetaHuman)
+
 ### Interruption
 
 Interrupt the character's current response to allow the user to speak immediately.

package/dist/core/index.d.ts

@@ -6,5 +6,6 @@ export { VideoManager } from './VideoManager';
 export { ScreenShareManager } from './ScreenShareManager';
 export { MessageHandler } from './MessageHandler';
 export { BlendshapeQueue } from './BlendshapeQueue';
+export type { TurnStats } from './BlendshapeQueue';
 export { EventEmitter } from './EventEmitter';
 //# sourceMappingURL=index.d.ts.map

package/dist/core/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,cAAc,SAAS,CAAC;AAGxB,YAAY,EAAE,YAAY,IAAI,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGvE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGlD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAGpD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,cAAc,SAAS,CAAC;AAGxB,YAAY,EAAE,YAAY,IAAI,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAGvE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGlD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,YAAY,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/core/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,4CAA4C;AAC5C,cAAc,SAAS,CAAC;AAKxB,gCAAgC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,+BAA+B;AAC/B,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,qCAAqC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,4CAA4C;AAC5C,cAAc,SAAS,CAAC;AAKxB,gCAAgC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,+BAA+B;AAC/B,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAGpD,qCAAqC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@convai/web-sdk",
-  "version": "1.0.0-beta.1",
+  "version": "1.0.0",
   "description": "Convai Web SDK for AI voice assistants. Supports both React and vanilla JavaScript/TypeScript. Build voice-powered AI interactions with real-time audio/video streaming.",
   "private": false,
   "type": "module",
@@ -51,10 +51,14 @@
     "example:react:build": "cd examples/react && pnpm build",
     "example:vanilla": "cd examples/vanilla && pnpm dev",
     "example:vanilla:build": "cd examples/vanilla && pnpm build",
+    "example:chatbot-testing": "cd examples/chatbot-testing && pnpm dev",
+    "example:chatbot-testing:build": "cd examples/chatbot-testing && pnpm build",
+    "example:lipsync-testing": "cd examples/lipsync-testing && pnpm dev",
+    "example:lipsync-testing:build": "cd examples/lipsync-testing && pnpm build",
     "examples:dev": "pnpm example:react",
-    "examples:build": "pnpm example:react:build && pnpm example:vanilla:build",
-    "examples:install": "pnpm install && cd examples/react && pnpm install && cd ../vanilla && pnpm install",
-    "examples:clean": "pnpm clean && cd examples/react && pnpm clean && cd ../vanilla && pnpm clean"
+    "examples:build": "pnpm example:react:build && pnpm example:vanilla:build && pnpm example:chatbot-testing:build && pnpm example:lipsync-testing:build",
+    "examples:install": "pnpm install && cd examples/react && pnpm install && cd ../vanilla && pnpm install && cd ../chatbot-testing && pnpm install && cd ../lipsync-testing && pnpm install",
+    "examples:clean": "pnpm clean && cd examples/react && pnpm clean && cd ../vanilla && pnpm clean && cd ../chatbot-testing && pnpm clean && cd ../lipsync-testing && pnpm clean"
   },
   "keywords": [
     "convai",