@convai/web-sdk 0.2.4 → 0.3.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +535 -1031
- package/dist/core/AudioManager.d.ts.map +1 -1
- package/dist/core/AudioManager.js +0 -5
- package/dist/core/AudioManager.js.map +1 -1
- package/dist/core/BlendshapeQueue.d.ts +128 -0
- package/dist/core/BlendshapeQueue.d.ts.map +1 -0
- package/dist/core/BlendshapeQueue.js +229 -0
- package/dist/core/BlendshapeQueue.js.map +1 -0
- package/dist/core/ConvaiClient.d.ts +19 -0
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +67 -29
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MessageHandler.d.ts +7 -0
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +35 -2
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/ScreenShareManager.d.ts.map +1 -1
- package/dist/core/ScreenShareManager.js +0 -3
- package/dist/core/ScreenShareManager.js.map +1 -1
- package/dist/core/VideoManager.d.ts.map +1 -1
- package/dist/core/VideoManager.js +0 -4
- package/dist/core/VideoManager.js.map +1 -1
- package/dist/core/index.d.ts +2 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +2 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/types.d.ts +19 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts +80 -0
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts.map +1 -0
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.js +201 -0
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.js.map +1 -0
- package/dist/lipsync-helpers/arkitOrder61.d.ts +121 -0
- package/dist/lipsync-helpers/arkitOrder61.d.ts.map +1 -0
- package/dist/lipsync-helpers/arkitOrder61.js +287 -0
- package/dist/lipsync-helpers/arkitOrder61.js.map +1 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.d.ts +155 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.d.ts.map +1 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.js +362 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.js.map +1 -0
- package/dist/lipsync-helpers/index.d.ts +10 -0
- package/dist/lipsync-helpers/index.d.ts.map +1 -0
- package/dist/lipsync-helpers/index.js +21 -0
- package/dist/lipsync-helpers/index.js.map +1 -0
- package/dist/lipsync-helpers/metahumanOrder251.d.ts +115 -0
- package/dist/lipsync-helpers/metahumanOrder251.d.ts.map +1 -0
- package/dist/lipsync-helpers/metahumanOrder251.js +432 -0
- package/dist/lipsync-helpers/metahumanOrder251.js.map +1 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts +30 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts.map +1 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js +315 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js.map +1 -0
- package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
- package/dist/react/components/ConvaiWidget.js +43 -3
- package/dist/react/components/ConvaiWidget.js.map +1 -1
- package/dist/react/hooks/useCharacterInfo.d.ts.map +1 -1
- package/dist/react/hooks/useCharacterInfo.js +1 -1
- package/dist/react/hooks/useCharacterInfo.js.map +1 -1
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
- package/dist/react/hooks/useConvaiClient.js +5 -0
- package/dist/react/hooks/useConvaiClient.js.map +1 -1
- package/dist/utils/speakerManagement.d.ts.map +1 -1
- package/dist/utils/speakerManagement.js +0 -5
- package/dist/utils/speakerManagement.js.map +1 -1
- package/dist/vanilla/AudioRenderer.d.ts +5 -0
- package/dist/vanilla/AudioRenderer.d.ts.map +1 -1
- package/dist/vanilla/AudioRenderer.js +27 -18
- package/dist/vanilla/AudioRenderer.js.map +1 -1
- package/package.json +4 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AudioManager.d.ts","sourceRoot":"","sources":["../../src/core/AudioManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;IACrE,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,WAAW,CAAa;gBAEpB,IAAI,CAAC,EAAE,IAAI;IAOvB,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,YAAY,IAAI,OAAO,CAE1B;IAED,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAczB;;OAEG;IACH,OAAO,CAAC,cAAc;IA2CtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAKxB;;;;OAIG;IACH,iBAAiB,CAAC,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,IAAI;IA0CzD;;OAEG;IACH,OAAO,CAAC,WAAW;IAInB;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"AudioManager.d.ts","sourceRoot":"","sources":["../../src/core/AudioManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;IACrE,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,WAAW,CAAa;gBAEpB,IAAI,CAAC,EAAE,IAAI;IAOvB,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,YAAY,IAAI,OAAO,CAE1B;IAED,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI;IAczB;;OAEG;IACH,OAAO,CAAC,cAAc;IA2CtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAKxB;;;;OAIG;IACH,iBAAiB,CAAC,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,IAAI;IA0CzD;;OAEG;IACH,OAAO,CAAC,WAAW;IAInB;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBlC;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCnC;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAIhC;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAIlC;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQlC;;OAEG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYrD;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAUnD;;OAEG;IACH,yBAAyB,IAAI,IAAI;IAOjC;;OAEG;IACH,wBAAwB,IAAI,IAAI;IAIhC;;OAEG;IACH,KAAK,IAAI,IAAI;CAUd"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { logger } from "../utils/logger";
|
|
2
1
|
import { EventEmitter } from "./EventEmitter";
|
|
3
2
|
/**
|
|
4
3
|
* Manages audio controls for LiveKit room
|
|
@@ -149,7 +148,6 @@ export class AudioManager extends EventEmitter {
|
|
|
149
148
|
});
|
|
150
149
|
}
|
|
151
150
|
catch (error) {
|
|
152
|
-
logger.error("Failed to enable audio:", error);
|
|
153
151
|
throw error;
|
|
154
152
|
}
|
|
155
153
|
}
|
|
@@ -179,7 +177,6 @@ export class AudioManager extends EventEmitter {
|
|
|
179
177
|
});
|
|
180
178
|
}
|
|
181
179
|
catch (error) {
|
|
182
|
-
logger.error("Failed to disable audio:", error);
|
|
183
180
|
throw error;
|
|
184
181
|
}
|
|
185
182
|
}
|
|
@@ -217,7 +214,6 @@ export class AudioManager extends EventEmitter {
|
|
|
217
214
|
await this.room.localParticipant.setMicrophoneEnabled(true, { deviceId });
|
|
218
215
|
}
|
|
219
216
|
catch (error) {
|
|
220
|
-
logger.error("Failed to set audio device:", error);
|
|
221
217
|
throw error;
|
|
222
218
|
}
|
|
223
219
|
}
|
|
@@ -231,7 +227,6 @@ export class AudioManager extends EventEmitter {
|
|
|
231
227
|
return audioDevices;
|
|
232
228
|
}
|
|
233
229
|
catch (error) {
|
|
234
|
-
logger.error("Failed to get audio devices:", error);
|
|
235
230
|
throw error;
|
|
236
231
|
}
|
|
237
232
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AudioManager.js","sourceRoot":"","sources":["../../src/core/AudioManager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"AudioManager.js","sourceRoot":"","sources":["../../src/core/AudioManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAY;IAM5C,YAAY,IAAW;QACrB,KAAK,EAAE,CAAC;QANF,SAAI,GAAgB,IAAI,CAAC;QACzB,oBAAe,GAAY,KAAK,CAAC;QACjC,kBAAa,GAAY,KAAK,CAAC;QAC/B,gBAAW,GAAW,CAAC,CAAC;QAI9B,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAAU;QAChB,mCAAmC;QACnC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,cAAc,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QAEvB,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAEpD,MAAM,gBAAgB,GAAG,CAAC,KAAU,EAAE,EAAE;YACtC,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC1B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,kBAAkB,GAAG,CAAC,KAAU,EAAE,EAAE;YACxC,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;gBAC3B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,oBAAoB,GAAG,CAAC,KAAU,EAAE,EAAE;YAC1C,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;gBAC5B,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,sBAAsB,GAAG,CAAC,KAAU,EAAE,EAAE;YAC5C,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;gBAClC,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;gBAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC,CAAC;QAEF,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QACpD,gBAAgB,CAAC,EAAE,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;QACxD,gBAAgB,CAAC,EAAE,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAC;QAC5D,gBAAgB,CAAC,EAAE,CAAC,kBAAkB,EAAE,sBAAsB,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACK,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO;QACvB,oEAAoE;IACtE,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,UAA8B,EAAE;QAChD,MAAM,EAAE,IAAI,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEjC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,cAAc,EAAE,CAAC;YACrD,MAAM,YAAY,GAChB,IAAI,CAAC,eAAe,KAAK,KAAK;gBAC9B,IAAI,CAAC,aAAa,KAAK,IAAI;gBAC3B,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC;YAEzB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;YAErB,IAAI,IAAI,IAAI,YAAY,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE;oBAC5B,YAAY,EAAE,IAAI,CAAC,aAAa;oBAChC,cAAc,EAAE,IAAI,CAAC,eAAe;oBACpC,UAAU,EAAE,IAAI,CAAC,WAAW;iBAC7B,CAAC,CAAC;YACL,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;QACpD,MAAM,YAAY,GAAG,gBAAgB,EAAE,mBAAmB,IAAI,KAAK,CAAC;QACpE,MAAM,cAAc,GAAG,CAAC,YAAY,CAAC;QACrC,MAAM,YAAY,GAChB,IAAI,CAAC,eAAe,KAAK,YAAY;YACrC,IAAI,CAAC,aAAa,KAAK,cAAc,CAAC;QAExC,IAAI,CAAC,eAAe,GAAG,YAAY,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,cAAc,CAAC;QAEpC,IAAI,IAAI,IAAI,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5B,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,cAAc,EAAE,IAAI,CAAC,eAAe;gBACpC,UAAU,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW;QACjB,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5B,YAAY,EAAE,KAAK;gBACnB,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACH,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAC1F,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,KAAK,YAAY,IAAI,WAAW,CAAC,KAAK,CAC1E,EAAE,KAAK,CAAC;YAET,6CAA6C;YAC7C,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE7D,sEAAsE;YACtE,+CAA+C;YAC/C,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,CAAC;YAED,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC5B,YAAY,EAAE,IAAI;gBAClB,cAAc,EAAE,KAAK;gBACrB,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,QAAgB;QACnC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;YAChE,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;YAC5E,OAAO,YAAY,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,yBAAyB;QACvB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,uEAAuE;IACzE,CAAC;IAED;;OAEG;IACH,wBAAwB;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE;YAC5B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,KAAK;YACrB,UAAU,EAAE,CAAC;SACd,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-processor function that can modify blendshape frames
|
|
3
|
+
* @param frame - The blendshape frame to process
|
|
4
|
+
* @param frameIndex - Index of the frame in the sequence
|
|
5
|
+
* @param totalFrames - Total number of frames in the sequence
|
|
6
|
+
* @returns Modified frame
|
|
7
|
+
*/
|
|
8
|
+
export type BlendshapePostProcessor = (frame: Float32Array, frameIndex: number, totalFrames: number) => Float32Array;
|
|
9
|
+
/**
|
|
10
|
+
* BlendshapeQueue - Manages buffering and lifecycle of blendshape frames
|
|
11
|
+
*
|
|
12
|
+
* This queue buffers blendshape chunks (typically 10 frames each) as they arrive from the server,
|
|
13
|
+
* before the bot starts speaking. Playback begins when bot-started-speaking is triggered.
|
|
14
|
+
*
|
|
15
|
+
* Flow:
|
|
16
|
+
* 1. Blendshapes arrive in chunks and are buffered in the queue
|
|
17
|
+
* 2. bot-started-speaking triggers conversation start
|
|
18
|
+
* 3. Consumer reads frames from queue using time-based indexing (elapsed_time * 60fps)
|
|
19
|
+
* 4. blendshape-turn-stats marks conversation end
|
|
20
|
+
* 5. Queue drains remaining frames and resets for next conversation
|
|
21
|
+
*
|
|
22
|
+
* Features:
|
|
23
|
+
* - Zero frame blending for smooth transitions (always enabled)
|
|
24
|
+
* - Frame interpolation for smooth animation (always enabled)
|
|
25
|
+
* - Post-processing hooks for custom frame modifications
|
|
26
|
+
*/
|
|
27
|
+
export declare class BlendshapeQueue {
|
|
28
|
+
private queue;
|
|
29
|
+
private conversationActive;
|
|
30
|
+
private conversationEnded;
|
|
31
|
+
private postProcessors;
|
|
32
|
+
private static readonly ZERO_FRAME;
|
|
33
|
+
private static readonly TARGET_FPS;
|
|
34
|
+
/**
|
|
35
|
+
* Add a chunk of blendshapes to the queue
|
|
36
|
+
* @param blendshapes - Array of blendshape frames (each frame is an array of numbers)
|
|
37
|
+
*/
|
|
38
|
+
addChunk(blendshapes: number[][]): void;
|
|
39
|
+
/**
|
|
40
|
+
* Get all frames currently in the queue
|
|
41
|
+
* @returns Array of blendshape frames as Float32Arrays
|
|
42
|
+
*/
|
|
43
|
+
getFrames(): Float32Array[];
|
|
44
|
+
/**
|
|
45
|
+
* Get a specific frame by index
|
|
46
|
+
* @param index - Frame index
|
|
47
|
+
* @returns The frame at the given index, or null if out of bounds
|
|
48
|
+
*/
|
|
49
|
+
getFrame(index: number): Float32Array | null;
|
|
50
|
+
/**
|
|
51
|
+
* Remove frames from the beginning of the queue (consume frames)
|
|
52
|
+
* @param count - Number of frames to remove
|
|
53
|
+
*/
|
|
54
|
+
consumeFrames(count: number): void;
|
|
55
|
+
/**
|
|
56
|
+
* Get the number of frames currently in the queue
|
|
57
|
+
*/
|
|
58
|
+
get length(): number;
|
|
59
|
+
/**
|
|
60
|
+
* Check if queue has frames
|
|
61
|
+
*/
|
|
62
|
+
hasFrames(): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Check if conversation is active
|
|
65
|
+
*/
|
|
66
|
+
isConversationActive(): boolean;
|
|
67
|
+
/**
|
|
68
|
+
* Check if conversation has ended (blendshape-turn-stats received)
|
|
69
|
+
*/
|
|
70
|
+
isConversationEnded(): boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Mark conversation start (called when bot-started-speaking)
|
|
73
|
+
*/
|
|
74
|
+
startConversation(): void;
|
|
75
|
+
/**
|
|
76
|
+
* Mark conversation end (called when blendshape-turn-stats received)
|
|
77
|
+
* Queue should be drained after this point
|
|
78
|
+
*/
|
|
79
|
+
endConversation(): void;
|
|
80
|
+
/**
|
|
81
|
+
* Reset queue for new conversation
|
|
82
|
+
* Should be called after all frames have been consumed
|
|
83
|
+
*/
|
|
84
|
+
reset(): void;
|
|
85
|
+
/**
|
|
86
|
+
* Add a post-processor to modify frames during playback
|
|
87
|
+
*/
|
|
88
|
+
addPostProcessor(processor: BlendshapePostProcessor): void;
|
|
89
|
+
/**
|
|
90
|
+
* Remove a post-processor
|
|
91
|
+
*/
|
|
92
|
+
removePostProcessor(processor: BlendshapePostProcessor): void;
|
|
93
|
+
/**
|
|
94
|
+
* Clear all post-processors
|
|
95
|
+
*/
|
|
96
|
+
clearPostProcessors(): void;
|
|
97
|
+
/**
|
|
98
|
+
* Apply all post-processors to a frame
|
|
99
|
+
*/
|
|
100
|
+
private applyPostProcessing;
|
|
101
|
+
/**
|
|
102
|
+
* Get interpolated frame with zero-frame blending (always enabled)
|
|
103
|
+
* @param elapsedTime - Elapsed time in seconds
|
|
104
|
+
* @returns Interpolated frame with blending info
|
|
105
|
+
*/
|
|
106
|
+
getInterpolatedFrame(elapsedTime: number): {
|
|
107
|
+
frame: Float32Array;
|
|
108
|
+
frameIndex: number;
|
|
109
|
+
isBlendingToZero: boolean;
|
|
110
|
+
} | null;
|
|
111
|
+
/**
|
|
112
|
+
* Interpolate between two frames
|
|
113
|
+
*/
|
|
114
|
+
private interpolate;
|
|
115
|
+
/**
|
|
116
|
+
* Get the zero frame (neutral pose)
|
|
117
|
+
*/
|
|
118
|
+
static getZeroFrame(): Float32Array;
|
|
119
|
+
/**
|
|
120
|
+
* Get debug info about the queue state
|
|
121
|
+
*/
|
|
122
|
+
getDebugInfo(): {
|
|
123
|
+
frameCount: number;
|
|
124
|
+
conversationActive: boolean;
|
|
125
|
+
conversationEnded: boolean;
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=BlendshapeQueue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BlendshapeQueue.d.ts","sourceRoot":"","sources":["../../src/core/BlendshapeQueue.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,MAAM,uBAAuB,GAAG,CACpC,KAAK,EAAE,YAAY,EACnB,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,KAChB,YAAY,CAAC;AAElB;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,kBAAkB,CAAkB;IAC5C,OAAO,CAAC,iBAAiB,CAAkB;IAG3C,OAAO,CAAC,cAAc,CAAiC;IAGvD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAgC;IAClE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,GAAG,IAAI;IASvC;;;OAGG;IACH,SAAS,IAAI,YAAY,EAAE;IAI3B;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IAO5C;;;OAGG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAKlC;;OAEG;IACH,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;OAEG;IACH,oBAAoB,IAAI,OAAO;IAI/B;;OAEG;IACH,mBAAmB,IAAI,OAAO;IAI9B;;OAEG;IACH,iBAAiB,IAAI,IAAI;IAKzB;;;OAGG;IACH,eAAe,IAAI,IAAI;IAKvB;;;OAGG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,gBAAgB,CAAC,SAAS,EAAE,uBAAuB,GAAG,IAAI;IAI1D;;OAEG;IACH,mBAAmB,CAAC,SAAS,EAAE,uBAAuB,GAAG,IAAI;IAO7D;;OAEG;IACH,mBAAmB,IAAI,IAAI;IAI3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAW3B;;;;OAIG;IACH,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG;QACzC,KAAK,EAAE,YAAY,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,gBAAgB,EAAE,OAAO,CAAC;KAC3B,GAAG,IAAI;IA6DR;;OAEG;IACH,OAAO,CAAC,WAAW;IAiBnB;;OAEG;IACH,MAAM,CAAC,YAAY,IAAI,YAAY;IAInC;;OAEG;IACH,YAAY,IAAI;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,kBAAkB,EAAE,OAAO,CAAC;QAC5B,iBAAiB,EAAE,OAAO,CAAC;KAC5B;CAOF"}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BlendshapeQueue - Manages buffering and lifecycle of blendshape frames
|
|
3
|
+
*
|
|
4
|
+
* This queue buffers blendshape chunks (typically 10 frames each) as they arrive from the server,
|
|
5
|
+
* before the bot starts speaking. Playback begins when bot-started-speaking is triggered.
|
|
6
|
+
*
|
|
7
|
+
* Flow:
|
|
8
|
+
* 1. Blendshapes arrive in chunks and are buffered in the queue
|
|
9
|
+
* 2. bot-started-speaking triggers conversation start
|
|
10
|
+
* 3. Consumer reads frames from queue using time-based indexing (elapsed_time * 60fps)
|
|
11
|
+
* 4. blendshape-turn-stats marks conversation end
|
|
12
|
+
* 5. Queue drains remaining frames and resets for next conversation
|
|
13
|
+
*
|
|
14
|
+
* Features:
|
|
15
|
+
* - Zero frame blending for smooth transitions (always enabled)
|
|
16
|
+
* - Frame interpolation for smooth animation (always enabled)
|
|
17
|
+
* - Post-processing hooks for custom frame modifications
|
|
18
|
+
*/
|
|
19
|
+
export class BlendshapeQueue {
|
|
20
|
+
constructor() {
|
|
21
|
+
this.queue = [];
|
|
22
|
+
this.conversationActive = false;
|
|
23
|
+
this.conversationEnded = false;
|
|
24
|
+
// Post-processing
|
|
25
|
+
this.postProcessors = [];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Add a chunk of blendshapes to the queue
|
|
29
|
+
* @param blendshapes - Array of blendshape frames (each frame is an array of numbers)
|
|
30
|
+
*/
|
|
31
|
+
addChunk(blendshapes) {
|
|
32
|
+
const before = this.queue.length;
|
|
33
|
+
for (let i = 0; i < blendshapes.length; i++) {
|
|
34
|
+
this.queue.push(new Float32Array(blendshapes[i]));
|
|
35
|
+
}
|
|
36
|
+
const after = this.queue.length;
|
|
37
|
+
// console.log(`[BlendshapeQueue] Added ${blendshapes.length} frames. Queue: ${before} -> ${after}`);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Get all frames currently in the queue
|
|
41
|
+
* @returns Array of blendshape frames as Float32Arrays
|
|
42
|
+
*/
|
|
43
|
+
getFrames() {
|
|
44
|
+
return this.queue;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Get a specific frame by index
|
|
48
|
+
* @param index - Frame index
|
|
49
|
+
* @returns The frame at the given index, or null if out of bounds
|
|
50
|
+
*/
|
|
51
|
+
getFrame(index) {
|
|
52
|
+
if (index < 0 || index >= this.queue.length) {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
return this.queue[index];
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Remove frames from the beginning of the queue (consume frames)
|
|
59
|
+
* @param count - Number of frames to remove
|
|
60
|
+
*/
|
|
61
|
+
consumeFrames(count) {
|
|
62
|
+
if (count <= 0)
|
|
63
|
+
return;
|
|
64
|
+
this.queue.splice(0, Math.min(count, this.queue.length));
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Get the number of frames currently in the queue
|
|
68
|
+
*/
|
|
69
|
+
get length() {
|
|
70
|
+
return this.queue.length;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Check if queue has frames
|
|
74
|
+
*/
|
|
75
|
+
hasFrames() {
|
|
76
|
+
return this.queue.length > 0;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Check if conversation is active
|
|
80
|
+
*/
|
|
81
|
+
isConversationActive() {
|
|
82
|
+
return this.conversationActive;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Check if conversation has ended (blendshape-turn-stats received)
|
|
86
|
+
*/
|
|
87
|
+
isConversationEnded() {
|
|
88
|
+
return this.conversationEnded;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Mark conversation start (called when bot-started-speaking)
|
|
92
|
+
*/
|
|
93
|
+
startConversation() {
|
|
94
|
+
this.conversationActive = true;
|
|
95
|
+
this.conversationEnded = false;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Mark conversation end (called when blendshape-turn-stats received)
|
|
99
|
+
* Queue should be drained after this point
|
|
100
|
+
*/
|
|
101
|
+
endConversation() {
|
|
102
|
+
this.conversationEnded = true;
|
|
103
|
+
// Note: Don't set conversationActive to false yet - let consumer drain remaining frames
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Reset queue for new conversation
|
|
107
|
+
* Should be called after all frames have been consumed
|
|
108
|
+
*/
|
|
109
|
+
reset() {
|
|
110
|
+
this.queue = [];
|
|
111
|
+
this.conversationActive = false;
|
|
112
|
+
this.conversationEnded = false;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Add a post-processor to modify frames during playback
|
|
116
|
+
*/
|
|
117
|
+
addPostProcessor(processor) {
|
|
118
|
+
this.postProcessors.push(processor);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Remove a post-processor
|
|
122
|
+
*/
|
|
123
|
+
removePostProcessor(processor) {
|
|
124
|
+
const index = this.postProcessors.indexOf(processor);
|
|
125
|
+
if (index !== -1) {
|
|
126
|
+
this.postProcessors.splice(index, 1);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Clear all post-processors
|
|
131
|
+
*/
|
|
132
|
+
clearPostProcessors() {
|
|
133
|
+
this.postProcessors = [];
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Apply all post-processors to a frame
|
|
137
|
+
*/
|
|
138
|
+
applyPostProcessing(frame, frameIndex) {
|
|
139
|
+
return this.postProcessors.reduce((processedFrame, processor) => processor(processedFrame, frameIndex, this.queue.length), frame);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Get interpolated frame with zero-frame blending (always enabled)
|
|
143
|
+
* @param elapsedTime - Elapsed time in seconds
|
|
144
|
+
* @returns Interpolated frame with blending info
|
|
145
|
+
*/
|
|
146
|
+
getInterpolatedFrame(elapsedTime) {
|
|
147
|
+
if (this.queue.length === 0) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const totalDuration = this.queue.length / BlendshapeQueue.TARGET_FPS;
|
|
151
|
+
const frameDuration = 1 / BlendshapeQueue.TARGET_FPS;
|
|
152
|
+
const halfFrameDuration = frameDuration * 0.5;
|
|
153
|
+
let startFrame;
|
|
154
|
+
let endFrame;
|
|
155
|
+
let alpha;
|
|
156
|
+
let frameIndex;
|
|
157
|
+
let isBlendingToZero = false;
|
|
158
|
+
// Beginning: blend from zero to first frame
|
|
159
|
+
if (elapsedTime <= halfFrameDuration) {
|
|
160
|
+
startFrame = BlendshapeQueue.ZERO_FRAME;
|
|
161
|
+
endFrame = this.queue[0];
|
|
162
|
+
alpha = elapsedTime / halfFrameDuration;
|
|
163
|
+
frameIndex = 0;
|
|
164
|
+
}
|
|
165
|
+
// End: blend from last frame to zero
|
|
166
|
+
else if (elapsedTime >= totalDuration - halfFrameDuration) {
|
|
167
|
+
startFrame = this.queue[this.queue.length - 1];
|
|
168
|
+
endFrame = BlendshapeQueue.ZERO_FRAME;
|
|
169
|
+
alpha =
|
|
170
|
+
(elapsedTime - (totalDuration - halfFrameDuration)) / halfFrameDuration;
|
|
171
|
+
frameIndex = this.queue.length - 1;
|
|
172
|
+
isBlendingToZero = true;
|
|
173
|
+
}
|
|
174
|
+
// Middle: normal interpolation
|
|
175
|
+
else {
|
|
176
|
+
const adjustedTime = elapsedTime - halfFrameDuration;
|
|
177
|
+
const currentFrameIndex = Math.floor(adjustedTime / frameDuration);
|
|
178
|
+
const nextFrameIndex = Math.min(currentFrameIndex + 1, this.queue.length - 1);
|
|
179
|
+
startFrame = this.queue[currentFrameIndex];
|
|
180
|
+
endFrame = this.queue[nextFrameIndex];
|
|
181
|
+
alpha = (adjustedTime % frameDuration) / frameDuration;
|
|
182
|
+
frameIndex = currentFrameIndex;
|
|
183
|
+
}
|
|
184
|
+
// Always interpolate for smooth animation
|
|
185
|
+
let resultFrame = this.interpolate(startFrame, endFrame, alpha);
|
|
186
|
+
// Apply post-processing
|
|
187
|
+
if (this.postProcessors.length > 0) {
|
|
188
|
+
resultFrame = this.applyPostProcessing(resultFrame, frameIndex);
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
frame: resultFrame,
|
|
192
|
+
frameIndex,
|
|
193
|
+
isBlendingToZero,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Interpolate between two frames
|
|
198
|
+
*/
|
|
199
|
+
interpolate(start, end, alpha) {
|
|
200
|
+
const result = new Float32Array(Math.max(start.length, end.length));
|
|
201
|
+
const minLength = Math.min(start.length, end.length);
|
|
202
|
+
for (let i = 0; i < minLength; i++) {
|
|
203
|
+
const startVal = start[i] || 0;
|
|
204
|
+
const endVal = end[i] || 0;
|
|
205
|
+
result[i] = startVal + (endVal - startVal) * alpha;
|
|
206
|
+
}
|
|
207
|
+
return result;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Get the zero frame (neutral pose)
|
|
211
|
+
*/
|
|
212
|
+
static getZeroFrame() {
|
|
213
|
+
return new Float32Array(BlendshapeQueue.ZERO_FRAME);
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Get debug info about the queue state
|
|
217
|
+
*/
|
|
218
|
+
getDebugInfo() {
|
|
219
|
+
return {
|
|
220
|
+
frameCount: this.queue.length,
|
|
221
|
+
conversationActive: this.conversationActive,
|
|
222
|
+
conversationEnded: this.conversationEnded,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
// Constants
|
|
227
|
+
BlendshapeQueue.ZERO_FRAME = new Float32Array(52).fill(0);
|
|
228
|
+
BlendshapeQueue.TARGET_FPS = 60;
|
|
229
|
+
//# sourceMappingURL=BlendshapeQueue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BlendshapeQueue.js","sourceRoot":"","sources":["../../src/core/BlendshapeQueue.ts"],"names":[],"mappings":"AAaA;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,eAAe;IAA5B;QACU,UAAK,GAAmB,EAAE,CAAC;QAC3B,uBAAkB,GAAY,KAAK,CAAC;QACpC,sBAAiB,GAAY,KAAK,CAAC;QAE3C,kBAAkB;QACV,mBAAc,GAA8B,EAAE,CAAC;IA4PzD,CAAC;IAtPC;;;OAGG;IACH,QAAQ,CAAC,WAAuB;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAChC,qGAAqG;IACvG,CAAC;IAED;;;OAGG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;OAIG;IACH,QAAQ,CAAC,KAAa;QACpB,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5C,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,KAAa;QACzB,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO;QACvB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,oBAAoB;QAClB,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,wFAAwF;IAC1F,CAAC;IAED;;;OAGG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QAChC,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,SAAkC;QACjD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,SAAkC;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,KAAmB,EACnB,UAAkB;QAElB,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAC/B,CAAC,cAAc,EAAE,SAAS,EAAE,EAAE,CAC5B,SAAS,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAC1D,KAAK,CACN,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,WAAmB;QAKtC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC;QACrE,MAAM,aAAa,GAAG,CAAC,GAAG,eAAe,CAAC,UAAU,CAAC;QACrD,MAAM,iBAAiB,GAAG,aAAa,GAAG,GAAG,CAAC;QAE9C,IAAI,UAAwB,CAAC;QAC7B,IAAI,QAAsB,CAAC;QAC3B,IAAI,KAAa,CAAC;QAClB,IAAI,UAAkB,CAAC;QACvB,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAE7B,4CAA4C;QAC5C,IAAI,WAAW,IAAI,iBAAiB,EAAE,CAAC;YACrC,UAAU,GAAG,eAAe,CAAC,UAAU,CAAC;YACxC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,KAAK,GAAG,WAAW,GAAG,iBAAiB,CAAC;YACxC,UAAU,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,qCAAqC;aAChC,IAAI,WAAW,IAAI,aAAa,GAAG,iBAAiB,EAAE,CAAC;YAC1D,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/C,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC;YACtC,KAAK;gBACH,CAAC,WAAW,GAAG,CAAC,aAAa,GAAG,iBAAiB,CAAC,CAAC,GAAG,iBAAiB,CAAC;YAC1E,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACnC,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC;QACD,+BAA+B;aAC1B,CAAC;YACJ,MAAM,YAAY,GAAG,WAAW,GAAG,iBAAiB,CAAC;YACrD,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC;YACnE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC7B,iBAAiB,GAAG,CAAC,EACrB,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CACtB,CAAC;YAEF,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC3C,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACtC,KAAK,GAAG,CAAC,YAAY,GAAG,aAAa,CAAC,GAAG,aAAa,CAAC;YACvD,UAAU,GAAG,iBAAiB,CAAC;QACjC,CAAC;QAED,0CAA0C;QAC1C,IAAI,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEhE,wBAAwB;QACxB,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAClE,CAAC;QAED,OAAO;YACL,KAAK,EAAE,WAAW;YAClB,UAAU;YACV,gBAAgB;SACjB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,WAAW,CACjB,KAAmB,EACnB,GAAiB,EACjB,KAAa;QAEb,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAErD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC3B,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,KAAK,CAAC;QACrD,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,YAAY;QACjB,OAAO,IAAI,YAAY,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,YAAY;QAKV,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YAC7B,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;YAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;SAC1C,CAAC;IACJ,CAAC;;AAzPD,YAAY;AACY,0BAAU,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,AAA/B,CAAgC;AAC1C,0BAAU,GAAG,EAAE,AAAL,CAAM"}
|
|
@@ -2,6 +2,7 @@ import { Room } from "livekit-client";
|
|
|
2
2
|
import { ConvaiConfig, ConvaiClientState, ChatMessage, IConvaiClient, AudioControls, VideoControls, ScreenShareControls } from "./types";
|
|
3
3
|
import { EventEmitter } from "./EventEmitter";
|
|
4
4
|
import { LatencyMonitor } from "../utils/LatencyMonitor";
|
|
5
|
+
import { BlendshapeQueue } from "./BlendshapeQueue";
|
|
5
6
|
/**
|
|
6
7
|
* Main Convai client class for managing AI voice assistant connections
|
|
7
8
|
* Provides complete interface for connecting to Convai's voice assistants,
|
|
@@ -55,6 +56,8 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
55
56
|
private _screenShareManager;
|
|
56
57
|
private _messageHandler;
|
|
57
58
|
private _latencyMonitor;
|
|
59
|
+
private _conversationSessionId;
|
|
60
|
+
private _conversationStartTime;
|
|
58
61
|
constructor(config?: ConvaiConfig);
|
|
59
62
|
get state(): ConvaiClientState;
|
|
60
63
|
get connectionType(): "audio" | "video" | null;
|
|
@@ -70,6 +73,8 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
70
73
|
get videoControls(): VideoControls;
|
|
71
74
|
get screenShareControls(): ScreenShareControls;
|
|
72
75
|
get latencyMonitor(): LatencyMonitor;
|
|
76
|
+
get blendshapeQueue(): BlendshapeQueue;
|
|
77
|
+
get conversationSessionId(): number;
|
|
73
78
|
/**
|
|
74
79
|
* Setup event listeners for room and managers
|
|
75
80
|
*/
|
|
@@ -134,5 +139,19 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
134
139
|
* Toggle text-to-speech on or off
|
|
135
140
|
*/
|
|
136
141
|
toggleTts(enabled: boolean): void;
|
|
142
|
+
/**
|
|
143
|
+
* Add a post-processor to modify blendshape frames during playback
|
|
144
|
+
* @param processor - Function that processes blendshape frames
|
|
145
|
+
*/
|
|
146
|
+
addLipsyncPostProcessor(processor: import("./BlendshapeQueue").BlendshapePostProcessor): void;
|
|
147
|
+
/**
|
|
148
|
+
* Remove a post-processor
|
|
149
|
+
* @param processor - The processor function to remove
|
|
150
|
+
*/
|
|
151
|
+
removeLipsyncPostProcessor(processor: import("./BlendshapeQueue").BlendshapePostProcessor): void;
|
|
152
|
+
/**
|
|
153
|
+
* Clear all post-processors
|
|
154
|
+
*/
|
|
155
|
+
clearLipsyncPostProcessors(): void;
|
|
137
156
|
}
|
|
138
157
|
//# sourceMappingURL=ConvaiClient.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAa,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,WAAW,EAEX,aAAa,EACb,aAAa,EACb,aAAa,EACb,mBAAmB,EACpB,MAAM,SAAS,CAAC;AAKjB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAa,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,WAAW,EAEX,aAAa,EACb,aAAa,EACb,aAAa,EACb,mBAAmB,EACpB,MAAM,SAAS,CAAC;AAKjB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAG9C,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;IACrE,OAAO,CAAC,KAAK,CAAO;IACpB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,aAAa,CAA6B;IAGlD,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,mBAAmB,CAAqB;IAChD,OAAO,CAAC,eAAe,CAAiB;IACxC,OAAO,CAAC,eAAe,CAAiB;IAGxC,OAAO,CAAC,sBAAsB,CAAa;IAC3C,OAAO,CAAC,sBAAsB,CAAa;gBAE/B,MAAM,CAAC,EAAE,YAAY;IA0CjC,IAAI,KAAK,IAAI,iBAAiB,CAE7B;IAED,IAAI,cAAc,IAAI,OAAO,GAAG,OAAO,GAAG,IAAI,CAE7C;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,IAAI,CAE1B;IAED,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAE/B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,IAAI,IAAI,IAAI,CAEf;IAED,IAAI,YAAY,IAAI,WAAW,EAAE,CAEhC;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAEtC;IAED,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,mBAAmB,IAAI,mBAAmB,CAE7C;IAED,IAAI,cAAc,IAAI,cAAc,CAEnC;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAED;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqE3B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAuBpC;;OAEG;IACH,OAAO,CAAC,WAAW;IAuBnB;;OAEG;IACG,OAAO,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA8JnD;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAYjC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAmB5B;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAQhC;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAkDvC;;OAEG;IACH,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;IAmBvE;;OAEG;IACH,oBAAoB,IAAI,IAAI;IA2B5B;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,IAAI;IAsBjE;;OAEG;IACH,iBAAiB,CAAC,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAoBtD;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IA4BjC;;;OAGG;IACH,uBAAuB,CACrB,SAAS,EAAE,OAAO,mBAAmB,EAAE,uBAAuB,GAC7D,IAAI;IAIP;;;OAGG;IACH,0BAA0B,CACxB,SAAS,EAAE,OAAO,mBAAmB,EAAE,uBAAuB,GAC7D,IAAI;IAIP;;OAEG;IACH,0BAA0B,IAAI,IAAI;CAGnC"}
|