@unith-ai/core-client 2.0.4 → 2.0.5-beta.1

package/README.md

@@ -18,6 +18,16 @@ yarn add @unith-ai/core-client
 pnpm install @unith-ai/core-client
 ```
 
+### Azure Speech SDK (Optional)
+
+If using `microphoneProvider: "azure"`, install the Azure Speech SDK:
+
+```shell
+npm install microsoft-cognitiveservices-speech-sdk
+```
+
+The SDK is dynamically imported only when the Azure provider is used, so it won't affect bundle size if you use a different provider like `eleven_labs`.
+
 ## Usage
 
 This library is designed for use in plain JavaScript applications or to serve as a foundation for framework-specific implementations. Before integrating it, verify if a dedicated library exists for your particular framework. That said, it's compatible with any project built on JavaScript.
@@ -62,6 +72,7 @@ const conversation = await Conversation.startDigitalHuman({
 - **microphoneProvider** - Provider for the microphone - `"azure" | "eleven_labs"`
 - **microphoneOptions** - Callbacks for microphone events
   - **onMicrophoneSpeechRecognitionResult ({ transcript: string })** - Called when microphone recognizes your user's speech. This returns a transcript. Ideal behavior is to call the `.sendMessage` method with your transcript as microphone doesn't automatically commit / send users text to our AI.
+  - **onMicrophonePartialSpeechRecognitionResult ()** - Called when microphone recognizes that the user is trying to speak. This **doesn't** returns a transcript. Ideal behavior is to check if digital human is currently responding (`speaking`) and trigger the stop response method (`.stopResponse`). 
   - **onMicrophoneStatusChange ({status})** Called when microphone status changes
     - **status** `"ON" | "OFF" | "PROCESSING"` Shows current status of microphone.
   - **onMicrophoneError ({ message: string })** - Called when microphone has an error with the error message.
@@ -71,6 +82,7 @@ const conversation = await Conversation.startDigitalHuman({
   - **vadThreshold `Number`**
   - **minSpeechDurationMs `Number`**
   - **minSilenceDurationMs `Number`**
+  - **disableDynamicSpeechRecognition `Boolean`** - This disables ElevenLabs dynamic speech recognition language detection and uses the language specified during digital human creation.
 
 #### Callbacks
 
@@ -101,6 +113,8 @@ Register callbacks to handle various events:
 - **onTimeoutWarning** - Called before the session times out. This event warns you that the customers session is going to end in a bit. You can call the `keepSession` method to extend the customers session.
 - **onKeepSession** - Called when a keep-alive request is processed
 - **onError** - Called when an error occurs
+- **onStoppingStart** - Called immediately when a stop request is initiated
+- **onStoppingEnd** - Called once the current response stops playing
 
 ### Getting Background Video
 
@@ -135,7 +149,7 @@ conversation.sendMessage("Hello, how are you?");
 
 #### toggleMicrophone()
 
-Toogles microphone status between ON/OFF.
+Toggles microphone status between ON/OFF.
 
 ```js
 conversation.toggleMicrophone();
@@ -176,6 +190,17 @@ Get the current user's ID:
 const userId = conversation.getUserId();
 ```
 
+#### stopResponse()
+
+Stops the ongoing response and notifies you via two callbacks:
+
+- `onStoppingStart()` - Called immediately when stop is initiated.
+- `onStoppingEnd()` - Called once the current response is stopped.
+
+```js
+await conversation.stopResponse();
+```
+
 #### endSession()
 
 End the conversation session and clean up resources:

package/dist/index.d.ts

@@ -29,6 +29,7 @@ export declare class Conversation {
     private status;
     protected volume: number;
     private sessionStarted;
+    private isStoppingLastResponse;
     private messageCounter;
     private syncController;
     private avController;
@@ -71,12 +72,11 @@ export declare class Conversation {
     private handleEndSession;
     private updateStatus;
     /**
-     * To stop current response, we'll do the following:
-     * 1. set a flag that'll prevent adding new audio & video events to their respective queues
-     * 2. clear video queue & switch state to idle
-     * 3. clear audio queue
-     * 4. send an event if all expected response from BE has been received. If not, FE will keep status in 'stopping' mode
+     * To stop streaming response, we'll send a stop message to the BE. The BE will then stop sending audio and video frames, which will naturally end the response. This is more efficient and leads to a better user experience as it allows for a smoother transition when stopping the response.
+     *
+     * To stop cached response, we just stop the video and fade to idle.
      */
+    stopResponse(): Promise<void>;
     toggleMute(): Promise<number>;
     startSession(): Promise<Connection>;
     toggleMicrophone(): Promise<void>;

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAIjD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAElB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAEhB,cAAc,EACf,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAiBlD,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAKxC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,MAAM,MAAM,OAAO,GAAG,mBAAmB,GAAG,kBAAkB,GAAG,cAAc,CAAC;AAChF,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAC3E,MAAM,MAAM,cAAc,GAAG,iBAAiB,GAC5C,OAAO,CAAC,cAAc,CAAC,GACvB,gBAAgB,GAChB,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAC9B,qBAAa,YAAY;IAsMrB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,QAAQ;IA7MlB,OAAO,CAAC,MAAM,CAAwB;IACtC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAK;IAC7B,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,cAAc,CAAK;IAE3B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,UAAU,CAA2B;IAE7C,OAAO,CAAC,eAAe,CAAwB;IAE/C,OAAO,CAAC,mBAAmB,CAGlB;IAET,OAAO,CAAC,gBAAgB,CAAgB;IAExC,OAAO,CAAC,MAAM,CAAC,cAAc;IA8B7B;;;;OAIG;WACiB,iBAAiB,CACnC,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,YAAY,CAAC;IAuHxB;;;;OAIG;WACiB,kBAAkB,CAAC,OAAO,EAAE;QAC9C,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B;gBAaS,OAAO,EAAE,OAAO,EAChB,gBAAgB,EAAE,OAAO,EACzB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE,gBAAgB,GAAG,IAAI,EACjC,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,QAAQ;IA4B5B,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,mBAAmB;IAc3B,OAAO,CAAC,sBAAsB,CAE5B;IAEF,OAAO,CAAC,aAAa,CAgBnB;IAEF,OAAO,CAAC,iBAAiB,CA2DvB;IAEF,OAAO,CAAC,oBAAoB,CA6D1B;IAEF,OAAO,CAAC,gBAAgB,CAYtB;IAEF,OAAO,CAAC,gBAAgB,CAStB;IAEF,OAAO,CAAC,gBAAgB,CA0BtB;IAEF,OAAO,CAAC,SAAS,CAuEf;IAEF,OAAO,CAAC,qBAAqB,CAQ3B;IAEF,OAAO,CAAC,qBAAqB,CAQ3B;IAEK,SAAS;YAIF,gBAAgB;IAG9B,OAAO,CAAC,YAAY;IAQpB;;;;;;OAMG;IAaU,UAAU;IASV,YAAY;IA8BZ,gBAAgB;IAqBtB,mBAAmB;IAInB,UAAU;IAIV,WAAW,CAAC,IAAI,EAAE,MAAM;IAuBxB,WAAW;CAsBnB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAIjD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAElB,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAEhB,cAAc,EACf,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAgBlD,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAKxC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,MAAM,MAAM,OAAO,GAAG,mBAAmB,GAAG,kBAAkB,GAAG,cAAc,CAAC;AAChF,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAC3E,MAAM,MAAM,cAAc,GAAG,iBAAiB,GAC5C,OAAO,CAAC,cAAc,CAAC,GACvB,gBAAgB,GAChB,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAC9B,qBAAa,YAAY;IAyMrB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,gBAAgB;IACxB,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,IAAI;IACZ,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,QAAQ;IAhNlB,OAAO,CAAC,MAAM,CAAwB;IACtC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAK;IAC7B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,sBAAsB,CAAS;IACvC,OAAO,CAAC,cAAc,CAAK;IAE3B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,UAAU,CAA2B;IAE7C,OAAO,CAAC,eAAe,CAAwB;IAE/C,OAAO,CAAC,mBAAmB,CAGlB;IAET,OAAO,CAAC,gBAAgB,CAAgB;IAExC,OAAO,CAAC,MAAM,CAAC,cAAc;IAgC7B;;;;OAIG;WACiB,iBAAiB,CACnC,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,YAAY,CAAC;IAuHxB;;;;OAIG;WACiB,kBAAkB,CAAC,OAAO,EAAE;QAC9C,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B;gBAaS,OAAO,EAAE,OAAO,EAChB,gBAAgB,EAAE,OAAO,EACzB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE,gBAAgB,GAAG,IAAI,EACjC,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,WAAW,EACxB,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,QAAQ;IA6B5B,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,mBAAmB;IAc3B,OAAO,CAAC,sBAAsB,CAE5B;IAEF,OAAO,CAAC,aAAa,CAgBnB;IAEF,OAAO,CAAC,iBAAiB,CAiEvB;IAEF,OAAO,CAAC,oBAAoB,CAuE1B;IAEF,OAAO,CAAC,gBAAgB,CAYtB;IAEF,OAAO,CAAC,gBAAgB,CAStB;IAEF,OAAO,CAAC,gBAAgB,CA0BtB;IAEF,OAAO,CAAC,SAAS,CA6Df;IAEF,OAAO,CAAC,qBAAqB,CAQ3B;IAEF,OAAO,CAAC,qBAAqB,CAQ3B;IAEK,SAAS;YAIF,gBAAgB;IAG9B,OAAO,CAAC,YAAY;IAQpB;;;;OAIG;IACU,YAAY;IA8CZ,UAAU;IASV,YAAY;IA8BZ,gBAAgB;IAqBtB,mBAAmB;IAInB,UAAU;IAIV,WAAW,CAAC,IAAI,EAAE,MAAM;IAuBxB,WAAW;CAsBnB"}