@unith-ai/core-client 2.0.5 → 2.0.6

package/dist/lib.web.js

@@ -503,6 +503,7 @@ var EventType = /* @__PURE__ */ ((EventType2) => {
   EventType2["PING"] = "ping";
   EventType2["PONG"] = "pong";
   EventType2["BINARY"] = "binary";
+  EventType2["STOP"] = "stop";
   return EventType2;
 })(EventType || {});
 var StreamingEventType = /* @__PURE__ */ ((StreamingEventType2) => {
@@ -511,6 +512,7 @@ var StreamingEventType = /* @__PURE__ */ ((StreamingEventType2) => {
   StreamingEventType2["METADATA"] = "metadata";
   StreamingEventType2["ERROR"] = "error";
   StreamingEventType2["CACHE"] = "cache";
+  StreamingEventType2["STOP"] = "stop";
   return StreamingEventType2;
 })(StreamingEventType || {});
 function isJoinEvent(event) {
@@ -596,6 +598,17 @@ var Connection = class _Connection {
       });
     });
     this.socket.addEventListener("close", (event) => {
+      if (this.onMessageCallback) {
+        const parsedEvent = {
+          event: "timeout" /* TIME_OUT */,
+          user_id: "",
+          username: ""
+        };
+        if (isValidSocketEvent(parsedEvent)) {
+          this.onMessageCallback(parsedEvent);
+          return;
+        }
+      }
       this.disconnect({
         reason: "error",
         message: event.reason || "The connection was closed by the server.",
@@ -700,6 +713,10 @@ var Connection = class _Connection {
     this.socket.close();
   }
   sendMessage(message) {
+    if (this.socket.readyState !== WebSocket.OPEN) {
+      console.warn("Attempted to send a message on a closed socket.");
+      return;
+    }
     this.socket.send(JSON.stringify(message));
   }
   sendPingEvent(message) {
@@ -707,6 +724,7 @@ var Connection = class _Connection {
       console.warn("Ping event sent without a callback set.");
       return;
     }
+    if (this.socket.readyState !== WebSocket.OPEN) return;
     this.socket.send(JSON.stringify(message));
   }
   onPingPong(callback) {
@@ -753,11 +771,11 @@ var IdleVideo = class _IdleVideo {
         );
       }
     } else {
-      const errorMessage = await response.text();
+      const errorMessage = await response.json();
       const error = new Error(
-        `An error occurred retrieving idle video: ${response.status} ${response.statusText}. Response: ${errorMessage}`
+        errorMessage.detail || "An error occurred retrieving your digital human."
       );
-      error.name = "IdleVideoError";
+      error.name = "headError";
       throw error;
     }
   }
@@ -774,11 +792,11 @@ var IdleVideo = class _IdleVideo {
         );
       }
     } else {
-      const errorMessage = await response.text();
+      const errorMessage = await response.json();
       const error = new Error(
-        `An error occurred retrieving avatar: ${response.status} ${response.statusText}. Response: ${errorMessage}`
+        errorMessage.detail || "An error occurred retrieving your digital human."
       );
-      error.name = "AvatarError";
+      error.name = "headError";
       throw error;
     }
   }
@@ -795,11 +813,11 @@ var IdleVideo = class _IdleVideo {
         );
       }
     } else {
-      const errorMessage = await response.text();
+      const errorMessage = await response.json();
       const error = new Error(
-        `An error occurred retrieving idle video: ${response.status} ${response.statusText}. Response: ${errorMessage}`
+        errorMessage.detail || "An error occurred retrieving your digital human."
       );
-      error.name = "IdleVideoError";
+      error.name = "headError";
       throw error;
     }
   }
@@ -25406,15 +25424,54 @@ var N = class _N {
 };
 N.DEFAULT_BASE_URI = "wss://api.elevenlabs.io";
 
+// src/utils/storage.ts
+var storage = window.localStorage;
+var location = window.location.origin + window.location.pathname;
+var getFromStorage = (key, orgId, headId) => {
+  if (typeof storage !== "undefined") {
+    const prefix = `chat:${location}:${orgId}:${headId}:`;
+    return storage.getItem(`${prefix}${key}`);
+  }
+};
+var setToStorage = (key, val, orgId, headId) => {
+  if (typeof storage !== "undefined") {
+    const prefix = `chat:${location}:${orgId}:${headId}:`;
+    storage.setItem(`${prefix}${key}`, val);
+    return;
+  }
+};
+
+// src/utils/microphone.ts
+var EXPIRATION_OFFSET = 5 * 60 * 1e3;
+var defaultElevenLabsOptions = {
+  vadSilenceThresholdSecs: 1.5,
+  noiseSuppression: true,
+  vadThreshold: 0.4,
+  minSpeechDurationMs: 100,
+  minSilenceDurationMs: 100,
+  disableDynamicSpeechRecognition: false
+};
+function getLanguageCode(languageTag) {
+  if (!languageTag || typeof languageTag !== "string") {
+    throw new Error("Invalid language tag");
+  }
+  const parts = languageTag.trim().split("-");
+  const languageCode = parts[0].toLowerCase();
+  if (!/^[a-z]{2,3}$/.test(languageCode)) {
+    throw new Error("Invalid ISO 639 language code format");
+  }
+  return languageCode;
+}
+
 // src/modules/microphone.ts
 var Microphone = class _Microphone {
-  constructor(provider, options, elevenLabsOptions, user, headInfo, microphoneAccess, token, sendMessage) {
+  constructor(provider, options, elevenLabsOptions, user, headInfo, microphoneAccess, token, voiceInterruptions) {
     this.provider = provider;
     this.options = options;
     this.elevenLabsOptions = elevenLabsOptions;
     this.user = user;
     this.headInfo = headInfo;
-    this.sendMessage = sendMessage;
+    this.voiceInterruptions = voiceInterruptions;
     this.microphoneStatus = "OFF";
     this.microphoneAccess = false;
     this.recognizer = null;
@@ -25447,7 +25504,7 @@ var Microphone = class _Microphone {
    * @param sendMessage
    * @returns
    */
-  static async initializeMicrophone(options, provider, elevenLabsOptions, user, headInfo, microphoneAccess, sendMessage) {
+  static async initializeMicrophone(options, provider, elevenLabsOptions, user, headInfo, microphoneAccess, voiceInterruptions) {
     let token = {
       token: "",
       generatedAt: null,
@@ -25469,7 +25526,7 @@ var Microphone = class _Microphone {
       headInfo,
       microphoneAccess,
       token,
-      sendMessage
+      voiceInterruptions
     );
   }
   updateMicrophoneStatus(status) {
@@ -25499,13 +25556,33 @@ var Microphone = class _Microphone {
     const sdk = speechSdk;
     let SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar;
     if (sdk.SpeechConfig) {
-      ({ SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar } = sdk);
+      ({
+        SpeechConfig,
+        AudioConfig,
+        SpeechRecognizer,
+        ResultReason,
+        PhraseListGrammar
+      } = sdk);
     } else if ((_a = sdk.default) == null ? void 0 : _a.SpeechConfig) {
-      ({ SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar } = sdk.default);
+      ({
+        SpeechConfig,
+        AudioConfig,
+        SpeechRecognizer,
+        ResultReason,
+        PhraseListGrammar
+      } = sdk.default);
     } else if ((_b = sdk.SpeechSDK) == null ? void 0 : _b.SpeechConfig) {
-      ({ SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar } = sdk.SpeechSDK);
+      ({
+        SpeechConfig,
+        AudioConfig,
+        SpeechRecognizer,
+        ResultReason,
+        PhraseListGrammar
+      } = sdk.SpeechSDK);
     } else {
-      throw new Error("Azure Speech SDK does not expose expected symbols. Ensure microsoft-cognitiveservices-speech-sdk is installed.");
+      throw new Error(
+        "Azure Speech SDK does not expose expected symbols. Ensure microsoft-cognitiveservices-speech-sdk is installed."
+      );
     }
     const speechConfig = SpeechConfig.fromAuthorizationToken(
       this.tokenObj.token,
@@ -25528,6 +25605,12 @@ var Microphone = class _Microphone {
         });
       }
     };
+    this.recognizer.recognizing = (_3, event) => {
+      if (!this.voiceInterruptions || event.result.text.length < 2) return;
+      this.options.onMicrophonePartialSpeechRecognitionResult({
+        transcript: event.result.text
+      });
+    };
     this.recognizer.startContinuousRecognitionAsync(
       () => {
         this.micBeep.play();
@@ -25565,11 +25648,10 @@ var Microphone = class _Microphone {
         generatedAt: Date.now()
       };
     }
-    this.connection = N.connect({
+    const params = {
       token: this.tokenObj.token,
       modelId: "scribe_v2_realtime",
       includeTimestamps: false,
-      // languageCode: getLanguageCode(this.headInfo.lang_speech_recognition),
       microphone: {
         echoCancellation: true,
         noiseSuppression: this.elevenLabsOptions.noiseSuppression
@@ -25579,7 +25661,13 @@ var Microphone = class _Microphone {
       vadThreshold: this.elevenLabsOptions.vadThreshold,
       minSpeechDurationMs: this.elevenLabsOptions.minSpeechDurationMs,
       minSilenceDurationMs: this.elevenLabsOptions.minSilenceDurationMs
-    });
+    };
+    if (this.elevenLabsOptions.disableDynamicSpeechRecognition) {
+      params.languageCode = getLanguageCode(
+        this.headInfo.lang_speech_recognition || "en-US"
+      );
+    }
+    this.connection = N.connect(params);
     this.connection.on(P.SESSION_STARTED, () => {
       this.micBeep.play();
       this.updateMicrophoneStatus("ON");
@@ -25588,6 +25676,13 @@ var Microphone = class _Microphone {
       if (data.text.length < 2 || data.text[0] === "(") return;
       this.handleRecognitionResult(data.text);
     });
+    this.connection.on(P.PARTIAL_TRANSCRIPT, (data) => {
+      if (!this.voiceInterruptions || data.text.length < 2 || data.text[0] === "(")
+        return;
+      this.options.onMicrophonePartialSpeechRecognitionResult({
+        transcript: data.text
+      });
+    });
     this.connection.on(P.ERROR, (error) => {
       console.error("Error:", error);
       this.options.onMicrophoneError({
@@ -25769,23 +25864,6 @@ var SyncController = class {
   }
 };
 
-// src/utils/storage.ts
-var storage = window.localStorage;
-var location = window.location.origin + window.location.pathname;
-var getFromStorage = (key, orgId, headId) => {
-  if (typeof storage !== "undefined") {
-    const prefix = `chat:${location}:${orgId}:${headId}:`;
-    return storage.getItem(`${prefix}${key}`);
-  }
-};
-var setToStorage = (key, val, orgId, headId) => {
-  if (typeof storage !== "undefined") {
-    const prefix = `chat:${location}:${orgId}:${headId}:`;
-    storage.setItem(`${prefix}${key}`, val);
-    return;
-  }
-};
-
 // src/modules/user.ts
 var User = class _User {
   constructor(accessToken, tokenType, id, username, password, orgId, headId, apiBase) {
@@ -26173,7 +26251,7 @@ var Vp8VideoOutput = class _Vp8VideoOutput {
       cancelAnimationFrame(this.animationFrameId);
       this.animationFrameId = null;
     }
-    this.frameBuffer = [];
+    this.frameBuffer.splice(0);
     this.isProcessingFrame = false;
     this.currentSequenceId = 0;
     this.startTime = 0;
@@ -26181,35 +26259,8 @@ var Vp8VideoOutput = class _Vp8VideoOutput {
       count: 0,
       lastTime: 0
     };
-    if (this.decoder && this.decoder.state === "configured") {
-      try {
-        this.decoder.flush();
-      } catch (error) {
-        console.warn("Error flushing decoder:", error);
-      }
-    }
-    this.clearCanvas();
     this.state = "ready" /* READY */;
-  }
-  interrupt(fadeOut = false) {
-    if (this.state === "destroyed" /* DESTROYED */) return;
-    this.state = "interrupted" /* INTERRUPTED */;
-    this.frameBuffer = [];
-    this.isProcessingFrame = false;
-    if (this.decoder && this.decoder.state === "configured") {
-      try {
-        this.decoder.flush();
-      } catch (error) {
-        console.warn("Error flushing decoder:", error);
-      }
-    }
-    if (fadeOut) {
-      this.fadeOutCanvas().then(() => {
-        this.clearCanvas();
-      });
-    } else {
-      this.clearCanvas();
-    }
+    return;
   }
   destroy() {
     if (this.state === "destroyed" /* DESTROYED */) return;
@@ -26315,22 +26366,6 @@ var Vp8VideoOutput = class _Vp8VideoOutput {
   clearCanvas() {
     this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
   }
-  async fadeOutCanvas() {
-    return new Promise((resolve) => {
-      let opacity = 1;
-      const fadeStep = () => {
-        opacity -= 0.05;
-        this.canvas.style.opacity = opacity.toString();
-        if (opacity <= 0) {
-          this.canvas.style.opacity = "1";
-          resolve();
-        } else {
-          requestAnimationFrame(fadeStep);
-        }
-      };
-      fadeStep();
-    });
-  }
   setupContextLossHandling() {
     this.handleContextLoss = (event) => {
       event.preventDefault();
@@ -26365,6 +26400,7 @@ var VideoOutput = class _VideoOutput {
     this.bufferCheckAnimationId = null;
     this.lastBufferCheckTime = 0;
     this.sessionStarted = false;
+    this.isRestoringIdleVideo = false;
     //cached video
     this.isShowingCachedVideo = false;
     this.onSpeakingStartCallback = null;
@@ -26468,7 +26504,7 @@ var VideoOutput = class _VideoOutput {
     this.lastBufferCheckTime = 0;
     const checkBuffer = (timestamp) => {
       if (timestamp - this.lastBufferCheckTime >= 100) {
-        if (this.sessionStarted && this.videoOutput.getBufferLength() > 0) {
+        if (this.sessionStarted && this.videoOutput.getBufferLength() > 0 && !this.isRestoringIdleVideo) {
           this.hideIdleVideoBeforeStream();
         } else if (!this.videoOutput.getStreamingStatus() && this.videoOutput.getBufferLength() === 0) {
           this.showIdleVideoAfterStream();
@@ -26512,7 +26548,20 @@ var VideoOutput = class _VideoOutput {
         { once: true }
       );
     } catch (error) {
-      console.error("Failed to play cached video:", error);
+      this.cachedVideo.style.opacity = "0";
+      this.isShowingCachedVideo = false;
+      await this.showIdleVideo();
+    }
+  }
+  async stopCachedVideo() {
+    if (!this.isShowingCachedVideo || !this.cachedVideo || this.isTransitioning)
+      return;
+    this.isShowingCachedVideo = false;
+    try {
+      await this.crossfadeFromCachedToIdle();
+      await this.cleanupCachedVideo();
+      await this.showIdleVideo();
+    } catch (error) {
       this.cachedVideo.style.opacity = "0";
       this.isShowingCachedVideo = false;
       await this.showIdleVideo();
@@ -26587,6 +26636,9 @@ var VideoOutput = class _VideoOutput {
   getStreamingStatus() {
     return this.videoOutput.getStreamingStatus();
   }
+  isPlayingCachedVideo() {
+    return this.isShowingCachedVideo;
+  }
   async showIdleVideo() {
     var _a, _b;
     if (!this.idleVideo || this.isShowingIdleVideo) return;
@@ -26623,15 +26675,16 @@ var VideoOutput = class _VideoOutput {
     return this.videoOutput.addFrame(uint8Array, timeStamp, isKeyframe);
   }
   clearFrame() {
-    this.showIdleVideo();
-    return this.videoOutput.clearFrame();
+    this.isRestoringIdleVideo = true;
+    this.showIdleVideoAfterStream();
+    this.videoOutput.clearFrame();
   }
   async toggleStream(status) {
+    if (status) {
+      this.isRestoringIdleVideo = false;
+    }
     return this.videoOutput.toggleStream(status);
   }
-  interrupt(fadeOut) {
-    this.videoOutput.interrupt(fadeOut);
-  }
   destroy() {
     if (this.bufferCheckAnimationId) {
       cancelAnimationFrame(this.bufferCheckAnimationId);
@@ -26655,16 +26708,6 @@ var VideoOutput = class _VideoOutput {
   }
 };
 
-// src/utils/microphone.ts
-var EXPIRATION_OFFSET = 5 * 60 * 1e3;
-var defaultElevenLabsOptions = {
-  vadSilenceThresholdSecs: 1.5,
-  noiseSuppression: true,
-  vadThreshold: 0.4,
-  minSpeechDurationMs: 100,
-  minSilenceDurationMs: 100
-};
-
 // src/utils/sync.ts
 var DEFAULT_SYNC_CONFIG = {
   tolerance: 40,
@@ -26705,6 +26748,7 @@ var Conversation = class _Conversation {
     this.status = "connecting";
     this.volume = 1;
     this.sessionStarted = false;
+    this.isStoppingLastResponse = false;
     this.messageCounter = 0;
     this.monitor = null;
     this.microphone = null;
@@ -26723,7 +26767,7 @@ var Conversation = class _Conversation {
       });
       if ("suggestions" in event) {
         const suggestions = event.suggestions || [];
-        if (suggestions.length > 1) {
+        if (suggestions.length > 0) {
           this.suggestionsQueue = suggestions;
         }
       }
@@ -26733,7 +26777,8 @@ var Conversation = class _Conversation {
       if (!errorType || ![
         "resource_exhausted",
         "deadline_exceeded",
-        "inactivity_timeout"
+        "inactivity_timeout",
+        "canceled"
       ].includes(errorType)) {
         this.options.onError({
           message: "A connection error occurred. Please try again.",
@@ -26742,21 +26787,12 @@ var Conversation = class _Conversation {
         });
         return;
       }
+      if (errorType === "canceled") {
+        return;
+      }
       if (errorType === "resource_exhausted") {
-        if (this.avController.isPlaying) {
-          this.options.onError({
-            message: "We are experiencing heavy traffic and we can't deliver the response, try again",
-            endConversation: false,
-            type: "toast"
-          });
-        } else {
-          this.options.onError({
-            message: "The system is experiencing heavy traffic. Please try again later.",
-            endConversation: true,
-            type: "toast"
-          });
-          return;
-        }
+        this.options.onHighDemand();
+        return;
       } else if (errorType === "deadline_exceeded") {
         this.options.onError({
           message: "We are experiencing heavy traffic and we can't deliver the response, try again",
@@ -26817,6 +26853,15 @@ var Conversation = class _Conversation {
         if (this.avController.isStoppingAV) return;
         this.handleVideoFrame(event);
       }
+      if (event.type === "stop") {
+        this.videoOutput.clearFrame();
+        this.audioOutput.worklet.port.postMessage({
+          type: "stopPlayback"
+        });
+        this.options.onStoppingEnd();
+        this.isStoppingLastResponse = false;
+        return;
+      }
       this.avController.playAudioVideo();
     };
     this.handleVideoFrame = async (event) => {
@@ -26942,7 +26987,7 @@ var Conversation = class _Conversation {
     this.options.onConnect({
       userId: connection.userId,
       headInfo: {
-        name: this.headInfo.name,
+        name: this.headInfo.alias,
         phrases: this.headInfo.phrases,
         language: this.headInfo.language,
         avatar: this.headInfo.avatarSrc
@@ -26971,6 +27016,8 @@ var Conversation = class _Conversation {
         onMicrophoneStatusChange: () => {
         },
         onMicrophoneSpeechRecognitionResult: () => {
+        },
+        onMicrophonePartialSpeechRecognitionResult: () => {
         }
       },
       elevenLabsOptions: defaultElevenLabsOptions,
@@ -26992,6 +27039,8 @@ var Conversation = class _Conversation {
       },
       onStoppingEnd: () => {
       },
+      onStoppingStart: () => {
+      },
       onTimeout: () => {
       },
       onTimeoutWarning: () => {
@@ -27000,6 +27049,8 @@ var Conversation = class _Conversation {
       },
       onError: () => {
       },
+      onHighDemand: () => {
+      },
       ...partialOptions
     };
   }
@@ -27163,23 +27214,47 @@ var Conversation = class _Conversation {
     }
   }
   /**
-   * To stop current response, we'll do the following:
-   * 1. set a flag that'll prevent adding new audio & video events to their respective queues
-   * 2. clear video queue & switch state to idle
-   * 3. clear audio queue
-   * 4. send an event if all expected response from BE has been received. If not, FE will keep status in 'stopping' mode
+   * To stop streaming response, we'll send a stop message to the BE. The BE will then stop sending audio and video frames, which will naturally end the response. This is more efficient and leads to a better user experience as it allows for a smoother transition when stopping the response.
+   *
+   * To stop cached response, we just stop the video and fade to idle.
    */
-  // public stopCurrentResponse() {
-  //   this.avController.toggleStoppingVideo(true);
-  //   this.videoOutput.clearFrame();
-  //   this.audioOutput.worklet.port.postMessage({
-  //     type: "stopPlayback",
-  //   });
-  //   if (!this.videoOutput.getStreamingStatus()) {
-  //     this.options.onStoppingEnd();
-  //     this.avController.toggleStoppingVideo(false);
-  //   }
-  // }
+  async stopResponse() {
+    if (this.isStoppingLastResponse) {
+      return;
+    }
+    this.isStoppingLastResponse = true;
+    this.options.onStoppingStart();
+    if (!this.connection) {
+      this.isStoppingLastResponse = false;
+      this.options.onStoppingEnd();
+      throw new Error("Connection not established");
+    }
+    if (this.videoOutput.isPlayingCachedVideo()) {
+      await this.videoOutput.stopCachedVideo();
+      this.isStoppingLastResponse = false;
+      this.options.onStoppingEnd();
+      return;
+    }
+    if (!this.avController.isPlaying) {
+      this.isStoppingLastResponse = false;
+      this.options.onStoppingEnd();
+      return;
+    }
+    const sessionId = `${this.user.id}::${this.user.orgId}::${this.user.headId}::${this.user.sessionId.toString().padStart(5, "0")}`;
+    const message = {
+      id: this.messageCounter,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      speaker: "user",
+      text: "",
+      isSent: false,
+      user_id: this.user.id,
+      username: this.user.username,
+      event: "stop" /* STOP */,
+      visible: true,
+      session_id: sessionId
+    };
+    this.connection.sendMessage(message);
+  }
   async toggleMute() {
     this.volume = this.volume === 0 ? 1 : 0;
     this.audioOutput.toggleMute();
@@ -27188,7 +27263,7 @@ var Conversation = class _Conversation {
     return this.volume;
   }
   async startSession() {
-    var _a;
+    var _a, _b;
     this.sessionStarted = true;
     if (this.audioOutput.context.state === "suspended") {
       await this.audioOutput.context.resume();
@@ -27208,13 +27283,13 @@ var Conversation = class _Conversation {
         this.user,
         this.headInfo,
         this.microphoneAccess,
-        this.sendMessage
+        (_b = this.options.voiceInterruptions) != null ? _b : false
       );
     }
     return this.connection;
   }
   async toggleMicrophone() {
-    var _a;
+    var _a, _b;
     if (this.options.microphoneProvider === "custom") {
       throw new Error("Cannot toggle microphone for custom provider.");
     }
@@ -27226,7 +27301,7 @@ var Conversation = class _Conversation {
         this.user,
         this.headInfo,
         this.microphoneAccess,
-        this.sendMessage
+        (_b = this.options.voiceInterruptions) != null ? _b : false
       );
     }
     if (!this.microphoneAccess && this.microphone.status() === "OFF") {
@@ -27284,6 +27359,7 @@ export {
   EventType,
   StreamingEventType,
   VideoTransitionType,
+  defaultElevenLabsOptions,
   isBinaryEvent,
   isConversationEndEvent,
   isJoinEvent,