@unith-ai/core-client 2.0.4-beta.5 → 2.0.5-beta.1

package/dist/lib.web.js

@@ -503,6 +503,7 @@ var EventType = /* @__PURE__ */ ((EventType2) => {
   EventType2["PING"] = "ping";
   EventType2["PONG"] = "pong";
   EventType2["BINARY"] = "binary";
+  EventType2["STOP"] = "stop";
   return EventType2;
 })(EventType || {});
 var StreamingEventType = /* @__PURE__ */ ((StreamingEventType2) => {
@@ -511,6 +512,7 @@ var StreamingEventType = /* @__PURE__ */ ((StreamingEventType2) => {
   StreamingEventType2["METADATA"] = "metadata";
   StreamingEventType2["ERROR"] = "error";
   StreamingEventType2["CACHE"] = "cache";
+  StreamingEventType2["STOP"] = "stop";
   return StreamingEventType2;
 })(StreamingEventType || {});
 function isJoinEvent(event) {
@@ -25406,6 +25408,45 @@ var N = class _N {
 };
 N.DEFAULT_BASE_URI = "wss://api.elevenlabs.io";
 
+// src/utils/storage.ts
+var storage = window.localStorage;
+var location = window.location.origin + window.location.pathname;
+var getFromStorage = (key, orgId, headId) => {
+  if (typeof storage !== "undefined") {
+    const prefix = `chat:${location}:${orgId}:${headId}:`;
+    return storage.getItem(`${prefix}${key}`);
+  }
+};
+var setToStorage = (key, val, orgId, headId) => {
+  if (typeof storage !== "undefined") {
+    const prefix = `chat:${location}:${orgId}:${headId}:`;
+    storage.setItem(`${prefix}${key}`, val);
+    return;
+  }
+};
+
+// src/utils/microphone.ts
+var EXPIRATION_OFFSET = 5 * 60 * 1e3;
+var defaultElevenLabsOptions = {
+  vadSilenceThresholdSecs: 1.5,
+  noiseSuppression: true,
+  vadThreshold: 0.4,
+  minSpeechDurationMs: 100,
+  minSilenceDurationMs: 100,
+  disableDynamicSpeechRecognition: false
+};
+function getLanguageCode(languageTag) {
+  if (!languageTag || typeof languageTag !== "string") {
+    throw new Error("Invalid language tag");
+  }
+  const parts = languageTag.trim().split("-");
+  const languageCode = parts[0].toLowerCase();
+  if (!/^[a-z]{2,3}$/.test(languageCode)) {
+    throw new Error("Invalid ISO 639 language code format");
+  }
+  return languageCode;
+}
+
 // src/modules/microphone.ts
 var Microphone = class _Microphone {
   constructor(provider, options, elevenLabsOptions, user, headInfo, microphoneAccess, token, sendMessage) {
@@ -25499,13 +25540,33 @@ var Microphone = class _Microphone {
     const sdk = speechSdk;
     let SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar;
     if (sdk.SpeechConfig) {
-      ({ SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar } = sdk);
+      ({
+        SpeechConfig,
+        AudioConfig,
+        SpeechRecognizer,
+        ResultReason,
+        PhraseListGrammar
+      } = sdk);
     } else if ((_a = sdk.default) == null ? void 0 : _a.SpeechConfig) {
-      ({ SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar } = sdk.default);
+      ({
+        SpeechConfig,
+        AudioConfig,
+        SpeechRecognizer,
+        ResultReason,
+        PhraseListGrammar
+      } = sdk.default);
     } else if ((_b = sdk.SpeechSDK) == null ? void 0 : _b.SpeechConfig) {
-      ({ SpeechConfig, AudioConfig, SpeechRecognizer, ResultReason, PhraseListGrammar } = sdk.SpeechSDK);
+      ({
+        SpeechConfig,
+        AudioConfig,
+        SpeechRecognizer,
+        ResultReason,
+        PhraseListGrammar
+      } = sdk.SpeechSDK);
     } else {
-      throw new Error("Azure Speech SDK does not expose expected symbols. Ensure microsoft-cognitiveservices-speech-sdk is installed.");
+      throw new Error(
+        "Azure Speech SDK does not expose expected symbols. Ensure microsoft-cognitiveservices-speech-sdk is installed."
+      );
     }
     const speechConfig = SpeechConfig.fromAuthorizationToken(
       this.tokenObj.token,
@@ -25528,6 +25589,12 @@ var Microphone = class _Microphone {
         });
       }
     };
+    this.recognizer.recognizing = (_3, event) => {
+      if (event.result.text.length < 2) return;
+      this.options.onMicrophonePartialSpeechRecognitionResult({
+        transcript: event.result.text
+      });
+    };
     this.recognizer.startContinuousRecognitionAsync(
       () => {
         this.micBeep.play();
@@ -25565,11 +25632,10 @@ var Microphone = class _Microphone {
         generatedAt: Date.now()
       };
     }
-    this.connection = N.connect({
+    const params = {
       token: this.tokenObj.token,
       modelId: "scribe_v2_realtime",
       includeTimestamps: false,
-      // languageCode: getLanguageCode(this.headInfo.lang_speech_recognition),
       microphone: {
         echoCancellation: true,
         noiseSuppression: this.elevenLabsOptions.noiseSuppression
@@ -25579,7 +25645,13 @@ var Microphone = class _Microphone {
       vadThreshold: this.elevenLabsOptions.vadThreshold,
       minSpeechDurationMs: this.elevenLabsOptions.minSpeechDurationMs,
       minSilenceDurationMs: this.elevenLabsOptions.minSilenceDurationMs
-    });
+    };
+    if (this.elevenLabsOptions.disableDynamicSpeechRecognition) {
+      params.languageCode = getLanguageCode(
+        this.headInfo.lang_speech_recognition || "en-US"
+      );
+    }
+    this.connection = N.connect(params);
     this.connection.on(P.SESSION_STARTED, () => {
       this.micBeep.play();
       this.updateMicrophoneStatus("ON");
@@ -25588,6 +25660,12 @@ var Microphone = class _Microphone {
       if (data.text.length < 2 || data.text[0] === "(") return;
       this.handleRecognitionResult(data.text);
     });
+    this.connection.on(P.PARTIAL_TRANSCRIPT, (data) => {
+      if (data.text.length < 2 || data.text[0] === "(") return;
+      this.options.onMicrophonePartialSpeechRecognitionResult({
+        transcript: data.text
+      });
+    });
     this.connection.on(P.ERROR, (error) => {
       console.error("Error:", error);
       this.options.onMicrophoneError({
@@ -25769,23 +25847,6 @@ var SyncController = class {
   }
 };
 
-// src/utils/storage.ts
-var storage = window.localStorage;
-var location = window.location.origin + window.location.pathname;
-var getFromStorage = (key, orgId, headId) => {
-  if (typeof storage !== "undefined") {
-    const prefix = `chat:${location}:${orgId}:${headId}:`;
-    return storage.getItem(`${prefix}${key}`);
-  }
-};
-var setToStorage = (key, val, orgId, headId) => {
-  if (typeof storage !== "undefined") {
-    const prefix = `chat:${location}:${orgId}:${headId}:`;
-    storage.setItem(`${prefix}${key}`, val);
-    return;
-  }
-};
-
 // src/modules/user.ts
 var User = class _User {
   constructor(accessToken, tokenType, id, username, password, orgId, headId, apiBase) {
@@ -26173,7 +26234,7 @@ var Vp8VideoOutput = class _Vp8VideoOutput {
       cancelAnimationFrame(this.animationFrameId);
       this.animationFrameId = null;
     }
-    this.frameBuffer = [];
+    this.frameBuffer.splice(0);
     this.isProcessingFrame = false;
     this.currentSequenceId = 0;
     this.startTime = 0;
@@ -26181,35 +26242,8 @@ var Vp8VideoOutput = class _Vp8VideoOutput {
       count: 0,
       lastTime: 0
     };
-    if (this.decoder && this.decoder.state === "configured") {
-      try {
-        this.decoder.flush();
-      } catch (error) {
-        console.warn("Error flushing decoder:", error);
-      }
-    }
-    this.clearCanvas();
     this.state = "ready" /* READY */;
-  }
-  interrupt(fadeOut = false) {
-    if (this.state === "destroyed" /* DESTROYED */) return;
-    this.state = "interrupted" /* INTERRUPTED */;
-    this.frameBuffer = [];
-    this.isProcessingFrame = false;
-    if (this.decoder && this.decoder.state === "configured") {
-      try {
-        this.decoder.flush();
-      } catch (error) {
-        console.warn("Error flushing decoder:", error);
-      }
-    }
-    if (fadeOut) {
-      this.fadeOutCanvas().then(() => {
-        this.clearCanvas();
-      });
-    } else {
-      this.clearCanvas();
-    }
+    return;
   }
   destroy() {
     if (this.state === "destroyed" /* DESTROYED */) return;
@@ -26315,22 +26349,6 @@ var Vp8VideoOutput = class _Vp8VideoOutput {
   clearCanvas() {
     this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
   }
-  async fadeOutCanvas() {
-    return new Promise((resolve) => {
-      let opacity = 1;
-      const fadeStep = () => {
-        opacity -= 0.05;
-        this.canvas.style.opacity = opacity.toString();
-        if (opacity <= 0) {
-          this.canvas.style.opacity = "1";
-          resolve();
-        } else {
-          requestAnimationFrame(fadeStep);
-        }
-      };
-      fadeStep();
-    });
-  }
   setupContextLossHandling() {
     this.handleContextLoss = (event) => {
       event.preventDefault();
@@ -26365,6 +26383,7 @@ var VideoOutput = class _VideoOutput {
     this.bufferCheckAnimationId = null;
     this.lastBufferCheckTime = 0;
     this.sessionStarted = false;
+    this.isRestoringIdleVideo = false;
     //cached video
     this.isShowingCachedVideo = false;
     this.onSpeakingStartCallback = null;
@@ -26468,7 +26487,7 @@ var VideoOutput = class _VideoOutput {
     this.lastBufferCheckTime = 0;
     const checkBuffer = (timestamp) => {
       if (timestamp - this.lastBufferCheckTime >= 100) {
-        if (this.sessionStarted && this.videoOutput.getBufferLength() > 0) {
+        if (this.sessionStarted && this.videoOutput.getBufferLength() > 0 && !this.isRestoringIdleVideo) {
           this.hideIdleVideoBeforeStream();
         } else if (!this.videoOutput.getStreamingStatus() && this.videoOutput.getBufferLength() === 0) {
           this.showIdleVideoAfterStream();
@@ -26512,7 +26531,20 @@ var VideoOutput = class _VideoOutput {
         { once: true }
       );
     } catch (error) {
-      console.error("Failed to play cached video:", error);
+      this.cachedVideo.style.opacity = "0";
+      this.isShowingCachedVideo = false;
+      await this.showIdleVideo();
+    }
+  }
+  async stopCachedVideo() {
+    if (!this.isShowingCachedVideo || !this.cachedVideo || this.isTransitioning)
+      return;
+    this.isShowingCachedVideo = false;
+    try {
+      await this.crossfadeFromCachedToIdle();
+      await this.cleanupCachedVideo();
+      await this.showIdleVideo();
+    } catch (error) {
       this.cachedVideo.style.opacity = "0";
       this.isShowingCachedVideo = false;
       await this.showIdleVideo();
@@ -26587,6 +26619,9 @@ var VideoOutput = class _VideoOutput {
   getStreamingStatus() {
     return this.videoOutput.getStreamingStatus();
   }
+  isPlayingCachedVideo() {
+    return this.isShowingCachedVideo;
+  }
   async showIdleVideo() {
     var _a, _b;
     if (!this.idleVideo || this.isShowingIdleVideo) return;
@@ -26623,15 +26658,16 @@ var VideoOutput = class _VideoOutput {
     return this.videoOutput.addFrame(uint8Array, timeStamp, isKeyframe);
   }
   clearFrame() {
-    this.showIdleVideo();
-    return this.videoOutput.clearFrame();
+    this.isRestoringIdleVideo = true;
+    this.showIdleVideoAfterStream();
+    this.videoOutput.clearFrame();
   }
   async toggleStream(status) {
+    if (status) {
+      this.isRestoringIdleVideo = false;
+    }
     return this.videoOutput.toggleStream(status);
   }
-  interrupt(fadeOut) {
-    this.videoOutput.interrupt(fadeOut);
-  }
   destroy() {
     if (this.bufferCheckAnimationId) {
       cancelAnimationFrame(this.bufferCheckAnimationId);
@@ -26655,16 +26691,6 @@ var VideoOutput = class _VideoOutput {
   }
 };
 
-// src/utils/microphone.ts
-var EXPIRATION_OFFSET = 5 * 60 * 1e3;
-var defaultElevenLabsOptions = {
-  vadSilenceThresholdSecs: 1.5,
-  noiseSuppression: true,
-  vadThreshold: 0.4,
-  minSpeechDurationMs: 100,
-  minSilenceDurationMs: 100
-};
-
 // src/utils/sync.ts
 var DEFAULT_SYNC_CONFIG = {
   tolerance: 40,
@@ -26705,6 +26731,7 @@ var Conversation = class _Conversation {
     this.status = "connecting";
     this.volume = 1;
     this.sessionStarted = false;
+    this.isStoppingLastResponse = false;
     this.messageCounter = 0;
     this.monitor = null;
     this.microphone = null;
@@ -26723,7 +26750,7 @@ var Conversation = class _Conversation {
       });
       if ("suggestions" in event) {
         const suggestions = event.suggestions || [];
-        if (suggestions.length > 1) {
+        if (suggestions.length > 0) {
           this.suggestionsQueue = suggestions;
         }
       }
@@ -26733,7 +26760,8 @@ var Conversation = class _Conversation {
       if (!errorType || ![
         "resource_exhausted",
         "deadline_exceeded",
-        "inactivity_timeout"
+        "inactivity_timeout",
+        "canceled"
       ].includes(errorType)) {
         this.options.onError({
           message: "A connection error occurred. Please try again.",
@@ -26742,6 +26770,9 @@ var Conversation = class _Conversation {
         });
         return;
       }
+      if (errorType === "canceled") {
+        return;
+      }
       if (errorType === "resource_exhausted") {
         if (this.avController.isPlaying) {
           this.options.onError({
@@ -26817,6 +26848,15 @@ var Conversation = class _Conversation {
         if (this.avController.isStoppingAV) return;
         this.handleVideoFrame(event);
       }
+      if (event.type === "stop") {
+        this.videoOutput.clearFrame();
+        this.audioOutput.worklet.port.postMessage({
+          type: "stopPlayback"
+        });
+        this.options.onStoppingEnd();
+        this.isStoppingLastResponse = false;
+        return;
+      }
       this.avController.playAudioVideo();
     };
     this.handleVideoFrame = async (event) => {
@@ -26942,7 +26982,7 @@ var Conversation = class _Conversation {
     this.options.onConnect({
       userId: connection.userId,
       headInfo: {
-        name: this.headInfo.name,
+        name: this.headInfo.alias,
         phrases: this.headInfo.phrases,
         language: this.headInfo.language,
         avatar: this.headInfo.avatarSrc
@@ -26971,6 +27011,8 @@ var Conversation = class _Conversation {
         onMicrophoneStatusChange: () => {
         },
         onMicrophoneSpeechRecognitionResult: () => {
+        },
+        onMicrophonePartialSpeechRecognitionResult: () => {
         }
       },
       elevenLabsOptions: defaultElevenLabsOptions,
@@ -26992,6 +27034,8 @@ var Conversation = class _Conversation {
       },
       onStoppingEnd: () => {
       },
+      onStoppingStart: () => {
+      },
       onTimeout: () => {
       },
       onTimeoutWarning: () => {
@@ -27163,23 +27207,47 @@ var Conversation = class _Conversation {
     }
   }
   /**
-   * To stop current response, we'll do the following:
-   * 1. set a flag that'll prevent adding new audio & video events to their respective queues
-   * 2. clear video queue & switch state to idle
-   * 3. clear audio queue
-   * 4. send an event if all expected response from BE has been received. If not, FE will keep status in 'stopping' mode
+   * To stop streaming response, we'll send a stop message to the BE. The BE will then stop sending audio and video frames, which will naturally end the response. This is more efficient and leads to a better user experience as it allows for a smoother transition when stopping the response.
+   *
+   * To stop cached response, we just stop the video and fade to idle.
    */
-  // public stopCurrentResponse() {
-  //   this.avController.toggleStoppingVideo(true);
-  //   this.videoOutput.clearFrame();
-  //   this.audioOutput.worklet.port.postMessage({
-  //     type: "stopPlayback",
-  //   });
-  //   if (!this.videoOutput.getStreamingStatus()) {
-  //     this.options.onStoppingEnd();
-  //     this.avController.toggleStoppingVideo(false);
-  //   }
-  // }
+  async stopResponse() {
+    if (this.isStoppingLastResponse) {
+      return;
+    }
+    this.isStoppingLastResponse = true;
+    this.options.onStoppingStart();
+    if (!this.connection) {
+      this.isStoppingLastResponse = false;
+      this.options.onStoppingEnd();
+      throw new Error("Connection not established");
+    }
+    if (this.videoOutput.isPlayingCachedVideo()) {
+      await this.videoOutput.stopCachedVideo();
+      this.isStoppingLastResponse = false;
+      this.options.onStoppingEnd();
+      return;
+    }
+    if (!this.avController.isPlaying) {
+      this.isStoppingLastResponse = false;
+      this.options.onStoppingEnd();
+      return;
+    }
+    const sessionId = `${this.user.id}::${this.user.orgId}::${this.user.headId}::${this.user.sessionId.toString().padStart(5, "0")}`;
+    const message = {
+      id: this.messageCounter,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      speaker: "user",
+      text: "",
+      isSent: false,
+      user_id: this.user.id,
+      username: this.user.username,
+      event: "stop" /* STOP */,
+      visible: true,
+      session_id: sessionId
+    };
+    this.connection.sendMessage(message);
+  }
   async toggleMute() {
     this.volume = this.volume === 0 ? 1 : 0;
     this.audioOutput.toggleMute();