@layercode/js-sdk 1.0.21 → 1.0.22

package/dist/layercode-js-sdk.min.js

@@ -516,6 +516,24 @@ registerProcessor('stream_processor', StreamProcessor);
       this.isPlaying = false;
     }
 
+    /**
+     * Clears interrupted track IDs to prevent memory leaks
+     * @param {string[]} [keepTrackIds] - Track IDs to keep in the interrupted list
+     */
+    clearInterruptedTracks(keepTrackIds = []) {
+      if (keepTrackIds.length === 0) {
+        this.interruptedTrackIds = {};
+      } else {
+        const newInterruptedTracks = {};
+        for (const trackId of keepTrackIds) {
+          if (this.interruptedTrackIds[trackId]) {
+            newInterruptedTracks[trackId] = true;
+          }
+        }
+        this.interruptedTrackIds = newInterruptedTracks;
+      }
+    }
+
     /**
      * Connects the audio context and enables output to speakers
      * @returns {Promise<true>}
@@ -749,7 +767,7 @@ registerProcessor('stream_processor', StreamProcessor);
         this.analyser.disconnect();
       }
 
-      if (this.context) {
+      if (this.context && this.context.state !== 'closed') {
         this.context.close().catch((err) => console.error("Error closing audio context:", err));
       }
       
@@ -3504,13 +3522,15 @@ registerProcessor('audio_processor', AudioProcessor);
           this.agentAudioAmplitude = 0;
           this.sessionId = options.sessionId || null;
           this.pushToTalkActive = false;
-          this.vadPausedPlayer = false;
           this.pushToTalkEnabled = false;
           this.canInterrupt = false;
           this.userIsSpeaking = false;
           this.endUserTurn = false;
           this.recorderStarted = false;
           this.readySent = false;
+          this.currentTurnText = '';
+          this.currentTurnId = null;
+          this.audioBuffer = [];
           // Bind event handlers
           this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
           this._handleDataAvailable = this._handleDataAvailable.bind(this);
@@ -3537,17 +3557,48 @@ registerProcessor('audio_processor', AudioProcessor);
                   minSpeechFrames: 15,
                   preSpeechPadFrames: 0,
                   onSpeechStart: () => {
-                      if (!this.wavPlayer.isPlaying) {
-                          this.userIsSpeaking = true;
-                          this.options.onUserIsSpeakingChange(true);
-                      }
+                      this.userIsSpeaking = true;
+                      this.options.onUserIsSpeakingChange(true);
+                      console.log('onSpeechStart: sending vad_start');
+                      this._wsSend({
+                          type: 'vad_events',
+                          event: 'vad_start',
+                      });
                   },
                   onVADMisfire: () => {
+                      console.log('onVADMisfire: Short utterance detected, resuming bot');
+                      this.audioBuffer = []; // Clear buffer on misfire
                       this.userIsSpeaking = false;
                       this.options.onUserIsSpeakingChange(false);
+                      // Send vad_end to indicate the short utterance is over
+                      this._wsSend({
+                          type: 'vad_events',
+                          event: 'vad_end',
+                      });
+                      // End the user's turn
+                      this._wsSend({
+                          type: 'trigger.turn.end',
+                          role: 'user',
+                      });
+                      // Resume bot audio if it was playing
+                      if (!this.wavPlayer.isPlaying) {
+                          console.log('onVADMisfire: Resuming bot audio');
+                          this.wavPlayer.play();
+                      }
                   },
                   onSpeechEnd: () => {
-                      this.endUserTurn = true; // Set flag to indicate that the user turn has ended, so we can send a vad_end event to the server
+                      console.log('onSpeechEnd: sending vad_end');
+                      this.endUserTurn = true; // Set flag to indicate that the user turn has ended
+                      this.audioBuffer = []; // Clear buffer on speech end
+                      this.userIsSpeaking = false;
+                      this.options.onUserIsSpeakingChange(false);
+                      console.log('onSpeechEnd: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
+                      // Send vad_end immediately instead of waiting for next audio chunk
+                      this._wsSend({
+                          type: 'vad_events',
+                          event: 'vad_end',
+                      });
+                      this.endUserTurn = false; // Reset the flag after sending vad_end
                   },
               })
                   .then((vad) => {
@@ -3569,41 +3620,59 @@ registerProcessor('audio_processor', AudioProcessor);
                   positiveSpeechThreshold: 0.3,
                   negativeSpeechThreshold: 0.2,
                   redemptionFrames: 25, // Number of frames of silence before onVADMisfire or onSpeechEnd is called. Effectively a delay before restarting.
-                  minSpeechFrames: 15,
+                  minSpeechFrames: 5,
                   preSpeechPadFrames: 0,
                   onSpeechStart: () => {
                       // Only pause agent audio if it's currently playing
                       if (this.wavPlayer.isPlaying) {
                           console.log('onSpeechStart: WavPlayer is playing, pausing it.');
                           this.wavPlayer.pause();
-                          this.vadPausedPlayer = true; // VAD is responsible for this pause
                       }
                       else {
                           console.log('onSpeechStart: WavPlayer is not playing, VAD will not pause.');
                       }
-                      this.userIsSpeaking = true;
-                      this.options.onUserIsSpeakingChange(true);
                       console.log('onSpeechStart: sending vad_start');
                       this._wsSend({
                           type: 'vad_events',
                           event: 'vad_start',
                       });
+                      this.userIsSpeaking = true;
+                      this.options.onUserIsSpeakingChange(true);
+                      this.endUserTurn = false; // Reset endUserTurn when speech starts
+                      console.log('onSpeechStart: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
                   },
                   onVADMisfire: () => {
                       // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd, and we should resume the assistant audio as it was a false interruption. We include a configurable delay so the assistant isn't too quick to start speaking again.
                       this.userIsSpeaking = false;
+                      this.audioBuffer = []; // Clear buffer on misfire
                       this.options.onUserIsSpeakingChange(false);
-                      if (this.vadPausedPlayer) {
-                          console.log('onSpeechEnd: VAD paused the player, resuming');
-                          this.wavPlayer.play();
-                          this.vadPausedPlayer = false; // Reset flag
-                      }
-                      else {
-                          console.log('onVADMisfire: VAD did not pause the player, no action taken to resume.');
-                      }
+                      // Add the missing delay before resuming to prevent race conditions
+                      setTimeout(() => {
+                          if (!this.wavPlayer.isPlaying) {
+                              console.log('onVADMisfire: Resuming after delay');
+                              this.wavPlayer.play();
+                              this.userIsSpeaking = true;
+                              this.options.onUserIsSpeakingChange(true);
+                          }
+                          else {
+                              console.log('onVADMisfire: Not resuming - either no pause or user speaking again');
+                              this.endUserTurn = true;
+                          }
+                      }, this.options.vadResumeDelay);
                   },
                   onSpeechEnd: () => {
-                      this.endUserTurn = true; // Set flag to indicate that the user turn has ended, so we can send a vad_end event to the server
+                      console.log('onSpeechEnd: sending vad_end');
+                      this.endUserTurn = true; // Set flag to indicate that the user turn has ended
+                      this.audioBuffer = []; // Clear buffer on speech end
+                      this.userIsSpeaking = false;
+                      this.options.onUserIsSpeakingChange(false);
+                      console.log('onSpeechEnd: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
+                      // Send vad_end immediately instead of waiting for next audio chunk
+                      this._wsSend({
+                          type: 'vad_events',
+                          event: 'vad_end',
+                      });
+                      this.endUserTurn = false; // Reset the flag after sending vad_end
                   },
               })
                   .then((vad) => {
@@ -3637,14 +3706,33 @@ registerProcessor('audio_processor', AudioProcessor);
               reason: 'completed',
           });
       }
+      _estimateWordsHeard(text, playbackOffsetSeconds) {
+          const words = text.split(/\s+/).filter((word) => word.length > 0);
+          const totalWords = words.length;
+          // Rough estimation: average speaking rate is ~150 words per minute (2.5 words per second)
+          const estimatedWordsPerSecond = 2.5;
+          const estimatedWordsHeard = Math.min(Math.floor(playbackOffsetSeconds * estimatedWordsPerSecond), totalWords);
+          const textHeard = words.slice(0, estimatedWordsHeard).join(' ');
+          return { wordsHeard: estimatedWordsHeard, textHeard };
+      }
       async _clientInterruptAssistantReplay() {
-          await this.wavPlayer.interrupt();
-          // TODO: Use in voice pipeline to know how much of the audio has been played and how much to truncate transcript
-          // this._wsSend({
-          //   type: 'trigger.response.audio.replay_finished',
-          //   reason: 'interrupted',
-          //   delta_id: 'TODO'
-          // });
+          const offsetData = await this.wavPlayer.interrupt();
+          if (offsetData && this.currentTurnText && this.currentTurnId) {
+              const { wordsHeard, textHeard } = this._estimateWordsHeard(this.currentTurnText, offsetData.currentTime);
+              const totalWords = this.currentTurnText.split(/\s+/).filter((word) => word.length > 0).length;
+              console.log(`Interruption detected: ${wordsHeard}/${totalWords} words heard, text: "${textHeard}"`);
+              // Send interruption event with context
+              this._wsSend({
+                  type: 'trigger.response.audio.interrupted',
+                  playback_offset: offsetData.currentTime,
+                  interruption_context: {
+                      turn_id: this.currentTurnId,
+                      estimated_words_heard: wordsHeard,
+                      total_words: totalWords,
+                      text_heard: textHeard,
+                  },
+              });
+          }
       }
       async triggerUserTurnStarted() {
           if (!this.pushToTalkActive) {
@@ -3675,20 +3763,44 @@ registerProcessor('audio_processor', AudioProcessor);
                       // Sent from the server to this client when a new user turn is detected
                       console.log('received turn.start from server');
                       console.log(message);
-                      if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
+                      if (message.role === 'assistant') {
+                          // Start tracking new assistant turn
+                          // Note: Don't reset currentTurnId here - let response.audio set it
+                          // This prevents race conditions where text arrives before audio
+                          console.log('Assistant turn started, will track new turn ID from audio/text');
+                      }
+                      else if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
                           // Interrupt any playing assistant audio if this is a turn trigged by the server (and not push to talk, which will have already called interrupt)
                           console.log('interrupting assistant audio, as user turn has started and pushToTalkEnabled is false');
                           await this._clientInterruptAssistantReplay();
                       }
-                      // if (message.role === 'assistant') {
-                      //   // Clear the buffer of audio when the assisatnt starts a new turn, as it may have been paused previously by VAD, leaving some audio frames in the buffer.
-                      //   console.log('Clearing audio buffer as assistant turn has started');
-                      //   await this._clientInterruptAssistantReplay();
-                      // }
                       break;
                   case 'response.audio':
                       const audioBuffer = base64ToArrayBuffer(message.content);
                       this.wavPlayer.add16BitPCM(audioBuffer, message.turn_id);
+                      // Set current turn ID from first audio message, or update if different turn
+                      if (!this.currentTurnId || this.currentTurnId !== message.turn_id) {
+                          console.log(`Setting current turn ID to: ${message.turn_id} (was: ${this.currentTurnId})`);
+                          const oldTurnId = this.currentTurnId;
+                          this.currentTurnId = message.turn_id;
+                          this.currentTurnText = ''; // Reset text for new turn
+                          // Clean up interrupted tracks, keeping only the current turn
+                          this.wavPlayer.clearInterruptedTracks(this.currentTurnId ? [this.currentTurnId] : []);
+                      }
+                      break;
+                  case 'response.text':
+                      // Set turn ID from first text message if not set, or accumulate if matches current turn
+                      if (!this.currentTurnId || message.turn_id === this.currentTurnId) {
+                          if (!this.currentTurnId) {
+                              console.log(`Setting current turn ID to: ${message.turn_id} from text message`);
+                              this.currentTurnId = message.turn_id;
+                              this.currentTurnText = '';
+                          }
+                          this.currentTurnText += message.content;
+                      }
+                      else {
+                          console.log(`Ignoring text for turn ${message.turn_id}, current turn is ${this.currentTurnId}`);
+                      }
                       break;
                   // case 'response.end':
                   //   console.log('received response.end');
@@ -3717,18 +3829,29 @@ registerProcessor('audio_processor', AudioProcessor);
               const base64 = arrayBufferToBase64(data.mono);
               const sendAudio = this.pushToTalkEnabled ? this.pushToTalkActive : this.userIsSpeaking;
               if (sendAudio) {
+                  // If we have buffered audio, send it first
+                  if (this.audioBuffer.length > 0) {
+                      console.log(`Sending ${this.audioBuffer.length} buffered audio chunks`);
+                      for (const bufferedAudio of this.audioBuffer) {
+                          this._wsSend({
+                              type: 'client.audio',
+                              content: bufferedAudio,
+                          });
+                      }
+                      this.audioBuffer = []; // Clear the buffer after sending
+                  }
+                  // Send the current audio
                   this._wsSend({
                       type: 'client.audio',
                       content: base64,
                   });
-                  if (this.endUserTurn) {
-                      this.endUserTurn = false;
-                      this.userIsSpeaking = false; // Reset userIsSpeaking to false so we don't send any more audio to the server
-                      this.options.onUserIsSpeakingChange(false);
-                      this._wsSend({
-                          type: 'vad_events',
-                          event: 'vad_end',
-                      });
+              }
+              else {
+                  // Buffer audio when not sending (to catch audio just before VAD triggers)
+                  this.audioBuffer.push(base64);
+                  // Keep buffer size reasonable (e.g., last 10 chunks ≈ 200ms at 20ms chunks)
+                  if (this.audioBuffer.length > 10) {
+                      this.audioBuffer.shift(); // Remove oldest chunk
                   }
               }
           }
@@ -3863,10 +3986,20 @@ registerProcessor('audio_processor', AudioProcessor);
           }
       }
       async disconnect() {
-          var _a;
+          // Clean up VAD if it exists
+          if (this.vad) {
+              this.vad.pause();
+              this.vad.destroy();
+              this.vad = null;
+          }
           this.wavRecorder.quit();
           this.wavPlayer.disconnect();
-          (_a = this.ws) === null || _a === void 0 ? void 0 : _a.close();
+          // Close websocket and ensure status is updated
+          if (this.ws) {
+              this.ws.close();
+              this._setStatus('disconnected');
+              this.options.onDisconnect();
+          }
       }
       /**
        * Gets the microphone MediaStream used by this client