@layercode/js-sdk 1.0.19 → 1.0.22

package/dist/layercode-js-sdk.esm.js

@@ -510,6 +510,24 @@ class WavStreamPlayer {
     this.isPlaying = false;
   }
 
+  /**
+   * Clears interrupted track IDs to prevent memory leaks
+   * @param {string[]} [keepTrackIds] - Track IDs to keep in the interrupted list
+   */
+  clearInterruptedTracks(keepTrackIds = []) {
+    if (keepTrackIds.length === 0) {
+      this.interruptedTrackIds = {};
+    } else {
+      const newInterruptedTracks = {};
+      for (const trackId of keepTrackIds) {
+        if (this.interruptedTrackIds[trackId]) {
+          newInterruptedTracks[trackId] = true;
+        }
+      }
+      this.interruptedTrackIds = newInterruptedTracks;
+    }
+  }
+
   /**
    * Connects the audio context and enables output to speakers
    * @returns {Promise<true>}
@@ -743,7 +761,7 @@ class WavStreamPlayer {
       this.analyser.disconnect();
     }
 
-    if (this.context) {
+    if (this.context && this.context.state !== 'closed') {
       this.context.close().catch((err) => console.error("Error closing audio context:", err));
     }
     
@@ -3482,6 +3500,7 @@ class LayercodeClient {
             onUserAmplitudeChange: options.onUserAmplitudeChange || (() => { }),
             onAgentAmplitudeChange: options.onAgentAmplitudeChange || (() => { }),
             onStatusChange: options.onStatusChange || (() => { }),
+            onUserIsSpeakingChange: options.onUserIsSpeakingChange || (() => { }),
         };
         this.AMPLITUDE_MONITORING_SAMPLE_RATE = 10;
         this._websocketUrl = 'wss://api.layercode.com/v1/pipelines/websocket';
@@ -3497,13 +3516,15 @@ class LayercodeClient {
         this.agentAudioAmplitude = 0;
         this.sessionId = options.sessionId || null;
         this.pushToTalkActive = false;
-        this.vadPausedPlayer = false;
         this.pushToTalkEnabled = false;
         this.canInterrupt = false;
         this.userIsSpeaking = false;
         this.endUserTurn = false;
         this.recorderStarted = false;
         this.readySent = false;
+        this.currentTurnText = '';
+        this.currentTurnId = null;
+        this.audioBuffer = [];
         // Bind event handlers
         this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
         this._handleDataAvailable = this._handleDataAvailable.bind(this);
@@ -3518,6 +3539,7 @@ class LayercodeClient {
             console.log('silero vad model timeout');
             // TODO: send message to server to indicate that the vad model timed out
             this.userIsSpeaking = true; // allow audio to be sent to the server
+            this.options.onUserIsSpeakingChange(true);
         }, 2000);
         if (!this.canInterrupt) {
             dist.MicVAD.new({
@@ -3529,15 +3551,48 @@ class LayercodeClient {
                 minSpeechFrames: 15,
                 preSpeechPadFrames: 0,
                 onSpeechStart: () => {
-                    if (!this.wavPlayer.isPlaying) {
-                        this.userIsSpeaking = true;
-                    }
+                    this.userIsSpeaking = true;
+                    this.options.onUserIsSpeakingChange(true);
+                    console.log('onSpeechStart: sending vad_start');
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_start',
+                    });
                 },
                 onVADMisfire: () => {
+                    console.log('onVADMisfire: Short utterance detected, resuming bot');
+                    this.audioBuffer = []; // Clear buffer on misfire
                     this.userIsSpeaking = false;
+                    this.options.onUserIsSpeakingChange(false);
+                    // Send vad_end to indicate the short utterance is over
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_end',
+                    });
+                    // End the user's turn
+                    this._wsSend({
+                        type: 'trigger.turn.end',
+                        role: 'user',
+                    });
+                    // Resume bot audio if it was playing
+                    if (!this.wavPlayer.isPlaying) {
+                        console.log('onVADMisfire: Resuming bot audio');
+                        this.wavPlayer.play();
+                    }
                 },
                 onSpeechEnd: () => {
-                    this.endUserTurn = true; // Set flag to indicate that the user turn has ended, so we can send a vad_end event to the server
+                    console.log('onSpeechEnd: sending vad_end');
+                    this.endUserTurn = true; // Set flag to indicate that the user turn has ended
+                    this.audioBuffer = []; // Clear buffer on speech end
+                    this.userIsSpeaking = false;
+                    this.options.onUserIsSpeakingChange(false);
+                    console.log('onSpeechEnd: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
+                    // Send vad_end immediately instead of waiting for next audio chunk
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_end',
+                    });
+                    this.endUserTurn = false; // Reset the flag after sending vad_end
                 },
             })
                 .then((vad) => {
@@ -3559,43 +3614,59 @@ class LayercodeClient {
                 positiveSpeechThreshold: 0.3,
                 negativeSpeechThreshold: 0.2,
                 redemptionFrames: 25, // Number of frames of silence before onVADMisfire or onSpeechEnd is called. Effectively a delay before restarting.
-                minSpeechFrames: 15,
+                minSpeechFrames: 5,
                 preSpeechPadFrames: 0,
                 onSpeechStart: () => {
                     // Only pause agent audio if it's currently playing
                     if (this.wavPlayer.isPlaying) {
                         console.log('onSpeechStart: WavPlayer is playing, pausing it.');
                         this.wavPlayer.pause();
-                        this.vadPausedPlayer = true; // VAD is responsible for this pause
                     }
                     else {
                         console.log('onSpeechStart: WavPlayer is not playing, VAD will not pause.');
                     }
-                    this.userIsSpeaking = true;
                     console.log('onSpeechStart: sending vad_start');
                     this._wsSend({
                         type: 'vad_events',
                         event: 'vad_start',
                     });
+                    this.userIsSpeaking = true;
+                    this.options.onUserIsSpeakingChange(true);
+                    this.endUserTurn = false; // Reset endUserTurn when speech starts
+                    console.log('onSpeechStart: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
                 },
                 onVADMisfire: () => {
                     // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd, and we should resume the assistant audio as it was a false interruption. We include a configurable delay so the assistant isn't too quick to start speaking again.
                     this.userIsSpeaking = false;
-                    if (this.vadPausedPlayer) {
-                        console.log('onSpeechEnd: VAD paused the player, resuming');
-                        this.wavPlayer.play();
-                        this.vadPausedPlayer = false; // Reset flag
-                    }
-                    else {
-                        console.log('onVADMisfire: VAD did not pause the player, no action taken to resume.');
-                    }
+                    this.audioBuffer = []; // Clear buffer on misfire
+                    this.options.onUserIsSpeakingChange(false);
+                    // Add the missing delay before resuming to prevent race conditions
+                    setTimeout(() => {
+                        if (!this.wavPlayer.isPlaying) {
+                            console.log('onVADMisfire: Resuming after delay');
+                            this.wavPlayer.play();
+                            this.userIsSpeaking = true;
+                            this.options.onUserIsSpeakingChange(true);
+                        }
+                        else {
+                            console.log('onVADMisfire: Not resuming - either no pause or user speaking again');
+                            this.endUserTurn = true;
+                        }
+                    }, this.options.vadResumeDelay);
                 },
                 onSpeechEnd: () => {
+                    console.log('onSpeechEnd: sending vad_end');
+                    this.endUserTurn = true; // Set flag to indicate that the user turn has ended
+                    this.audioBuffer = []; // Clear buffer on speech end
                     this.userIsSpeaking = false;
+                    this.options.onUserIsSpeakingChange(false);
+                    console.log('onSpeechEnd: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
+                    // Send vad_end immediately instead of waiting for next audio chunk
                     this._wsSend({
                         type: 'vad_events',
                         event: 'vad_end',
                     });
+                    this.endUserTurn = false; // Reset the flag after sending vad_end
                 },
             })
                 .then((vad) => {
@@ -3629,14 +3700,33 @@ class LayercodeClient {
             reason: 'completed',
         });
     }
+    _estimateWordsHeard(text, playbackOffsetSeconds) {
+        const words = text.split(/\s+/).filter((word) => word.length > 0);
+        const totalWords = words.length;
+        // Rough estimation: average speaking rate is ~150 words per minute (2.5 words per second)
+        const estimatedWordsPerSecond = 2.5;
+        const estimatedWordsHeard = Math.min(Math.floor(playbackOffsetSeconds * estimatedWordsPerSecond), totalWords);
+        const textHeard = words.slice(0, estimatedWordsHeard).join(' ');
+        return { wordsHeard: estimatedWordsHeard, textHeard };
+    }
     async _clientInterruptAssistantReplay() {
-        await this.wavPlayer.interrupt();
-        // TODO: Use in voice pipeline to know how much of the audio has been played and how much to truncate transcript
-        // this._wsSend({
-        //   type: 'trigger.response.audio.replay_finished',
-        //   reason: 'interrupted',
-        //   delta_id: 'TODO'
-        // });
+        const offsetData = await this.wavPlayer.interrupt();
+        if (offsetData && this.currentTurnText && this.currentTurnId) {
+            const { wordsHeard, textHeard } = this._estimateWordsHeard(this.currentTurnText, offsetData.currentTime);
+            const totalWords = this.currentTurnText.split(/\s+/).filter((word) => word.length > 0).length;
+            console.log(`Interruption detected: ${wordsHeard}/${totalWords} words heard, text: "${textHeard}"`);
+            // Send interruption event with context
+            this._wsSend({
+                type: 'trigger.response.audio.interrupted',
+                playback_offset: offsetData.currentTime,
+                interruption_context: {
+                    turn_id: this.currentTurnId,
+                    estimated_words_heard: wordsHeard,
+                    total_words: totalWords,
+                    text_heard: textHeard,
+                },
+            });
+        }
     }
     async triggerUserTurnStarted() {
         if (!this.pushToTalkActive) {
@@ -3667,20 +3757,44 @@ class LayercodeClient {
                     // Sent from the server to this client when a new user turn is detected
                     console.log('received turn.start from server');
                     console.log(message);
-                    if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
+                    if (message.role === 'assistant') {
+                        // Start tracking new assistant turn
+                        // Note: Don't reset currentTurnId here - let response.audio set it
+                        // This prevents race conditions where text arrives before audio
+                        console.log('Assistant turn started, will track new turn ID from audio/text');
+                    }
+                    else if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
                         // Interrupt any playing assistant audio if this is a turn trigged by the server (and not push to talk, which will have already called interrupt)
                         console.log('interrupting assistant audio, as user turn has started and pushToTalkEnabled is false');
                         await this._clientInterruptAssistantReplay();
                     }
-                    // if (message.role === 'assistant') {
-                    //   // Clear the buffer of audio when the assisatnt starts a new turn, as it may have been paused previously by VAD, leaving some audio frames in the buffer.
-                    //   console.log('Clearing audio buffer as assistant turn has started');
-                    //   await this._clientInterruptAssistantReplay();
-                    // }
                     break;
                 case 'response.audio':
                     const audioBuffer = base64ToArrayBuffer(message.content);
                     this.wavPlayer.add16BitPCM(audioBuffer, message.turn_id);
+                    // Set current turn ID from first audio message, or update if different turn
+                    if (!this.currentTurnId || this.currentTurnId !== message.turn_id) {
+                        console.log(`Setting current turn ID to: ${message.turn_id} (was: ${this.currentTurnId})`);
+                        const oldTurnId = this.currentTurnId;
+                        this.currentTurnId = message.turn_id;
+                        this.currentTurnText = ''; // Reset text for new turn
+                        // Clean up interrupted tracks, keeping only the current turn
+                        this.wavPlayer.clearInterruptedTracks(this.currentTurnId ? [this.currentTurnId] : []);
+                    }
+                    break;
+                case 'response.text':
+                    // Set turn ID from first text message if not set, or accumulate if matches current turn
+                    if (!this.currentTurnId || message.turn_id === this.currentTurnId) {
+                        if (!this.currentTurnId) {
+                            console.log(`Setting current turn ID to: ${message.turn_id} from text message`);
+                            this.currentTurnId = message.turn_id;
+                            this.currentTurnText = '';
+                        }
+                        this.currentTurnText += message.content;
+                    }
+                    else {
+                        console.log(`Ignoring text for turn ${message.turn_id}, current turn is ${this.currentTurnId}`);
+                    }
                     break;
                 // case 'response.end':
                 //   console.log('received response.end');
@@ -3709,17 +3823,29 @@ class LayercodeClient {
             const base64 = arrayBufferToBase64(data.mono);
             const sendAudio = this.pushToTalkEnabled ? this.pushToTalkActive : this.userIsSpeaking;
             if (sendAudio) {
+                // If we have buffered audio, send it first
+                if (this.audioBuffer.length > 0) {
+                    console.log(`Sending ${this.audioBuffer.length} buffered audio chunks`);
+                    for (const bufferedAudio of this.audioBuffer) {
+                        this._wsSend({
+                            type: 'client.audio',
+                            content: bufferedAudio,
+                        });
+                    }
+                    this.audioBuffer = []; // Clear the buffer after sending
+                }
+                // Send the current audio
                 this._wsSend({
                     type: 'client.audio',
                     content: base64,
                 });
-                if (this.endUserTurn) {
-                    this.endUserTurn = false;
-                    this.userIsSpeaking = false; // Reset userIsSpeaking to false so we don't send any more audio to the server
-                    this._wsSend({
-                        type: 'vad_events',
-                        event: 'vad_end',
-                    });
+            }
+            else {
+                // Buffer audio when not sending (to catch audio just before VAD triggers)
+                this.audioBuffer.push(base64);
+                // Keep buffer size reasonable (e.g., last 10 chunks ≈ 200ms at 20ms chunks)
+                if (this.audioBuffer.length > 10) {
+                    this.audioBuffer.shift(); // Remove oldest chunk
                 }
             }
         }
@@ -3854,10 +3980,20 @@ class LayercodeClient {
         }
     }
     async disconnect() {
-        var _a;
+        // Clean up VAD if it exists
+        if (this.vad) {
+            this.vad.pause();
+            this.vad.destroy();
+            this.vad = null;
+        }
         this.wavRecorder.quit();
         this.wavPlayer.disconnect();
-        (_a = this.ws) === null || _a === void 0 ? void 0 : _a.close();
+        // Close websocket and ensure status is updated
+        if (this.ws) {
+            this.ws.close();
+            this._setStatus('disconnected');
+            this.options.onDisconnect();
+        }
     }
     /**
      * Gets the microphone MediaStream used by this client
@@ -3866,6 +4002,25 @@ class LayercodeClient {
     getStream() {
         return this.wavRecorder.getStream();
     }
+    /**
+     * Switches the input device for the microphone and restarts recording
+     * @param {string} deviceId - The deviceId of the new microphone
+     */
+    async setInputDevice(deviceId) {
+        if (this.wavRecorder) {
+            try {
+                await this.wavRecorder.end();
+            }
+            catch (e) { }
+            try {
+                await this.wavRecorder.quit();
+            }
+            catch (e) { }
+        }
+        await this.wavRecorder.begin(deviceId);
+        await this.wavRecorder.record(this._handleDataAvailable, 1638);
+        this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
+    }
 }
 
 export { LayercodeClient as default };