@layercode/js-sdk 1.0.21 → 1.0.23

package/dist/layercode-js-sdk.esm.js

@@ -510,6 +510,24 @@ class WavStreamPlayer {
     this.isPlaying = false;
   }
 
+  /**
+   * Clears interrupted track IDs to prevent memory leaks
+   * @param {string[]} [keepTrackIds] - Track IDs to keep in the interrupted list
+   */
+  clearInterruptedTracks(keepTrackIds = []) {
+    if (keepTrackIds.length === 0) {
+      this.interruptedTrackIds = {};
+    } else {
+      const newInterruptedTracks = {};
+      for (const trackId of keepTrackIds) {
+        if (this.interruptedTrackIds[trackId]) {
+          newInterruptedTracks[trackId] = true;
+        }
+      }
+      this.interruptedTrackIds = newInterruptedTracks;
+    }
+  }
+
   /**
    * Connects the audio context and enables output to speakers
    * @returns {Promise<true>}
@@ -743,7 +761,7 @@ class WavStreamPlayer {
       this.analyser.disconnect();
     }
 
-    if (this.context) {
+    if (this.context && this.context.state !== 'closed') {
       this.context.close().catch((err) => console.error("Error closing audio context:", err));
     }
     
@@ -3498,17 +3516,59 @@ class LayercodeClient {
         this.agentAudioAmplitude = 0;
         this.sessionId = options.sessionId || null;
         this.pushToTalkActive = false;
-        this.vadPausedPlayer = false;
         this.pushToTalkEnabled = false;
         this.canInterrupt = false;
         this.userIsSpeaking = false;
         this.endUserTurn = false;
         this.recorderStarted = false;
         this.readySent = false;
+        this.currentTurnId = null;
+        this.audioBuffer = [];
+        this.audioPauseTime = null;
         // Bind event handlers
         this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
         this._handleDataAvailable = this._handleDataAvailable.bind(this);
     }
+    _setupAmplitudeBasedVAD() {
+        let isSpeakingByAmplitude = false;
+        let silenceFrames = 0;
+        const AMPLITUDE_THRESHOLD = 0.01; // Adjust based on testing
+        const SILENCE_FRAMES_THRESHOLD = 30; // ~600ms at 20ms chunks
+        // Monitor amplitude changes
+        this.wavRecorder.startAmplitudeMonitoring((amplitude) => {
+            const wasSpeaking = isSpeakingByAmplitude;
+            if (amplitude > AMPLITUDE_THRESHOLD) {
+                silenceFrames = 0;
+                if (!wasSpeaking) {
+                    // Speech started - pause audio if playing and track timing for interruption calculation
+                    if (this.canInterrupt && this.wavPlayer.isPlaying) {
+                        this.audioPauseTime = Date.now();
+                        this.wavPlayer.pause();
+                    }
+                    isSpeakingByAmplitude = true;
+                    this.userIsSpeaking = true;
+                    this.options.onUserIsSpeakingChange(true);
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_start',
+                    });
+                }
+            }
+            else {
+                silenceFrames++;
+                if (wasSpeaking && silenceFrames >= SILENCE_FRAMES_THRESHOLD) {
+                    // Speech ended
+                    isSpeakingByAmplitude = false;
+                    this.userIsSpeaking = false;
+                    this.options.onUserIsSpeakingChange(false);
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_end',
+                    });
+                }
+            }
+        });
+    }
     _initializeVAD() {
         console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt });
         // If we're in push to talk mode, we don't need to use the VAD model
@@ -3517,9 +3577,17 @@ class LayercodeClient {
         }
         const timeout = setTimeout(() => {
             console.log('silero vad model timeout');
-            // TODO: send message to server to indicate that the vad model timed out
-            this.userIsSpeaking = true; // allow audio to be sent to the server
-            this.options.onUserIsSpeakingChange(true);
+            console.warn('VAD model failed to load - falling back to amplitude-based detection');
+            // Send a message to server indicating VAD failure
+            this._wsSend({
+                type: 'vad_events',
+                event: 'vad_model_failed',
+            });
+            // In automatic mode without VAD, allow the bot to speak initially
+            this.userIsSpeaking = false;
+            this.options.onUserIsSpeakingChange(false);
+            // Set up amplitude-based fallback detection
+            this._setupAmplitudeBasedVAD();
         }, 2000);
         if (!this.canInterrupt) {
             dist.MicVAD.new({
@@ -3528,20 +3596,30 @@ class LayercodeClient {
                 positiveSpeechThreshold: 0.3,
                 negativeSpeechThreshold: 0.2,
                 redemptionFrames: 25, // Number of frames of silence before onVADMisfire or onSpeechEnd is called. Effectively a delay before restarting.
-                minSpeechFrames: 15,
+                minSpeechFrames: 0,
                 preSpeechPadFrames: 0,
                 onSpeechStart: () => {
-                    if (!this.wavPlayer.isPlaying) {
-                        this.userIsSpeaking = true;
-                        this.options.onUserIsSpeakingChange(true);
-                    }
+                    this.userIsSpeaking = true;
+                    this.options.onUserIsSpeakingChange(true);
+                    console.log('onSpeechStart: sending vad_start');
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_start',
+                    });
                 },
-                onVADMisfire: () => {
+                onSpeechEnd: () => {
+                    console.log('onSpeechEnd: sending vad_end');
+                    this.endUserTurn = true; // Set flag to indicate that the user turn has ended
+                    this.audioBuffer = []; // Clear buffer on speech end
                     this.userIsSpeaking = false;
                     this.options.onUserIsSpeakingChange(false);
-                },
-                onSpeechEnd: () => {
-                    this.endUserTurn = true; // Set flag to indicate that the user turn has ended, so we can send a vad_end event to the server
+                    console.log('onSpeechEnd: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
+                    // Send vad_end immediately instead of waiting for next audio chunk
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_end',
+                    });
+                    this.endUserTurn = false; // Reset the flag after sending vad_end
                 },
             })
                 .then((vad) => {
@@ -3563,41 +3641,59 @@ class LayercodeClient {
                 positiveSpeechThreshold: 0.3,
                 negativeSpeechThreshold: 0.2,
                 redemptionFrames: 25, // Number of frames of silence before onVADMisfire or onSpeechEnd is called. Effectively a delay before restarting.
-                minSpeechFrames: 15,
+                minSpeechFrames: 5,
                 preSpeechPadFrames: 0,
                 onSpeechStart: () => {
                     // Only pause agent audio if it's currently playing
                     if (this.wavPlayer.isPlaying) {
                         console.log('onSpeechStart: WavPlayer is playing, pausing it.');
+                        this.audioPauseTime = Date.now(); // Track when we paused
                         this.wavPlayer.pause();
-                        this.vadPausedPlayer = true; // VAD is responsible for this pause
                     }
                     else {
                         console.log('onSpeechStart: WavPlayer is not playing, VAD will not pause.');
                     }
-                    this.userIsSpeaking = true;
-                    this.options.onUserIsSpeakingChange(true);
                     console.log('onSpeechStart: sending vad_start');
                     this._wsSend({
                         type: 'vad_events',
                         event: 'vad_start',
                     });
+                    this.userIsSpeaking = true;
+                    this.options.onUserIsSpeakingChange(true);
+                    this.endUserTurn = false; // Reset endUserTurn when speech starts
+                    console.log('onSpeechStart: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
                 },
                 onVADMisfire: () => {
                     // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd, and we should resume the assistant audio as it was a false interruption. We include a configurable delay so the assistant isn't too quick to start speaking again.
                     this.userIsSpeaking = false;
+                    this.audioBuffer = []; // Clear buffer on misfire
                     this.options.onUserIsSpeakingChange(false);
-                    if (this.vadPausedPlayer) {
-                        console.log('onSpeechEnd: VAD paused the player, resuming');
-                        this.wavPlayer.play();
-                        this.vadPausedPlayer = false; // Reset flag
-                    }
-                    else {
-                        console.log('onVADMisfire: VAD did not pause the player, no action taken to resume.');
-                    }
+                    // Add the missing delay before resuming to prevent race conditions
+                    setTimeout(() => {
+                        if (!this.wavPlayer.isPlaying) {
+                            console.log('onVADMisfire: Resuming after delay');
+                            this.audioPauseTime = null; // Clear pause time since we're resuming
+                            this.wavPlayer.play();
+                        }
+                        else {
+                            console.log('onVADMisfire: Not resuming - either no pause or user speaking again');
+                            this.endUserTurn = true;
+                        }
+                    }, this.options.vadResumeDelay);
                 },
                 onSpeechEnd: () => {
-                    this.endUserTurn = true; // Set flag to indicate that the user turn has ended, so we can send a vad_end event to the server
+                    console.log('onSpeechEnd: sending vad_end');
+                    this.endUserTurn = true; // Set flag to indicate that the user turn has ended
+                    this.audioBuffer = []; // Clear buffer on speech end
+                    this.userIsSpeaking = false;
+                    this.options.onUserIsSpeakingChange(false);
+                    console.log('onSpeechEnd: State after update - endUserTurn:', this.endUserTurn, 'userIsSpeaking:', this.userIsSpeaking);
+                    // Send vad_end immediately instead of waiting for next audio chunk
+                    this._wsSend({
+                        type: 'vad_events',
+                        event: 'vad_end',
+                    });
+                    this.endUserTurn = false; // Reset the flag after sending vad_end
                 },
             })
                 .then((vad) => {
@@ -3632,13 +3728,36 @@ class LayercodeClient {
         });
     }
     async _clientInterruptAssistantReplay() {
-        await this.wavPlayer.interrupt();
-        // TODO: Use in voice pipeline to know how much of the audio has been played and how much to truncate transcript
-        // this._wsSend({
-        //   type: 'trigger.response.audio.replay_finished',
-        //   reason: 'interrupted',
-        //   delta_id: 'TODO'
-        // });
+        const offsetData = await this.wavPlayer.interrupt();
+        if (offsetData && this.currentTurnId) {
+            let offsetMs = offsetData.currentTime * 1000;
+            // Calculate accurate offset by subtracting pause time if audio was paused for VAD
+            if (this.audioPauseTime) {
+                const pauseDurationMs = Date.now() - this.audioPauseTime;
+                const adjustedOffsetMs = Math.max(0, offsetMs - pauseDurationMs);
+                console.log(`Interruption detected: Raw offset ${offsetMs}ms, pause duration ${pauseDurationMs}ms, adjusted offset ${adjustedOffsetMs}ms for turn ${this.currentTurnId}`);
+                offsetMs = adjustedOffsetMs;
+                this.audioPauseTime = null; // Clear the pause time
+            }
+            else {
+                console.log(`Interruption detected: ${offsetMs}ms offset for turn ${this.currentTurnId} (no pause adjustment needed)`);
+            }
+            // Send interruption event with accurate playback offset in milliseconds
+            this._wsSend({
+                type: 'trigger.response.audio.interrupted',
+                playback_offset: offsetMs,
+                interruption_context: {
+                    turn_id: this.currentTurnId,
+                    playback_offset_ms: offsetMs,
+                },
+            });
+        }
+        else {
+            console.warn('Interruption requested but missing required data:', {
+                hasOffsetData: !!offsetData,
+                hasTurnId: !!this.currentTurnId,
+            });
+        }
     }
     async triggerUserTurnStarted() {
         if (!this.pushToTalkActive) {
@@ -3669,24 +3788,38 @@ class LayercodeClient {
                     // Sent from the server to this client when a new user turn is detected
                     console.log('received turn.start from server');
                     console.log(message);
-                    if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
+                    if (message.role === 'assistant') {
+                        // Start tracking new assistant turn
+                        // Note: Don't reset currentTurnId here - let response.audio set it
+                        // This prevents race conditions where text arrives before audio
+                        console.log('Assistant turn started, will track new turn ID from audio/text');
+                    }
+                    else if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
                         // Interrupt any playing assistant audio if this is a turn trigged by the server (and not push to talk, which will have already called interrupt)
                         console.log('interrupting assistant audio, as user turn has started and pushToTalkEnabled is false');
                         await this._clientInterruptAssistantReplay();
                     }
-                    // if (message.role === 'assistant') {
-                    //   // Clear the buffer of audio when the assisatnt starts a new turn, as it may have been paused previously by VAD, leaving some audio frames in the buffer.
-                    //   console.log('Clearing audio buffer as assistant turn has started');
-                    //   await this._clientInterruptAssistantReplay();
-                    // }
                     break;
                 case 'response.audio':
                     const audioBuffer = base64ToArrayBuffer(message.content);
                     this.wavPlayer.add16BitPCM(audioBuffer, message.turn_id);
+                    // Set current turn ID from first audio message, or update if different turn
+                    if (!this.currentTurnId || this.currentTurnId !== message.turn_id) {
+                        console.log(`Setting current turn ID to: ${message.turn_id} (was: ${this.currentTurnId})`);
+                        this.currentTurnId = message.turn_id;
+                        // Clean up interrupted tracks, keeping only the current turn
+                        this.wavPlayer.clearInterruptedTracks(this.currentTurnId ? [this.currentTurnId] : []);
+                    }
                     break;
-                // case 'response.end':
-                //   console.log('received response.end');
-                //   break;
+                case 'response.text': {
+                    // Set turn ID from first text message if not set
+                    if (!this.currentTurnId) {
+                        this.currentTurnId = message.turn_id;
+                        console.log(`Setting current turn ID to: ${message.turn_id} from text message`);
+                    }
+                    // Note: We no longer track text content in the client - the pipeline handles interruption estimation
+                    break;
+                }
                 case 'response.data':
                     console.log('received response.data', message);
                     this.options.onDataMessage(message);
@@ -3711,18 +3844,29 @@ class LayercodeClient {
             const base64 = arrayBufferToBase64(data.mono);
             const sendAudio = this.pushToTalkEnabled ? this.pushToTalkActive : this.userIsSpeaking;
             if (sendAudio) {
+                // If we have buffered audio, send it first
+                if (this.audioBuffer.length > 0) {
+                    console.log(`Sending ${this.audioBuffer.length} buffered audio chunks`);
+                    for (const bufferedAudio of this.audioBuffer) {
+                        this._wsSend({
+                            type: 'client.audio',
+                            content: bufferedAudio,
+                        });
+                    }
+                    this.audioBuffer = []; // Clear the buffer after sending
+                }
+                // Send the current audio
                 this._wsSend({
                     type: 'client.audio',
                     content: base64,
                 });
-                if (this.endUserTurn) {
-                    this.endUserTurn = false;
-                    this.userIsSpeaking = false; // Reset userIsSpeaking to false so we don't send any more audio to the server
-                    this.options.onUserIsSpeakingChange(false);
-                    this._wsSend({
-                        type: 'vad_events',
-                        event: 'vad_end',
-                    });
+            }
+            else {
+                // Buffer audio when not sending (to catch audio just before VAD triggers)
+                this.audioBuffer.push(base64);
+                // Keep buffer size reasonable (e.g., last 10 chunks ≈ 200ms at 20ms chunks)
+                if (this.audioBuffer.length > 10) {
+                    this.audioBuffer.shift(); // Remove oldest chunk
                 }
             }
         }
@@ -3779,6 +3923,8 @@ class LayercodeClient {
     async connect() {
         try {
             this._setStatus('connecting');
+            // Reset turn tracking for clean start
+            this._resetTurnTracking();
             // Get session key from server
             let authorizeSessionRequestBody = {
                 pipeline_id: this.options.pipelineId,
@@ -3856,11 +4002,27 @@ class LayercodeClient {
             throw error;
         }
     }
+    _resetTurnTracking() {
+        this.currentTurnId = null;
+        console.log('Reset turn tracking state');
+    }
     async disconnect() {
-        var _a;
+        // Clean up VAD if it exists
+        if (this.vad) {
+            this.vad.pause();
+            this.vad.destroy();
+            this.vad = null;
+        }
         this.wavRecorder.quit();
         this.wavPlayer.disconnect();
-        (_a = this.ws) === null || _a === void 0 ? void 0 : _a.close();
+        // Reset turn tracking
+        this._resetTurnTracking();
+        // Close websocket and ensure status is updated
+        if (this.ws) {
+            this.ws.close();
+            this._setStatus('disconnected');
+            this.options.onDisconnect();
+        }
     }
     /**
      * Gets the microphone MediaStream used by this client