@layercode/js-sdk 1.0.26 → 1.0.27

package/dist/layercode-js-sdk.esm.js

@@ -3477,6 +3477,8 @@ function arrayBufferToBase64(arrayBuffer) {
 }
 
 /* eslint-env browser */
+// SDK version - updated when publishing
+const SDK_VERSION = '1.0.27';
 /**
  * @class LayercodeClient
  * @classdesc Core client for Layercode audio pipeline that manages audio recording, WebSocket communication, and speech processing.
@@ -3487,6 +3489,7 @@ class LayercodeClient {
      * @param {Object} options - Configuration options
      */
     constructor(options) {
+        this.deviceId = null;
         this.options = {
             pipelineId: options.pipelineId,
             sessionId: options.sessionId || null,
@@ -3496,6 +3499,7 @@ class LayercodeClient {
             onConnect: options.onConnect || (() => { }),
             onDisconnect: options.onDisconnect || (() => { }),
             onError: options.onError || (() => { }),
+            onDeviceSwitched: options.onDeviceSwitched || (() => { }),
             onDataMessage: options.onDataMessage || (() => { }),
             onUserAmplitudeChange: options.onUserAmplitudeChange || (() => { }),
             onAgentAmplitudeChange: options.onAgentAmplitudeChange || (() => { }),
@@ -3523,71 +3527,28 @@ class LayercodeClient {
         this.readySent = false;
         this.currentTurnId = null;
         this.audioBuffer = [];
+        this.vadConfig = null;
         // this.audioPauseTime = null;
         // Bind event handlers
         this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
         this._handleDataAvailable = this._handleDataAvailable.bind(this);
-    }
-    _setupAmplitudeBasedVAD() {
-        let isSpeakingByAmplitude = false;
-        let silenceFrames = 0;
-        const AMPLITUDE_THRESHOLD = 0.01; // Adjust based on testing
-        const SILENCE_FRAMES_THRESHOLD = 6.4; // 6.4 * 20ms chunks = 128ms silence. Same as Silero ((frame samples: 512 / sampleRate: 16000) * 1000 * redemptionFrames: 4) = 128 ms silence
-        // Monitor amplitude changes
-        this.wavRecorder.startAmplitudeMonitoring((amplitude) => {
-            const wasSpeaking = isSpeakingByAmplitude;
-            if (amplitude > AMPLITUDE_THRESHOLD) {
-                silenceFrames = 0;
-                if (!wasSpeaking) {
-                    isSpeakingByAmplitude = true;
-                    this.userIsSpeaking = true;
-                    this.options.onUserIsSpeakingChange(true);
-                    this._wsSend({
-                        type: 'vad_events',
-                        event: 'vad_start',
-                    });
-                }
-            }
-            else {
-                silenceFrames++;
-                if (wasSpeaking && silenceFrames >= SILENCE_FRAMES_THRESHOLD) {
-                    isSpeakingByAmplitude = false;
-                    this.userIsSpeaking = false;
-                    this.options.onUserIsSpeakingChange(false);
-                    this._wsSend({
-                        type: 'vad_events',
-                        event: 'vad_end',
-                    });
-                }
-            }
-        });
+        this._setupDeviceChangeListener();
     }
     _initializeVAD() {
-        console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt });
+        var _a;
+        console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt, vadConfig: this.vadConfig });
         // If we're in push to talk mode, we don't need to use the VAD model
         if (this.pushToTalkEnabled) {
             return;
         }
-        const vadLoadTimeout = setTimeout(() => {
-            console.log('silero vad model timeout');
-            console.warn('VAD model failed to load - falling back to amplitude-based detection');
-            // Send a message to server indicating VAD failure
-            this._wsSend({
-                type: 'vad_events',
-                event: 'vad_model_failed',
-            });
-            // Set up amplitude-based fallback detection
-            this._setupAmplitudeBasedVAD();
-        }, 2000);
-        dist.MicVAD.new({
+        // Check if VAD is disabled
+        if (((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.enabled) === false) {
+            console.log('VAD is disabled by backend configuration');
+            return;
+        }
+        // Build VAD configuration object, only including keys that are defined
+        const vadOptions = {
             stream: this.wavRecorder.getStream() || undefined,
-            model: 'v5',
-            positiveSpeechThreshold: 0.15,
-            negativeSpeechThreshold: 0.05,
-            redemptionFrames: 4,
-            minSpeechFrames: 2,
-            preSpeechPadFrames: 0,
-            frameSamples: 512, // Required for v5 as per https://docs.vad.ricky0123.com/user-guide/algorithm/#configuration
             onSpeechStart: () => {
                 console.log('onSpeechStart: sending vad_start');
                 this.userIsSpeaking = true;
@@ -3607,18 +3568,49 @@ class LayercodeClient {
                     event: 'vad_end',
                 });
             },
-            // onVADMisfire: () => {
-            //   // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd.
-            // },
-        })
+        };
+        // Apply VAD configuration from backend if available
+        if (this.vadConfig) {
+            // Only add keys that are explicitly defined (not undefined)
+            if (this.vadConfig.model !== undefined)
+                vadOptions.model = this.vadConfig.model;
+            if (this.vadConfig.positive_speech_threshold !== undefined)
+                vadOptions.positiveSpeechThreshold = this.vadConfig.positive_speech_threshold;
+            if (this.vadConfig.negative_speech_threshold !== undefined)
+                vadOptions.negativeSpeechThreshold = this.vadConfig.negative_speech_threshold;
+            if (this.vadConfig.redemption_frames !== undefined)
+                vadOptions.redemptionFrames = this.vadConfig.redemption_frames;
+            if (this.vadConfig.min_speech_frames !== undefined)
+                vadOptions.minSpeechFrames = this.vadConfig.min_speech_frames;
+            if (this.vadConfig.pre_speech_pad_frames !== undefined)
+                vadOptions.preSpeechPadFrames = this.vadConfig.pre_speech_pad_frames;
+            if (this.vadConfig.frame_samples !== undefined)
+                vadOptions.frameSamples = this.vadConfig.frame_samples;
+        }
+        else {
+            // Default values if no config from backend
+            vadOptions.model = 'v5';
+            vadOptions.positiveSpeechThreshold = 0.15;
+            vadOptions.negativeSpeechThreshold = 0.05;
+            vadOptions.redemptionFrames = 4;
+            vadOptions.minSpeechFrames = 2;
+            vadOptions.preSpeechPadFrames = 0;
+            vadOptions.frameSamples = 512; // Required for v5
+        }
+        console.log('Creating VAD with options:', vadOptions);
+        dist.MicVAD.new(vadOptions)
             .then((vad) => {
-            clearTimeout(vadLoadTimeout);
             this.vad = vad;
             this.vad.start();
-            console.log('VAD started');
+            console.log('VAD started successfully');
         })
             .catch((error) => {
-            console.error('Error initializing VAD:', error);
+            console.warn('Error initializing VAD:', error);
+            // Send a message to server indicating VAD failure
+            this._wsSend({
+                type: 'vad_events',
+                event: 'vad_model_failed',
+            });
         });
     }
     /**
@@ -3722,7 +3714,7 @@ class LayercodeClient {
                     this.options.onDataMessage(message);
                     break;
                 default:
-                    console.error('Unknown message type received:', message);
+                    console.warn('Unknown message type received:', message);
                     break;
             }
         }
@@ -3736,12 +3728,26 @@ class LayercodeClient {
      * @param {ArrayBuffer} data - The audio data buffer
      */
     _handleDataAvailable(data) {
+        var _a, _b, _c;
         try {
             const base64 = arrayBufferToBase64(data.mono);
-            const sendAudio = this.pushToTalkEnabled ? this.pushToTalkActive : this.userIsSpeaking;
+            // Determine if we should gate audio based on VAD configuration
+            const shouldGateAudio = ((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.gate_audio) !== false; // Default to true if not specified
+            const bufferFrames = (_c = (_b = this.vadConfig) === null || _b === void 0 ? void 0 : _b.buffer_frames) !== null && _c !== void 0 ? _c : 10; // Default to 10 if not specified
+            let sendAudio;
+            if (this.pushToTalkEnabled) {
+                sendAudio = this.pushToTalkActive;
+            }
+            else if (shouldGateAudio) {
+                sendAudio = this.userIsSpeaking;
+            }
+            else {
+                // If gate_audio is false, always send audio
+                sendAudio = true;
+            }
             if (sendAudio) {
-                // If we have buffered audio, send it first
-                if (this.audioBuffer.length > 0) {
+                // If we have buffered audio and we're gating, send it first
+                if (shouldGateAudio && this.audioBuffer.length > 0) {
                     console.log(`Sending ${this.audioBuffer.length} buffered audio chunks`);
                     for (const bufferedAudio of this.audioBuffer) {
                         this._wsSend({
@@ -3760,8 +3766,8 @@ class LayercodeClient {
             else {
                 // Buffer audio when not sending (to catch audio just before VAD triggers)
                 this.audioBuffer.push(base64);
-                // Keep buffer size reasonable (e.g., last 10 chunks ≈ 200ms at 20ms chunks)
-                if (this.audioBuffer.length > 10) {
+                // Keep buffer size based on configuration
+                if (this.audioBuffer.length > bufferFrames) {
                     this.audioBuffer.shift(); // Remove oldest chunk
                 }
             }
@@ -3824,6 +3830,7 @@ class LayercodeClient {
             let authorizeSessionRequestBody = {
                 pipeline_id: this.options.pipelineId,
                 metadata: this.options.metadata,
+                sdk_version: SDK_VERSION,
             };
             // If we're reconnecting to a previous session, we need to include the session_id in the request. Otherwise we don't send session_id, and a new session will be created and the session_id will be returned in the response.
             if (this.options.sessionId) {
@@ -3847,6 +3854,8 @@ class LayercodeClient {
             })}`);
             const config = authorizeSessionResponseBody.config;
             console.log('config', config);
+            // Store VAD configuration
+            this.vadConfig = config.vad || null;
             if (config.transcription.trigger === 'push_to_talk') {
                 this.pushToTalkEnabled = true;
             }
@@ -3857,7 +3866,6 @@ class LayercodeClient {
             else {
                 throw new Error(`Unknown trigger: ${config.transcription.trigger}`);
             }
-            this._initializeVAD();
             // Bind the websocket message callbacks
             this.ws.onmessage = this._handleWebSocketMessage;
             this.ws.onopen = () => {
@@ -3877,18 +3885,13 @@ class LayercodeClient {
                 this._setStatus('error');
                 this.options.onError(new Error('WebSocket connection error'));
             };
-            // Initialize microphone audio capture
-            await this.wavRecorder.begin();
-            await this.wavRecorder.record(this._handleDataAvailable, 1638);
-            // Set up microphone amplitude monitoring
-            this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
             // Initialize audio player
             await this.wavPlayer.connect();
             // Set up audio player amplitude monitoring
             this._setupAmplitudeMonitoring(this.wavPlayer, this.options.onAgentAmplitudeChange, (amp) => (this.agentAudioAmplitude = amp));
-            // Mark recorder as started and attempt to notify server
-            this.recorderStarted = true;
-            this._sendReadyIfNeeded();
+            // wavRecorder will be started from the onDeviceSwitched callback,
+            // which is called when the device is first initialized and also when the device is switched
+            // this is to ensure that the device is initialized before the recorder is started
         }
         catch (error) {
             console.error('Error connecting to Layercode pipeline:', error);
@@ -3908,6 +3911,7 @@ class LayercodeClient {
             this.vad.destroy();
             this.vad = null;
         }
+        this.wavRecorder.listenForDeviceChange(null);
         this.wavRecorder.quit();
         this.wavPlayer.disconnect();
         // Reset turn tracking
@@ -3931,19 +3935,101 @@ class LayercodeClient {
      * @param {string} deviceId - The deviceId of the new microphone
      */
     async setInputDevice(deviceId) {
-        if (this.wavRecorder) {
+        var _a;
+        try {
+            this.deviceId = deviceId;
+            // Restart recording with the new device
+            await this._restartAudioRecording();
+            // Reinitialize VAD with the new audio stream if VAD is enabled
+            const shouldUseVAD = !this.pushToTalkEnabled && ((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.enabled) !== false;
+            if (shouldUseVAD) {
+                console.log('Reinitializing VAD with new audio stream');
+                const newStream = this.wavRecorder.getStream();
+                await this._reinitializeVAD(newStream);
+            }
+            console.log(`Successfully switched to input device: ${deviceId}`);
+        }
+        catch (error) {
+            console.error(`Failed to switch to input device ${deviceId}:`, error);
+            throw new Error(`Failed to switch to input device: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Restarts audio recording after a device switch to ensure audio is captured from the new device
+     */
+    async _restartAudioRecording() {
+        try {
+            console.log('Restarting audio recording after device switch...');
             try {
                 await this.wavRecorder.end();
             }
-            catch (e) { }
-            try {
-                await this.wavRecorder.quit();
+            catch (_a) {
+                // Ignore cleanup errors
             }
-            catch (e) { }
+            // Start with new device
+            await this.wavRecorder.begin(this.deviceId || undefined);
+            await this.wavRecorder.record(this._handleDataAvailable, 1638);
+            // Re-setup amplitude monitoring with the new stream
+            this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
+            console.log('Audio recording restart completed successfully');
+        }
+        catch (error) {
+            console.error('Error restarting audio recording after device switch:', error);
+            this.options.onError(error instanceof Error ? error : new Error(String(error)));
+        }
+    }
+    /**
+     * Reinitializes VAD with a new stream (used after device switching)
+     */
+    async _reinitializeVAD(stream) {
+        // Clean up existing VAD
+        if (this.vad) {
+            this.vad.pause();
+            this.vad.destroy();
+            this.vad = null;
         }
-        await this.wavRecorder.begin(deviceId);
-        await this.wavRecorder.record(this._handleDataAvailable, 1638);
-        this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
+        // Reinitialize with new stream
+        if (stream) {
+            this._initializeVAD();
+        }
+    }
+    /**
+     * Sets up the device change event listener
+     */
+    _setupDeviceChangeListener() {
+        this.wavRecorder.listenForDeviceChange(async (devices) => {
+            try {
+                const currentDeviceExists = devices.some((device) => device.deviceId === this.deviceId);
+                if (!currentDeviceExists) {
+                    console.log('Current device disconnected, switching to next available device');
+                    try {
+                        const nextDevice = devices.find((d) => d.default);
+                        if (nextDevice) {
+                            await this.setInputDevice(nextDevice.deviceId);
+                            // Mark recorder as started and attempt to notify server
+                            if (!this.recorderStarted) {
+                                this.recorderStarted = true;
+                                this._sendReadyIfNeeded();
+                            }
+                            // Notify about device switch
+                            if (this.options.onDeviceSwitched) {
+                                this.options.onDeviceSwitched(nextDevice.deviceId);
+                            }
+                        }
+                        else {
+                            console.warn('No alternative audio device found');
+                        }
+                    }
+                    catch (error) {
+                        console.error('Error switching to next device:', error);
+                        throw error;
+                    }
+                }
+            }
+            catch (error) {
+                this.options.onError(error instanceof Error ? error : new Error(String(error)));
+            }
+        });
     }
 }