@layercode/js-sdk 1.0.26 → 2.0.0

package/README.md

@@ -1,6 +1,6 @@
 # Layercode JavaScript SDK
 
-A JavaScript SDK for integrating Layercode voice pipelines into web applications.
+A JavaScript SDK for integrating Layercode voice agents into web applications.
 
 ## Installation
 

package/dist/layercode-js-sdk.esm.js

@@ -3477,9 +3477,11 @@ function arrayBufferToBase64(arrayBuffer) {
 }
 
 /* eslint-env browser */
+// SDK version - updated when publishing
+const SDK_VERSION = '2.0.0';
 /**
  * @class LayercodeClient
- * @classdesc Core client for Layercode audio pipeline that manages audio recording, WebSocket communication, and speech processing.
+ * @classdesc Core client for Layercode audio agent that manages audio recording, WebSocket communication, and speech processing.
  */
 class LayercodeClient {
     /**
@@ -3487,15 +3489,17 @@ class LayercodeClient {
      * @param {Object} options - Configuration options
      */
     constructor(options) {
+        this.deviceId = null;
         this.options = {
-            pipelineId: options.pipelineId,
-            sessionId: options.sessionId || null,
+            agentId: options.agentId,
+            conversationId: options.conversationId || null,
             authorizeSessionEndpoint: options.authorizeSessionEndpoint,
             metadata: options.metadata || {},
             vadResumeDelay: options.vadResumeDelay || 500,
             onConnect: options.onConnect || (() => { }),
             onDisconnect: options.onDisconnect || (() => { }),
             onError: options.onError || (() => { }),
+            onDeviceSwitched: options.onDeviceSwitched || (() => { }),
             onDataMessage: options.onDataMessage || (() => { }),
             onUserAmplitudeChange: options.onUserAmplitudeChange || (() => { }),
             onAgentAmplitudeChange: options.onAgentAmplitudeChange || (() => { }),
@@ -3503,18 +3507,18 @@ class LayercodeClient {
             onUserIsSpeakingChange: options.onUserIsSpeakingChange || (() => { }),
         };
         this.AMPLITUDE_MONITORING_SAMPLE_RATE = 10;
-        this._websocketUrl = 'wss://api.layercode.com/v1/pipelines/websocket';
-        this.wavRecorder = new WavRecorder({ sampleRate: 8000 }); // TODO should be set my fetched pipeline config
+        this._websocketUrl = 'wss://api.layercode.com/v1/agents/websocket';
+        this.wavRecorder = new WavRecorder({ sampleRate: 8000 }); // TODO should be set my fetched agent config
         this.wavPlayer = new WavStreamPlayer({
             finishedPlayingCallback: this._clientResponseAudioReplayFinished.bind(this),
-            sampleRate: 16000, // TODO should be set my fetched pipeline config
+            sampleRate: 16000, // TODO should be set my fetched agent config
         });
         this.vad = null;
         this.ws = null;
         this.status = 'disconnected';
         this.userAudioAmplitude = 0;
         this.agentAudioAmplitude = 0;
-        this.sessionId = options.sessionId || null;
+        this.conversationId = options.conversationId || null;
         this.pushToTalkActive = false;
         this.pushToTalkEnabled = false;
         this.canInterrupt = false;
@@ -3523,71 +3527,28 @@ class LayercodeClient {
         this.readySent = false;
         this.currentTurnId = null;
         this.audioBuffer = [];
+        this.vadConfig = null;
         // this.audioPauseTime = null;
         // Bind event handlers
         this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
         this._handleDataAvailable = this._handleDataAvailable.bind(this);
-    }
-    _setupAmplitudeBasedVAD() {
-        let isSpeakingByAmplitude = false;
-        let silenceFrames = 0;
-        const AMPLITUDE_THRESHOLD = 0.01; // Adjust based on testing
-        const SILENCE_FRAMES_THRESHOLD = 6.4; // 6.4 * 20ms chunks = 128ms silence. Same as Silero ((frame samples: 512 / sampleRate: 16000) * 1000 * redemptionFrames: 4) = 128 ms silence
-        // Monitor amplitude changes
-        this.wavRecorder.startAmplitudeMonitoring((amplitude) => {
-            const wasSpeaking = isSpeakingByAmplitude;
-            if (amplitude > AMPLITUDE_THRESHOLD) {
-                silenceFrames = 0;
-                if (!wasSpeaking) {
-                    isSpeakingByAmplitude = true;
-                    this.userIsSpeaking = true;
-                    this.options.onUserIsSpeakingChange(true);
-                    this._wsSend({
-                        type: 'vad_events',
-                        event: 'vad_start',
-                    });
-                }
-            }
-            else {
-                silenceFrames++;
-                if (wasSpeaking && silenceFrames >= SILENCE_FRAMES_THRESHOLD) {
-                    isSpeakingByAmplitude = false;
-                    this.userIsSpeaking = false;
-                    this.options.onUserIsSpeakingChange(false);
-                    this._wsSend({
-                        type: 'vad_events',
-                        event: 'vad_end',
-                    });
-                }
-            }
-        });
+        this._setupDeviceChangeListener();
     }
     _initializeVAD() {
-        console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt });
+        var _a;
+        console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt, vadConfig: this.vadConfig });
         // If we're in push to talk mode, we don't need to use the VAD model
         if (this.pushToTalkEnabled) {
             return;
         }
-        const vadLoadTimeout = setTimeout(() => {
-            console.log('silero vad model timeout');
-            console.warn('VAD model failed to load - falling back to amplitude-based detection');
-            // Send a message to server indicating VAD failure
-            this._wsSend({
-                type: 'vad_events',
-                event: 'vad_model_failed',
-            });
-            // Set up amplitude-based fallback detection
-            this._setupAmplitudeBasedVAD();
-        }, 2000);
-        dist.MicVAD.new({
+        // Check if VAD is disabled
+        if (((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.enabled) === false) {
+            console.log('VAD is disabled by backend configuration');
+            return;
+        }
+        // Build VAD configuration object, only including keys that are defined
+        const vadOptions = {
             stream: this.wavRecorder.getStream() || undefined,
-            model: 'v5',
-            positiveSpeechThreshold: 0.15,
-            negativeSpeechThreshold: 0.05,
-            redemptionFrames: 4,
-            minSpeechFrames: 2,
-            preSpeechPadFrames: 0,
-            frameSamples: 512, // Required for v5 as per https://docs.vad.ricky0123.com/user-guide/algorithm/#configuration
             onSpeechStart: () => {
                 console.log('onSpeechStart: sending vad_start');
                 this.userIsSpeaking = true;
@@ -3607,18 +3568,49 @@ class LayercodeClient {
                     event: 'vad_end',
                 });
             },
-            // onVADMisfire: () => {
-            //   // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd.
-            // },
-        })
+        };
+        // Apply VAD configuration from backend if available
+        if (this.vadConfig) {
+            // Only add keys that are explicitly defined (not undefined)
+            if (this.vadConfig.model !== undefined)
+                vadOptions.model = this.vadConfig.model;
+            if (this.vadConfig.positive_speech_threshold !== undefined)
+                vadOptions.positiveSpeechThreshold = this.vadConfig.positive_speech_threshold;
+            if (this.vadConfig.negative_speech_threshold !== undefined)
+                vadOptions.negativeSpeechThreshold = this.vadConfig.negative_speech_threshold;
+            if (this.vadConfig.redemption_frames !== undefined)
+                vadOptions.redemptionFrames = this.vadConfig.redemption_frames;
+            if (this.vadConfig.min_speech_frames !== undefined)
+                vadOptions.minSpeechFrames = this.vadConfig.min_speech_frames;
+            if (this.vadConfig.pre_speech_pad_frames !== undefined)
+                vadOptions.preSpeechPadFrames = this.vadConfig.pre_speech_pad_frames;
+            if (this.vadConfig.frame_samples !== undefined)
+                vadOptions.frameSamples = this.vadConfig.frame_samples;
+        }
+        else {
+            // Default values if no config from backend
+            vadOptions.model = 'v5';
+            vadOptions.positiveSpeechThreshold = 0.15;
+            vadOptions.negativeSpeechThreshold = 0.05;
+            vadOptions.redemptionFrames = 4;
+            vadOptions.minSpeechFrames = 2;
+            vadOptions.preSpeechPadFrames = 0;
+            vadOptions.frameSamples = 512; // Required for v5
+        }
+        console.log('Creating VAD with options:', vadOptions);
+        dist.MicVAD.new(vadOptions)
             .then((vad) => {
-            clearTimeout(vadLoadTimeout);
             this.vad = vad;
             this.vad.start();
-            console.log('VAD started');
+            console.log('VAD started successfully');
         })
             .catch((error) => {
-            console.error('Error initializing VAD:', error);
+            console.warn('Error initializing VAD:', error);
+            // Send a message to server indicating VAD failure
+            this._wsSend({
+                type: 'vad_events',
+                event: 'vad_model_failed',
+            });
         });
     }
     /**
@@ -3722,7 +3714,7 @@ class LayercodeClient {
                     this.options.onDataMessage(message);
                     break;
                 default:
-                    console.error('Unknown message type received:', message);
+                    console.warn('Unknown message type received:', message);
                     break;
             }
         }
@@ -3736,12 +3728,26 @@ class LayercodeClient {
      * @param {ArrayBuffer} data - The audio data buffer
      */
     _handleDataAvailable(data) {
+        var _a, _b, _c;
         try {
             const base64 = arrayBufferToBase64(data.mono);
-            const sendAudio = this.pushToTalkEnabled ? this.pushToTalkActive : this.userIsSpeaking;
+            // Determine if we should gate audio based on VAD configuration
+            const shouldGateAudio = ((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.gate_audio) !== false; // Default to true if not specified
+            const bufferFrames = (_c = (_b = this.vadConfig) === null || _b === void 0 ? void 0 : _b.buffer_frames) !== null && _c !== void 0 ? _c : 10; // Default to 10 if not specified
+            let sendAudio;
+            if (this.pushToTalkEnabled) {
+                sendAudio = this.pushToTalkActive;
+            }
+            else if (shouldGateAudio) {
+                sendAudio = this.userIsSpeaking;
+            }
+            else {
+                // If gate_audio is false, always send audio
+                sendAudio = true;
+            }
             if (sendAudio) {
-                // If we have buffered audio, send it first
-                if (this.audioBuffer.length > 0) {
+                // If we have buffered audio and we're gating, send it first
+                if (shouldGateAudio && this.audioBuffer.length > 0) {
                     console.log(`Sending ${this.audioBuffer.length} buffered audio chunks`);
                     for (const bufferedAudio of this.audioBuffer) {
                         this._wsSend({
@@ -3760,8 +3766,8 @@ class LayercodeClient {
             else {
                 // Buffer audio when not sending (to catch audio just before VAD triggers)
                 this.audioBuffer.push(base64);
-                // Keep buffer size reasonable (e.g., last 10 chunks ≈ 200ms at 20ms chunks)
-                if (this.audioBuffer.length > 10) {
+                // Keep buffer size based on configuration
+                if (this.audioBuffer.length > bufferFrames) {
                     this.audioBuffer.shift(); // Remove oldest chunk
                 }
             }
@@ -3811,7 +3817,7 @@ class LayercodeClient {
         }
     }
     /**
-     * Connects to the Layercode pipeline and starts the audio session
+     * Connects to the Layercode agent and starts the audio conversation
      * @async
      * @returns {Promise<void>}
      */
@@ -3820,14 +3826,15 @@ class LayercodeClient {
             this._setStatus('connecting');
             // Reset turn tracking for clean start
             this._resetTurnTracking();
-            // Get session key from server
+            // Get conversation key from server
             let authorizeSessionRequestBody = {
-                pipeline_id: this.options.pipelineId,
+                agent_id: this.options.agentId,
                 metadata: this.options.metadata,
+                sdk_version: SDK_VERSION,
             };
-            // If we're reconnecting to a previous session, we need to include the session_id in the request. Otherwise we don't send session_id, and a new session will be created and the session_id will be returned in the response.
-            if (this.options.sessionId) {
-                authorizeSessionRequestBody.session_id = this.options.sessionId;
+            // If we're reconnecting to a previous conversation, we need to include the conversation_id in the request. Otherwise we don't send conversation_id, and a new conversation will be created and the conversation_id will be returned in the response.
+            if (this.options.conversationId) {
+                authorizeSessionRequestBody.conversation_id = this.options.conversationId;
             }
             const authorizeSessionResponse = await fetch(this.options.authorizeSessionEndpoint, {
                 method: 'POST',
@@ -3837,16 +3844,18 @@ class LayercodeClient {
                 body: JSON.stringify(authorizeSessionRequestBody),
             });
             if (!authorizeSessionResponse.ok) {
-                throw new Error(`Failed to authorize session: ${authorizeSessionResponse.statusText}`);
+                throw new Error(`Failed to authorize conversation: ${authorizeSessionResponse.statusText}`);
             }
             const authorizeSessionResponseBody = await authorizeSessionResponse.json();
-            this.sessionId = authorizeSessionResponseBody.session_id; // Save the session_id for use in future reconnects
+            this.conversationId = authorizeSessionResponseBody.conversation_id; // Save the conversation_id for use in future reconnects
             // Connect WebSocket
             this.ws = new WebSocket(`${this._websocketUrl}?${new URLSearchParams({
                 client_session_key: authorizeSessionResponseBody.client_session_key,
             })}`);
             const config = authorizeSessionResponseBody.config;
             console.log('config', config);
+            // Store VAD configuration
+            this.vadConfig = config.vad || null;
             if (config.transcription.trigger === 'push_to_talk') {
                 this.pushToTalkEnabled = true;
             }
@@ -3857,13 +3866,12 @@ class LayercodeClient {
             else {
                 throw new Error(`Unknown trigger: ${config.transcription.trigger}`);
             }
-            this._initializeVAD();
             // Bind the websocket message callbacks
             this.ws.onmessage = this._handleWebSocketMessage;
             this.ws.onopen = () => {
                 console.log('WebSocket connection established');
                 this._setStatus('connected');
-                this.options.onConnect({ sessionId: this.sessionId });
+                this.options.onConnect({ conversationId: this.conversationId });
                 // Attempt to send ready message if recorder already started
                 this._sendReadyIfNeeded();
             };
@@ -3877,21 +3885,16 @@ class LayercodeClient {
                 this._setStatus('error');
                 this.options.onError(new Error('WebSocket connection error'));
             };
-            // Initialize microphone audio capture
-            await this.wavRecorder.begin();
-            await this.wavRecorder.record(this._handleDataAvailable, 1638);
-            // Set up microphone amplitude monitoring
-            this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
             // Initialize audio player
             await this.wavPlayer.connect();
             // Set up audio player amplitude monitoring
             this._setupAmplitudeMonitoring(this.wavPlayer, this.options.onAgentAmplitudeChange, (amp) => (this.agentAudioAmplitude = amp));
-            // Mark recorder as started and attempt to notify server
-            this.recorderStarted = true;
-            this._sendReadyIfNeeded();
+            // wavRecorder will be started from the onDeviceSwitched callback,
+            // which is called when the device is first initialized and also when the device is switched
+            // this is to ensure that the device is initialized before the recorder is started
         }
         catch (error) {
-            console.error('Error connecting to Layercode pipeline:', error);
+            console.error('Error connecting to Layercode agent:', error);
             this._setStatus('error');
             this.options.onError(error instanceof Error ? error : new Error(String(error)));
             throw error;
@@ -3908,6 +3911,7 @@ class LayercodeClient {
             this.vad.destroy();
             this.vad = null;
         }
+        this.wavRecorder.listenForDeviceChange(null);
         this.wavRecorder.quit();
         this.wavPlayer.disconnect();
         // Reset turn tracking
@@ -3931,19 +3935,101 @@ class LayercodeClient {
      * @param {string} deviceId - The deviceId of the new microphone
      */
     async setInputDevice(deviceId) {
-        if (this.wavRecorder) {
+        var _a;
+        try {
+            this.deviceId = deviceId;
+            // Restart recording with the new device
+            await this._restartAudioRecording();
+            // Reinitialize VAD with the new audio stream if VAD is enabled
+            const shouldUseVAD = !this.pushToTalkEnabled && ((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.enabled) !== false;
+            if (shouldUseVAD) {
+                console.log('Reinitializing VAD with new audio stream');
+                const newStream = this.wavRecorder.getStream();
+                await this._reinitializeVAD(newStream);
+            }
+            console.log(`Successfully switched to input device: ${deviceId}`);
+        }
+        catch (error) {
+            console.error(`Failed to switch to input device ${deviceId}:`, error);
+            throw new Error(`Failed to switch to input device: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Restarts audio recording after a device switch to ensure audio is captured from the new device
+     */
+    async _restartAudioRecording() {
+        try {
+            console.log('Restarting audio recording after device switch...');
             try {
                 await this.wavRecorder.end();
             }
-            catch (e) { }
-            try {
-                await this.wavRecorder.quit();
+            catch (_a) {
+                // Ignore cleanup errors
             }
-            catch (e) { }
+            // Start with new device
+            await this.wavRecorder.begin(this.deviceId || undefined);
+            await this.wavRecorder.record(this._handleDataAvailable, 1638);
+            // Re-setup amplitude monitoring with the new stream
+            this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
+            console.log('Audio recording restart completed successfully');
+        }
+        catch (error) {
+            console.error('Error restarting audio recording after device switch:', error);
+            this.options.onError(error instanceof Error ? error : new Error(String(error)));
+        }
+    }
+    /**
+     * Reinitializes VAD with a new stream (used after device switching)
+     */
+    async _reinitializeVAD(stream) {
+        // Clean up existing VAD
+        if (this.vad) {
+            this.vad.pause();
+            this.vad.destroy();
+            this.vad = null;
         }
-        await this.wavRecorder.begin(deviceId);
-        await this.wavRecorder.record(this._handleDataAvailable, 1638);
-        this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
+        // Reinitialize with new stream
+        if (stream) {
+            this._initializeVAD();
+        }
+    }
+    /**
+     * Sets up the device change event listener
+     */
+    _setupDeviceChangeListener() {
+        this.wavRecorder.listenForDeviceChange(async (devices) => {
+            try {
+                const currentDeviceExists = devices.some((device) => device.deviceId === this.deviceId);
+                if (!currentDeviceExists) {
+                    console.log('Current device disconnected, switching to next available device');
+                    try {
+                        const nextDevice = devices.find((d) => d.default);
+                        if (nextDevice) {
+                            await this.setInputDevice(nextDevice.deviceId);
+                            // Mark recorder as started and attempt to notify server
+                            if (!this.recorderStarted) {
+                                this.recorderStarted = true;
+                                this._sendReadyIfNeeded();
+                            }
+                            // Notify about device switch
+                            if (this.options.onDeviceSwitched) {
+                                this.options.onDeviceSwitched(nextDevice.deviceId);
+                            }
+                        }
+                        else {
+                            console.warn('No alternative audio device found');
+                        }
+                    }
+                    catch (error) {
+                        console.error('Error switching to next device:', error);
+                        throw error;
+                    }
+                }
+            }
+            catch (error) {
+                this.options.onError(error instanceof Error ? error : new Error(String(error)));
+            }
+        });
     }
 }