@layercode/js-sdk 1.0.26 → 1.0.27

package/dist/layercode-js-sdk.min.js

@@ -3483,6 +3483,8 @@ registerProcessor('audio_processor', AudioProcessor);
   }
 
   /* eslint-env browser */
+  // SDK version - updated when publishing
+  const SDK_VERSION = '1.0.27';
   /**
    * @class LayercodeClient
    * @classdesc Core client for Layercode audio pipeline that manages audio recording, WebSocket communication, and speech processing.
@@ -3493,6 +3495,7 @@ registerProcessor('audio_processor', AudioProcessor);
        * @param {Object} options - Configuration options
        */
       constructor(options) {
+          this.deviceId = null;
           this.options = {
               pipelineId: options.pipelineId,
               sessionId: options.sessionId || null,
@@ -3502,6 +3505,7 @@ registerProcessor('audio_processor', AudioProcessor);
               onConnect: options.onConnect || (() => { }),
               onDisconnect: options.onDisconnect || (() => { }),
               onError: options.onError || (() => { }),
+              onDeviceSwitched: options.onDeviceSwitched || (() => { }),
               onDataMessage: options.onDataMessage || (() => { }),
               onUserAmplitudeChange: options.onUserAmplitudeChange || (() => { }),
               onAgentAmplitudeChange: options.onAgentAmplitudeChange || (() => { }),
@@ -3529,71 +3533,28 @@ registerProcessor('audio_processor', AudioProcessor);
           this.readySent = false;
           this.currentTurnId = null;
           this.audioBuffer = [];
+          this.vadConfig = null;
           // this.audioPauseTime = null;
           // Bind event handlers
           this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
           this._handleDataAvailable = this._handleDataAvailable.bind(this);
-      }
-      _setupAmplitudeBasedVAD() {
-          let isSpeakingByAmplitude = false;
-          let silenceFrames = 0;
-          const AMPLITUDE_THRESHOLD = 0.01; // Adjust based on testing
-          const SILENCE_FRAMES_THRESHOLD = 6.4; // 6.4 * 20ms chunks = 128ms silence. Same as Silero ((frame samples: 512 / sampleRate: 16000) * 1000 * redemptionFrames: 4) = 128 ms silence
-          // Monitor amplitude changes
-          this.wavRecorder.startAmplitudeMonitoring((amplitude) => {
-              const wasSpeaking = isSpeakingByAmplitude;
-              if (amplitude > AMPLITUDE_THRESHOLD) {
-                  silenceFrames = 0;
-                  if (!wasSpeaking) {
-                      isSpeakingByAmplitude = true;
-                      this.userIsSpeaking = true;
-                      this.options.onUserIsSpeakingChange(true);
-                      this._wsSend({
-                          type: 'vad_events',
-                          event: 'vad_start',
-                      });
-                  }
-              }
-              else {
-                  silenceFrames++;
-                  if (wasSpeaking && silenceFrames >= SILENCE_FRAMES_THRESHOLD) {
-                      isSpeakingByAmplitude = false;
-                      this.userIsSpeaking = false;
-                      this.options.onUserIsSpeakingChange(false);
-                      this._wsSend({
-                          type: 'vad_events',
-                          event: 'vad_end',
-                      });
-                  }
-              }
-          });
+          this._setupDeviceChangeListener();
       }
       _initializeVAD() {
-          console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt });
+          var _a;
+          console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt, vadConfig: this.vadConfig });
           // If we're in push to talk mode, we don't need to use the VAD model
           if (this.pushToTalkEnabled) {
               return;
           }
-          const vadLoadTimeout = setTimeout(() => {
-              console.log('silero vad model timeout');
-              console.warn('VAD model failed to load - falling back to amplitude-based detection');
-              // Send a message to server indicating VAD failure
-              this._wsSend({
-                  type: 'vad_events',
-                  event: 'vad_model_failed',
-              });
-              // Set up amplitude-based fallback detection
-              this._setupAmplitudeBasedVAD();
-          }, 2000);
-          dist.MicVAD.new({
+          // Check if VAD is disabled
+          if (((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.enabled) === false) {
+              console.log('VAD is disabled by backend configuration');
+              return;
+          }
+          // Build VAD configuration object, only including keys that are defined
+          const vadOptions = {
               stream: this.wavRecorder.getStream() || undefined,
-              model: 'v5',
-              positiveSpeechThreshold: 0.15,
-              negativeSpeechThreshold: 0.05,
-              redemptionFrames: 4,
-              minSpeechFrames: 2,
-              preSpeechPadFrames: 0,
-              frameSamples: 512, // Required for v5 as per https://docs.vad.ricky0123.com/user-guide/algorithm/#configuration
               onSpeechStart: () => {
                   console.log('onSpeechStart: sending vad_start');
                   this.userIsSpeaking = true;
@@ -3613,18 +3574,49 @@ registerProcessor('audio_processor', AudioProcessor);
                       event: 'vad_end',
                   });
               },
-              // onVADMisfire: () => {
-              //   // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd.
-              // },
-          })
+          };
+          // Apply VAD configuration from backend if available
+          if (this.vadConfig) {
+              // Only add keys that are explicitly defined (not undefined)
+              if (this.vadConfig.model !== undefined)
+                  vadOptions.model = this.vadConfig.model;
+              if (this.vadConfig.positive_speech_threshold !== undefined)
+                  vadOptions.positiveSpeechThreshold = this.vadConfig.positive_speech_threshold;
+              if (this.vadConfig.negative_speech_threshold !== undefined)
+                  vadOptions.negativeSpeechThreshold = this.vadConfig.negative_speech_threshold;
+              if (this.vadConfig.redemption_frames !== undefined)
+                  vadOptions.redemptionFrames = this.vadConfig.redemption_frames;
+              if (this.vadConfig.min_speech_frames !== undefined)
+                  vadOptions.minSpeechFrames = this.vadConfig.min_speech_frames;
+              if (this.vadConfig.pre_speech_pad_frames !== undefined)
+                  vadOptions.preSpeechPadFrames = this.vadConfig.pre_speech_pad_frames;
+              if (this.vadConfig.frame_samples !== undefined)
+                  vadOptions.frameSamples = this.vadConfig.frame_samples;
+          }
+          else {
+              // Default values if no config from backend
+              vadOptions.model = 'v5';
+              vadOptions.positiveSpeechThreshold = 0.15;
+              vadOptions.negativeSpeechThreshold = 0.05;
+              vadOptions.redemptionFrames = 4;
+              vadOptions.minSpeechFrames = 2;
+              vadOptions.preSpeechPadFrames = 0;
+              vadOptions.frameSamples = 512; // Required for v5
+          }
+          console.log('Creating VAD with options:', vadOptions);
+          dist.MicVAD.new(vadOptions)
               .then((vad) => {
-              clearTimeout(vadLoadTimeout);
               this.vad = vad;
               this.vad.start();
-              console.log('VAD started');
+              console.log('VAD started successfully');
           })
               .catch((error) => {
-              console.error('Error initializing VAD:', error);
+              console.warn('Error initializing VAD:', error);
+              // Send a message to server indicating VAD failure
+              this._wsSend({
+                  type: 'vad_events',
+                  event: 'vad_model_failed',
+              });
           });
       }
       /**
@@ -3728,7 +3720,7 @@ registerProcessor('audio_processor', AudioProcessor);
                       this.options.onDataMessage(message);
                       break;
                   default:
-                      console.error('Unknown message type received:', message);
+                      console.warn('Unknown message type received:', message);
                       break;
               }
           }
@@ -3742,12 +3734,26 @@ registerProcessor('audio_processor', AudioProcessor);
        * @param {ArrayBuffer} data - The audio data buffer
        */
       _handleDataAvailable(data) {
+          var _a, _b, _c;
           try {
               const base64 = arrayBufferToBase64(data.mono);
-              const sendAudio = this.pushToTalkEnabled ? this.pushToTalkActive : this.userIsSpeaking;
+              // Determine if we should gate audio based on VAD configuration
+              const shouldGateAudio = ((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.gate_audio) !== false; // Default to true if not specified
+              const bufferFrames = (_c = (_b = this.vadConfig) === null || _b === void 0 ? void 0 : _b.buffer_frames) !== null && _c !== void 0 ? _c : 10; // Default to 10 if not specified
+              let sendAudio;
+              if (this.pushToTalkEnabled) {
+                  sendAudio = this.pushToTalkActive;
+              }
+              else if (shouldGateAudio) {
+                  sendAudio = this.userIsSpeaking;
+              }
+              else {
+                  // If gate_audio is false, always send audio
+                  sendAudio = true;
+              }
               if (sendAudio) {
-                  // If we have buffered audio, send it first
-                  if (this.audioBuffer.length > 0) {
+                  // If we have buffered audio and we're gating, send it first
+                  if (shouldGateAudio && this.audioBuffer.length > 0) {
                       console.log(`Sending ${this.audioBuffer.length} buffered audio chunks`);
                       for (const bufferedAudio of this.audioBuffer) {
                           this._wsSend({
@@ -3766,8 +3772,8 @@ registerProcessor('audio_processor', AudioProcessor);
               else {
                   // Buffer audio when not sending (to catch audio just before VAD triggers)
                   this.audioBuffer.push(base64);
-                  // Keep buffer size reasonable (e.g., last 10 chunks ≈ 200ms at 20ms chunks)
-                  if (this.audioBuffer.length > 10) {
+                  // Keep buffer size based on configuration
+                  if (this.audioBuffer.length > bufferFrames) {
                       this.audioBuffer.shift(); // Remove oldest chunk
                   }
               }
@@ -3830,6 +3836,7 @@ registerProcessor('audio_processor', AudioProcessor);
               let authorizeSessionRequestBody = {
                   pipeline_id: this.options.pipelineId,
                   metadata: this.options.metadata,
+                  sdk_version: SDK_VERSION,
               };
               // If we're reconnecting to a previous session, we need to include the session_id in the request. Otherwise we don't send session_id, and a new session will be created and the session_id will be returned in the response.
               if (this.options.sessionId) {
@@ -3853,6 +3860,8 @@ registerProcessor('audio_processor', AudioProcessor);
             })}`);
               const config = authorizeSessionResponseBody.config;
               console.log('config', config);
+              // Store VAD configuration
+              this.vadConfig = config.vad || null;
               if (config.transcription.trigger === 'push_to_talk') {
                   this.pushToTalkEnabled = true;
               }
@@ -3863,7 +3872,6 @@ registerProcessor('audio_processor', AudioProcessor);
               else {
                   throw new Error(`Unknown trigger: ${config.transcription.trigger}`);
               }
-              this._initializeVAD();
               // Bind the websocket message callbacks
               this.ws.onmessage = this._handleWebSocketMessage;
               this.ws.onopen = () => {
@@ -3883,18 +3891,13 @@ registerProcessor('audio_processor', AudioProcessor);
                   this._setStatus('error');
                   this.options.onError(new Error('WebSocket connection error'));
               };
-              // Initialize microphone audio capture
-              await this.wavRecorder.begin();
-              await this.wavRecorder.record(this._handleDataAvailable, 1638);
-              // Set up microphone amplitude monitoring
-              this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
               // Initialize audio player
               await this.wavPlayer.connect();
               // Set up audio player amplitude monitoring
               this._setupAmplitudeMonitoring(this.wavPlayer, this.options.onAgentAmplitudeChange, (amp) => (this.agentAudioAmplitude = amp));
-              // Mark recorder as started and attempt to notify server
-              this.recorderStarted = true;
-              this._sendReadyIfNeeded();
+              // wavRecorder will be started from the onDeviceSwitched callback,
+              // which is called when the device is first initialized and also when the device is switched
+              // this is to ensure that the device is initialized before the recorder is started
           }
           catch (error) {
               console.error('Error connecting to Layercode pipeline:', error);
@@ -3914,6 +3917,7 @@ registerProcessor('audio_processor', AudioProcessor);
               this.vad.destroy();
               this.vad = null;
           }
+          this.wavRecorder.listenForDeviceChange(null);
           this.wavRecorder.quit();
           this.wavPlayer.disconnect();
           // Reset turn tracking
@@ -3937,19 +3941,101 @@ registerProcessor('audio_processor', AudioProcessor);
        * @param {string} deviceId - The deviceId of the new microphone
        */
       async setInputDevice(deviceId) {
-          if (this.wavRecorder) {
+          var _a;
+          try {
+              this.deviceId = deviceId;
+              // Restart recording with the new device
+              await this._restartAudioRecording();
+              // Reinitialize VAD with the new audio stream if VAD is enabled
+              const shouldUseVAD = !this.pushToTalkEnabled && ((_a = this.vadConfig) === null || _a === void 0 ? void 0 : _a.enabled) !== false;
+              if (shouldUseVAD) {
+                  console.log('Reinitializing VAD with new audio stream');
+                  const newStream = this.wavRecorder.getStream();
+                  await this._reinitializeVAD(newStream);
+              }
+              console.log(`Successfully switched to input device: ${deviceId}`);
+          }
+          catch (error) {
+              console.error(`Failed to switch to input device ${deviceId}:`, error);
+              throw new Error(`Failed to switch to input device: ${error instanceof Error ? error.message : String(error)}`);
+          }
+      }
+      /**
+       * Restarts audio recording after a device switch to ensure audio is captured from the new device
+       */
+      async _restartAudioRecording() {
+          try {
+              console.log('Restarting audio recording after device switch...');
               try {
                   await this.wavRecorder.end();
               }
-              catch (e) { }
-              try {
-                  await this.wavRecorder.quit();
+              catch (_a) {
+                  // Ignore cleanup errors
               }
-              catch (e) { }
+              // Start with new device
+              await this.wavRecorder.begin(this.deviceId || undefined);
+              await this.wavRecorder.record(this._handleDataAvailable, 1638);
+              // Re-setup amplitude monitoring with the new stream
+              this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
+              console.log('Audio recording restart completed successfully');
+          }
+          catch (error) {
+              console.error('Error restarting audio recording after device switch:', error);
+              this.options.onError(error instanceof Error ? error : new Error(String(error)));
+          }
+      }
+      /**
+       * Reinitializes VAD with a new stream (used after device switching)
+       */
+      async _reinitializeVAD(stream) {
+          // Clean up existing VAD
+          if (this.vad) {
+              this.vad.pause();
+              this.vad.destroy();
+              this.vad = null;
           }
-          await this.wavRecorder.begin(deviceId);
-          await this.wavRecorder.record(this._handleDataAvailable, 1638);
-          this._setupAmplitudeMonitoring(this.wavRecorder, this.options.onUserAmplitudeChange, (amp) => (this.userAudioAmplitude = amp));
+          // Reinitialize with new stream
+          if (stream) {
+              this._initializeVAD();
+          }
+      }
+      /**
+       * Sets up the device change event listener
+       */
+      _setupDeviceChangeListener() {
+          this.wavRecorder.listenForDeviceChange(async (devices) => {
+              try {
+                  const currentDeviceExists = devices.some((device) => device.deviceId === this.deviceId);
+                  if (!currentDeviceExists) {
+                      console.log('Current device disconnected, switching to next available device');
+                      try {
+                          const nextDevice = devices.find((d) => d.default);
+                          if (nextDevice) {
+                              await this.setInputDevice(nextDevice.deviceId);
+                              // Mark recorder as started and attempt to notify server
+                              if (!this.recorderStarted) {
+                                  this.recorderStarted = true;
+                                  this._sendReadyIfNeeded();
+                              }
+                              // Notify about device switch
+                              if (this.options.onDeviceSwitched) {
+                                  this.options.onDeviceSwitched(nextDevice.deviceId);
+                              }
+                          }
+                          else {
+                              console.warn('No alternative audio device found');
+                          }
+                      }
+                      catch (error) {
+                          console.error('Error switching to next device:', error);
+                          throw error;
+                      }
+                  }
+              }
+              catch (error) {
+                  this.options.onError(error instanceof Error ? error : new Error(String(error)));
+              }
+          });
       }
   }