@layercode/js-sdk 1.0.14 → 1.0.16

package/dist/layercode-js-sdk.min.js

@@ -3480,7 +3480,6 @@ registerProcessor('audio_processor', AudioProcessor);
               sessionId: options.sessionId || null,
               authorizeSessionEndpoint: options.authorizeSessionEndpoint,
               metadata: options.metadata || {},
-              vadEnabled: options.vadEnabled || true,
               vadResumeDelay: options.vadResumeDelay || 500,
               onConnect: options.onConnect || (() => { }),
               onDisconnect: options.onDisconnect || (() => { }),
@@ -3498,7 +3497,67 @@ registerProcessor('audio_processor', AudioProcessor);
               sampleRate: 16000, // TODO should be set my fetched pipeline config
           });
           this.vad = null;
-          if (this.options.vadEnabled) {
+          this.ws = null;
+          this.status = 'disconnected';
+          this.userAudioAmplitude = 0;
+          this.agentAudioAmplitude = 0;
+          this.sessionId = options.sessionId || null;
+          this.pushToTalkActive = false;
+          this.vadPausedPlayer = false;
+          this.pushToTalkEnabled = false;
+          this.canInterrupt = false;
+          this.userIsSpeaking = false;
+          // Bind event handlers
+          this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
+          this._handleDataAvailable = this._handleDataAvailable.bind(this);
+      }
+      _initializeVAD() {
+          console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt });
+          // If we're in push to talk mode, we don't need to use the VAD model
+          if (this.pushToTalkEnabled) {
+              return;
+          }
+          const timeout = setTimeout(() => {
+              console.log('silero vad model timeout');
+              // TODO: send message to server to indicate that the vad model timed out
+              this.userIsSpeaking = true; // allow audio to be sent to the server
+          }, 2000);
+          if (!this.canInterrupt) {
+              dist.MicVAD.new({
+                  stream: this.wavRecorder.getStream() || undefined,
+                  model: 'v5',
+                  positiveSpeechThreshold: 0.3,
+                  negativeSpeechThreshold: 0.2,
+                  redemptionFrames: 25, // Number of frames of silence before onVADMisfire or onSpeechEnd is called. Effectively a delay before restarting.
+                  minSpeechFrames: 15,
+                  preSpeechPadFrames: 0,
+                  onSpeechStart: () => {
+                      if (!this.wavPlayer.isPlaying) {
+                          this.userIsSpeaking = true;
+                      }
+                  },
+                  onVADMisfire: () => {
+                      this.userIsSpeaking = false;
+                  },
+                  onSpeechEnd: () => {
+                      this.userIsSpeaking = false;
+                      this._wsSend({
+                          type: 'vad_events',
+                          event: 'vad_end',
+                      });
+                  },
+              })
+                  .then((vad) => {
+                  clearTimeout(timeout);
+                  this.vad = vad;
+                  this.vad.start();
+                  console.log('VAD started');
+              })
+                  .catch((error) => {
+                  console.error('Error initializing VAD:', error);
+              });
+          }
+          else {
               dist.MicVAD.new({
                   stream: this.wavRecorder.getStream() || undefined,
                   model: 'v5',
@@ -3519,36 +3578,35 @@ registerProcessor('audio_processor', AudioProcessor);
                       else {
                           console.log('onSpeechStart: WavPlayer is not playing, VAD will not pause.');
                       }
+                      this.userIsSpeaking = true;
+                      console.log('onSpeechStart: sending vad_start');
+                      this._wsSend({
+                          type: 'vad_events',
+                          event: 'vad_start',
+                      });
                   },
                   onVADMisfire: () => {
                       // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd, and we should resume the assistant audio as it was a false interruption. We include a configurable delay so the assistant isn't too quick to start speaking again.
+                      this.userIsSpeaking = false;
                       if (this.vadPausedPlayer) {
                           console.log('onSpeechEnd: VAD paused the player, resuming');
                           this.wavPlayer.play();
                           this.vadPausedPlayer = false; // Reset flag
-                          // Option to extend delay in the case where the transcriber takes longer to detect a new turn
-                          // console.log('onVADMisfire: VAD paused the player, resuming in ' + this.options.vadResumeDelay + 'ms');
-                          // // Add configurable delay before resuming playback
-                          // setTimeout(() => {
-                          //   this.wavPlayer.play();
-                          //   this.vadPausedPlayer = false; // Reset flag
-                          // }, this.options.vadResumeDelay);
                       }
                       else {
                           console.log('onVADMisfire: VAD did not pause the player, no action taken to resume.');
                       }
                   },
-                  // onSpeechEnd: () => {
-                  //   if (this.vadPausedPlayer) {
-                  //     console.log('onSpeechEnd: VAD paused the player, resuming');
-                  //     this.wavPlayer.play();
-                  //     this.vadPausedPlayer = false; // Reset flag
-                  //   } else {
-                  //     console.log('onSpeechEnd: VAD did not pause the player, not resuming.');
-                  //   }
-                  // },
+                  onSpeechEnd: () => {
+                      this.userIsSpeaking = false;
+                      this._wsSend({
+                          type: 'vad_events',
+                          event: 'vad_end',
+                      });
+                  },
               })
                   .then((vad) => {
+                  clearTimeout(timeout);
                   this.vad = vad;
                   this.vad.start();
                   console.log('VAD started');
@@ -3557,16 +3615,6 @@ registerProcessor('audio_processor', AudioProcessor);
                   console.error('Error initializing VAD:', error);
               });
           }
-          this.ws = null;
-          this.status = 'disconnected';
-          this.userAudioAmplitude = 0;
-          this.agentAudioAmplitude = 0;
-          this.sessionId = options.sessionId || null;
-          this.pushToTalkActive = false;
-          this.vadPausedPlayer = false;
-          // Bind event handlers
-          this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
-          this._handleDataAvailable = this._handleDataAvailable.bind(this);
       }
       /**
        * Updates the connection status and triggers the callback
@@ -3626,10 +3674,9 @@ registerProcessor('audio_processor', AudioProcessor);
                       // Sent from the server to this client when a new user turn is detected
                       console.log('received turn.start from server');
                       console.log(message);
-                      // if (message.role === 'user' && !this.pushToTalkActive) {
-                      if (message.role === 'user') {
+                      if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
                           // Interrupt any playing assistant audio if this is a turn trigged by the server (and not push to talk, which will have already called interrupt)
-                          console.log('interrupting assistant audio, as user turn has started and pushToTalkActive is false');
+                          console.log('interrupting assistant audio, as user turn has started and pushToTalkEnabled is false');
                           await this._clientInterruptAssistantReplay();
                       }
                       // if (message.role === 'assistant') {
@@ -3667,10 +3714,13 @@ registerProcessor('audio_processor', AudioProcessor);
       _handleDataAvailable(data) {
           try {
               const base64 = arrayBufferToBase64(data.mono);
-              this._wsSend({
-                  type: 'client.audio',
-                  content: base64,
-              });
+              const sendAudio = this.pushToTalkEnabled ? this.pushToTalkActive : this.userIsSpeaking;
+              if (sendAudio) {
+                  this._wsSend({
+                      type: 'client.audio',
+                      content: base64,
+                  });
+              }
           }
           catch (error) {
               console.error('Error processing audio:', error);
@@ -3743,6 +3793,19 @@ registerProcessor('audio_processor', AudioProcessor);
               this.ws = new WebSocket(`${this._websocketUrl}?${new URLSearchParams({
                 client_session_key: authorizeSessionResponseBody.client_session_key,
             })}`);
+              const config = authorizeSessionResponseBody.config;
+              console.log('config', config);
+              if (config.transcription.trigger === 'push_to_talk') {
+                  this.pushToTalkEnabled = true;
+              }
+              else if (config.transcription.trigger === 'automatic') {
+                  this.pushToTalkEnabled = false;
+                  this.canInterrupt = config.transcription.can_interrupt;
+              }
+              else {
+                  throw new Error(`Unknown trigger: ${config.transcription.trigger}`);
+              }
+              this._initializeVAD();
               // Bind the websocket message callbacks
               this.ws.onmessage = this._handleWebSocketMessage;
               this.ws.onopen = () => {