@layercode/js-sdk 1.0.13 → 1.0.15

package/dist/layercode-js-sdk.min.js

@@ -3480,7 +3480,7 @@ registerProcessor('audio_processor', AudioProcessor);
               sessionId: options.sessionId || null,
               authorizeSessionEndpoint: options.authorizeSessionEndpoint,
               metadata: options.metadata || {},
-              vadEnabled: options.vadEnabled || true,
+              vadResumeDelay: options.vadResumeDelay || 500,
               onConnect: options.onConnect || (() => { }),
               onDisconnect: options.onDisconnect || (() => { }),
               onError: options.onError || (() => { }),
@@ -3497,16 +3497,31 @@ registerProcessor('audio_processor', AudioProcessor);
               sampleRate: 16000, // TODO should be set my fetched pipeline config
           });
           this.vad = null;
-          if (this.options.vadEnabled) {
+          this.ws = null;
+          this.status = 'disconnected';
+          this.userAudioAmplitude = 0;
+          this.agentAudioAmplitude = 0;
+          this.sessionId = options.sessionId || null;
+          this.pushToTalkActive = false;
+          this.vadPausedPlayer = false;
+          this.pushToTalkEnabled = false;
+          this.canInterrupt = false;
+          // Bind event handlers
+          this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
+          this._handleDataAvailable = this._handleDataAvailable.bind(this);
+      }
+      _initializeVAD() {
+          console.log('initializing VAD', { pushToTalkEnabled: this.pushToTalkEnabled, canInterrupt: this.canInterrupt });
+          if (!this.pushToTalkEnabled && this.canInterrupt) {
               dist.MicVAD.new({
                   stream: this.wavRecorder.getStream() || undefined,
                   model: 'v5',
                   // baseAssetPath: '/', // Use if bundling model locally
                   // onnxWASMBasePath: '/', // Use if bundling model locally
-                  positiveSpeechThreshold: 0.4,
-                  negativeSpeechThreshold: 0.4,
-                  redemptionFrames: 8,
-                  minSpeechFrames: 20,
+                  positiveSpeechThreshold: 0.3,
+                  negativeSpeechThreshold: 0.2,
+                  redemptionFrames: 25, // Number of frames of silence before onVADMisfire or onSpeechEnd is called. Effectively a delay before restarting.
+                  minSpeechFrames: 15,
                   preSpeechPadFrames: 0,
                   onSpeechStart: () => {
                       // Only pause agent audio if it's currently playing
@@ -3517,23 +3532,35 @@ registerProcessor('audio_processor', AudioProcessor);
                       }
                       else {
                           console.log('onSpeechStart: WavPlayer is not playing, VAD will not pause.');
-                          this.vadPausedPlayer = false;
                       }
                   },
                   onVADMisfire: () => {
-                      // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd, and we should resume the assistant audio as it was a false interruption
+                      // If the speech detected was for less than minSpeechFrames, this is called instead of onSpeechEnd, and we should resume the assistant audio as it was a false interruption. We include a configurable delay so the assistant isn't too quick to start speaking again.
                       if (this.vadPausedPlayer) {
-                          console.log('onVADMisfire: VAD paused the player, attempting to resume.');
+                          console.log('onSpeechEnd: VAD paused the player, resuming');
                           this.wavPlayer.play();
                           this.vadPausedPlayer = false; // Reset flag
+                          // Option to extend delay in the case where the transcriber takes longer to detect a new turn
+                          // console.log('onVADMisfire: VAD paused the player, resuming in ' + this.options.vadResumeDelay + 'ms');
+                          // // Add configurable delay before resuming playback
+                          // setTimeout(() => {
+                          //   this.wavPlayer.play();
+                          //   this.vadPausedPlayer = false; // Reset flag
+                          // }, this.options.vadResumeDelay);
                       }
                       else {
                           console.log('onVADMisfire: VAD did not pause the player, no action taken to resume.');
                       }
                   },
-                  onSpeechEnd: () => {
-                      // We don't take any action here, as the user speech is for more than minSpeechFrames and is very likely to result in a new turn start from the transcriber
-                  },
+                  // onSpeechEnd: () => {
+                  //   if (this.vadPausedPlayer) {
+                  //     console.log('onSpeechEnd: VAD paused the player, resuming');
+                  //     this.wavPlayer.play();
+                  //     this.vadPausedPlayer = false; // Reset flag
+                  //   } else {
+                  //     console.log('onSpeechEnd: VAD did not pause the player, not resuming.');
+                  //   }
+                  // },
               })
                   .then((vad) => {
                   this.vad = vad;
@@ -3544,16 +3571,6 @@ registerProcessor('audio_processor', AudioProcessor);
                   console.error('Error initializing VAD:', error);
               });
           }
-          this.ws = null;
-          this.status = 'disconnected';
-          this.userAudioAmplitude = 0;
-          this.agentAudioAmplitude = 0;
-          this.sessionId = options.sessionId || null;
-          this.pushToTalkActive = false;
-          this.vadPausedPlayer = false;
-          // Bind event handlers
-          this._handleWebSocketMessage = this._handleWebSocketMessage.bind(this);
-          this._handleDataAvailable = this._handleDataAvailable.bind(this);
       }
       /**
        * Updates the connection status and triggers the callback
@@ -3613,10 +3630,9 @@ registerProcessor('audio_processor', AudioProcessor);
                       // Sent from the server to this client when a new user turn is detected
                       console.log('received turn.start from server');
                       console.log(message);
-                      // if (message.role === 'user' && !this.pushToTalkActive) {
-                      if (message.role === 'user') {
+                      if (message.role === 'user' && !this.pushToTalkEnabled && this.canInterrupt) {
                           // Interrupt any playing assistant audio if this is a turn trigged by the server (and not push to talk, which will have already called interrupt)
-                          console.log('interrupting assistant audio, as user turn has started and pushToTalkActive is false');
+                          console.log('interrupting assistant audio, as user turn has started and pushToTalkEnabled is false');
                           await this._clientInterruptAssistantReplay();
                       }
                       // if (message.role === 'assistant') {
@@ -3730,6 +3746,19 @@ registerProcessor('audio_processor', AudioProcessor);
               this.ws = new WebSocket(`${this._websocketUrl}?${new URLSearchParams({
                 client_session_key: authorizeSessionResponseBody.client_session_key,
             })}`);
+              const config = authorizeSessionResponseBody.config;
+              console.log('config', config);
+              if (config.transcription.trigger === 'push_to_talk') {
+                  this.pushToTalkEnabled = true;
+              }
+              else if (config.transcription.trigger === 'automatic') {
+                  this.pushToTalkEnabled = false;
+                  this.canInterrupt = config.transcription.can_interrupt;
+              }
+              else {
+                  throw new Error(`Unknown trigger: ${config.transcription.trigger}`);
+              }
+              this._initializeVAD();
               // Bind the websocket message callbacks
               this.ws.onmessage = this._handleWebSocketMessage;
               this.ws.onopen = () => {