@eshal-bot/chat-widget 0.1.43 → 0.1.45

package/dist/chat-widget.js

@@ -8537,6 +8537,151 @@
     return pcm16Array.buffer;
   }
 
+  /**
+   * Client-side Voice-Activity gate for the widget's voice (Bidi) session.
+   *
+   * Ported from packages/agent-sdk/src/voice/audio/vad.ts (`createSpeechGate`),
+   * which was itself originally derived from this widget. The gate sits between
+   * the mic worklet and the WebSocket: it forwards PCM ONLY inside a confirmed
+   * speech turn and fires `onSpeechEnd` when the turn closes, so the caller can
+   * tell the backend the utterance is done (an `audio_end` signal).
+   *
+   * Why the widget needs this: knowledge-api's non-streaming voice path only
+   * transcribes a turn when it receives `audio_end` (or its flaky energy-VAD
+   * backstop trips). Without a client gate the widget streamed mic audio forever
+   * and the backend just buffered it ("Collecting Audio…") without ever
+   * transcribing — so voice connected but never replied. The agent-sdk already
+   * gates this way; this brings the widget to parity.
+   *
+   * Framework-free and side-effect-free (only calls the two callbacks) so it can
+   * be reasoned about / tested in isolation.
+   */
+
+  const DEFAULT_SPEECH_GATE_CONFIG = {
+    // RMS energy (0..1) at/above which a frame counts as voiced. Speech is
+    // typically 0.05–0.3; quiet room tone sits well under 0.02.
+    energyThreshold: 0.025,
+    // Continuous voiced audio (ms) required to OPEN a turn — rejects coughs/clicks.
+    speechStartMs: 200,
+    // Continuous silence (ms) after speech that CLOSES the turn. Long enough to
+    // ride over natural mid-sentence pauses, short enough to feel responsive.
+    hangoverMs: 1000,
+    // How much audio (ms) preceding turn-open to retain + flush so the leading
+    // phoneme of the first word survives the start-detection delay.
+    preRollMs: 300
+  };
+
+  /** RMS amplitude of an Int16 PCM frame (ArrayBuffer), normalised to 0..1. */
+  function computeRms(pcm) {
+    const samples = new Int16Array(pcm);
+    if (samples.length === 0) return 0;
+    let sumSquares = 0;
+    for (let i = 0; i < samples.length; i += 1) {
+      const v = samples[i] / 0x8000; // normalise Int16 → -1..1
+      sumSquares += v * v;
+    }
+    return Math.sqrt(sumSquares / samples.length);
+  }
+
+  /**
+   * Create a stateful speech gate. Returns `{ process, reset, isSpeaking }`.
+   * `config.sampleRate` is required; everything else falls back to
+   * DEFAULT_SPEECH_GATE_CONFIG.
+   *
+   * @param {{ sampleRate: number, energyThreshold?: number, speechStartMs?: number, hangoverMs?: number, preRollMs?: number }} config
+   * @param {{ onSpeechStart?: () => void, onVoiceFrame: (pcm: ArrayBuffer) => void, onSpeechEnd: () => void }} callbacks
+   *   onSpeechStart fires the instant a turn opens — use it for barge-in
+   *   (stop any assistant TTS still playing). onVoiceFrame emits each voiced
+   *   PCM frame; onSpeechEnd fires when the turn closes after the hangover.
+   */
+  function createSpeechGate(config, callbacks) {
+    var _config$energyThresho, _config$speechStartMs, _config$hangoverMs, _config$preRollMs;
+    const cfg = {
+      sampleRate: config.sampleRate,
+      energyThreshold: (_config$energyThresho = config.energyThreshold) !== null && _config$energyThresho !== void 0 ? _config$energyThresho : DEFAULT_SPEECH_GATE_CONFIG.energyThreshold,
+      speechStartMs: (_config$speechStartMs = config.speechStartMs) !== null && _config$speechStartMs !== void 0 ? _config$speechStartMs : DEFAULT_SPEECH_GATE_CONFIG.speechStartMs,
+      hangoverMs: (_config$hangoverMs = config.hangoverMs) !== null && _config$hangoverMs !== void 0 ? _config$hangoverMs : DEFAULT_SPEECH_GATE_CONFIG.hangoverMs,
+      preRollMs: (_config$preRollMs = config.preRollMs) !== null && _config$preRollMs !== void 0 ? _config$preRollMs : DEFAULT_SPEECH_GATE_CONFIG.preRollMs
+    };
+    let speaking = false;
+    // While idle: how much consecutive voiced audio we've seen (to open).
+    let voicedRunMs = 0;
+    // While speaking: how much consecutive silence we've seen (to close).
+    let silenceRunMs = 0;
+    // Ring of recent frames kept only while idle, for pre-roll on open.
+    let preRoll = [];
+    let preRollMsBuffered = 0;
+
+    // Int16 → 2 bytes/sample; ms = samples / sampleRate * 1000.
+    const frameDurationMs = pcm => pcm.byteLength / 2 / cfg.sampleRate * 1000;
+    const reset = () => {
+      speaking = false;
+      voicedRunMs = 0;
+      silenceRunMs = 0;
+      preRoll = [];
+      preRollMsBuffered = 0;
+    };
+    const process = pcm => {
+      const durationMs = frameDurationMs(pcm);
+      const voiced = computeRms(pcm) >= cfg.energyThreshold;
+      if (!speaking) {
+        // Keep a bounded pre-roll of the most recent audio so we don't clip the
+        // onset once we decide this is speech.
+        preRoll.push({
+          pcm,
+          durationMs
+        });
+        preRollMsBuffered += durationMs;
+        while (preRollMsBuffered > cfg.preRollMs && preRoll.length > 1) {
+          var _dropped$durationMs;
+          const dropped = preRoll.shift();
+          preRollMsBuffered -= (_dropped$durationMs = dropped === null || dropped === void 0 ? void 0 : dropped.durationMs) !== null && _dropped$durationMs !== void 0 ? _dropped$durationMs : 0;
+        }
+        if (voiced) {
+          voicedRunMs += durationMs;
+          if (voicedRunMs >= cfg.speechStartMs) {
+            var _callbacks$onSpeechSt;
+            // Confirmed speech — open the turn. Fire onSpeechStart FIRST so the
+            // caller can barge in (stop assistant TTS still playing) before we
+            // start streaming this utterance, then flush the pre-roll (which
+            // contains the voiced onset) and go live.
+            speaking = true;
+            silenceRunMs = 0;
+            (_callbacks$onSpeechSt = callbacks.onSpeechStart) === null || _callbacks$onSpeechSt === void 0 || _callbacks$onSpeechSt.call(callbacks);
+            for (const frame of preRoll) callbacks.onVoiceFrame(frame.pcm);
+            preRoll = [];
+            preRollMsBuffered = 0;
+          }
+        } else {
+          // Need *sustained* energy — reset on any silent frame so a brief
+          // transient can't accumulate its way over the threshold.
+          voicedRunMs = 0;
+        }
+        return;
+      }
+
+      // Inside a confirmed turn: forward everything, including short internal
+      // pauses, until the hangover elapses.
+      callbacks.onVoiceFrame(pcm);
+      if (voiced) {
+        silenceRunMs = 0;
+      } else {
+        silenceRunMs += durationMs;
+        if (silenceRunMs >= cfg.hangoverMs) {
+          speaking = false;
+          voicedRunMs = 0;
+          silenceRunMs = 0;
+          callbacks.onSpeechEnd();
+        }
+      }
+    };
+    return {
+      process,
+      reset,
+      isSpeaking: () => speaking
+    };
+  }
+
   /**
    * Clean CJK (Chinese, Japanese, Korean) spaces
    */
@@ -9059,6 +9204,11 @@
     const audioRecorderNodeRef = reactExports.useRef(null);
     const audioRecorderContextRef = reactExports.useRef(null);
     const micStreamRef = reactExports.useRef(null);
+    // Client-side VAD gate. Forwards only voiced PCM upstream and fires
+    // `audio_end` on end-of-speech so the backend's non-streaming voice path
+    // transcribes the turn (mirrors the agent-sdk). Without it the backend
+    // buffers audio forever and never responds.
+    const speechGateRef = reactExports.useRef(null);
     const currentInputMessageIdRef = reactExports.useRef(null);
     const currentAssistantMessageIdRef = reactExports.useRef(null);
     const isAudioReadyRef = reactExports.useRef(false);
@@ -10193,7 +10343,7 @@
       }));
     }, []);
     const stopVoiceSession = reactExports.useCallback(() => {
-      var _audioPlayerNodeRef$c, _audioPlayerContextRe, _audioRecorderNodeRef, _audioRecorderContext;
+      var _audioPlayerNodeRef$c, _audioPlayerContextRe, _audioRecorderNodeRef, _audioRecorderContext, _speechGateRef$curren;
       setIsVoiceSessionActive(false);
       setIsLoading(false);
       isVoiceSessionActiveRef.current = false;
@@ -10217,6 +10367,8 @@
       audioRecorderContextRef.current = null;
       stopMicrophone(micStreamRef.current);
       micStreamRef.current = null;
+      (_speechGateRef$curren = speechGateRef.current) === null || _speechGateRef$curren === void 0 || _speechGateRef$curren.reset();
+      speechGateRef.current = null;
       isAudioReadyRef.current = false;
     }, []);
     const handleBidiEvent = reactExports.useCallback(event => {
@@ -10396,11 +10548,22 @@
       }
     }, [appendAssistantContent, appendUserTranscription, handleContentParts, handleInterrupted, handleTurnComplete, stopVoiceSession]);
     const handleRecorderAudio = reactExports.useCallback(pcmData => {
-      const ws = websocketRef.current;
-      if (!isVoiceSessionActiveRef.current || !ws || ws.readyState !== WebSocket.OPEN) {
+      if (!isVoiceSessionActiveRef.current) {
         return;
       }
-      ws.send(pcmData);
+      // Route mic frames through the VAD gate: it forwards only voiced frames
+      // (onVoiceFrame → ws.send) and fires onSpeechEnd → audio_end so the
+      // backend finalizes the turn. Fall back to raw streaming only if the
+      // gate somehow wasn't built.
+      const gate = speechGateRef.current;
+      if (gate) {
+        gate.process(pcmData);
+        return;
+      }
+      const ws = websocketRef.current;
+      if (ws && ws.readyState === WebSocket.OPEN) {
+        ws.send(pcmData);
+      }
     }, []);
     const initializeAudioPipeline = reactExports.useCallback(async () => {
       if (isAudioReadyRef.current || typeof window === "undefined") {
@@ -10411,6 +10574,49 @@
         audioPlayerNodeRef.current = playerNode;
         audioPlayerContextRef.current = playerContext;
         await playerContext.resume();
+
+        // Build the VAD gate before the recorder starts producing frames.
+        // onVoiceFrame streams voiced PCM upstream; onSpeechEnd tells the backend
+        // the utterance is complete so it transcribes + responds. The recorder
+        // runs at 16 kHz (see audio-recorder.js).
+        const sendVoicePcm = pcm => {
+          const ws = websocketRef.current;
+          if (ws && ws.readyState === WebSocket.OPEN) {
+            ws.send(pcm);
+          }
+        };
+        speechGateRef.current = createSpeechGate({
+          sampleRate: 16000
+        }, {
+          // Barge-in: the instant the user starts talking, stop any assistant
+          // TTS still playing so we don't talk over them. endOfAudio drops the
+          // player's buffered audio immediately (readIndex → writeIndex). No-op
+          // when nothing is playing (e.g. the user's first utterance).
+          onSpeechStart: () => {
+            var _audioPlayerNodeRef$c2;
+            // Barge-in: user started talking → (1) stop assistant TTS already
+            // buffered locally, and (2) tell the backend to cancel the in-flight
+            // reply so its remaining TTS doesn't clobber the next turn's audio.
+            (_audioPlayerNodeRef$c2 = audioPlayerNodeRef.current) === null || _audioPlayerNodeRef$c2 === void 0 || _audioPlayerNodeRef$c2.port.postMessage({
+              command: "endOfAudio"
+            });
+            const ws = websocketRef.current;
+            if (ws && ws.readyState === WebSocket.OPEN) {
+              ws.send(JSON.stringify({
+                type: "interrupt"
+              }));
+            }
+          },
+          onVoiceFrame: sendVoicePcm,
+          onSpeechEnd: () => {
+            const ws = websocketRef.current;
+            if (ws && ws.readyState === WebSocket.OPEN) {
+              ws.send(JSON.stringify({
+                type: "audio_end"
+              }));
+            }
+          }
+        });
         const [recorderNode, recorderContext, micStream] = await startAudioRecorderWorklet(handleRecorderAudio);
         audioRecorderNodeRef.current = recorderNode;
         audioRecorderContextRef.current = recorderContext;
@@ -10573,11 +10779,11 @@
     }, [isVoiceSessionActive]);
     reactExports.useEffect(() => {
       return () => {
-        var _audioPlayerNodeRef$c2, _audioPlayerContextRe2, _audioRecorderNodeRef2, _audioRecorderContext2;
+        var _audioPlayerNodeRef$c3, _audioPlayerContextRe2, _audioRecorderNodeRef2, _audioRecorderContext2;
         if (websocketRef.current && websocketRef.current.readyState === WebSocket.OPEN) {
           websocketRef.current.close();
         }
-        (_audioPlayerNodeRef$c2 = audioPlayerNodeRef.current) === null || _audioPlayerNodeRef$c2 === void 0 || _audioPlayerNodeRef$c2.disconnect();
+        (_audioPlayerNodeRef$c3 = audioPlayerNodeRef.current) === null || _audioPlayerNodeRef$c3 === void 0 || _audioPlayerNodeRef$c3.disconnect();
         (_audioPlayerContextRe2 = audioPlayerContextRef.current) === null || _audioPlayerContextRe2 === void 0 || _audioPlayerContextRe2.close();
         (_audioRecorderNodeRef2 = audioRecorderNodeRef.current) === null || _audioRecorderNodeRef2 === void 0 || _audioRecorderNodeRef2.disconnect();
         (_audioRecorderContext2 = audioRecorderContextRef.current) === null || _audioRecorderContext2 === void 0 || _audioRecorderContext2.close();