@runtypelabs/persona 1.47.0 → 2.0.0

package/src/voice/audio-playback-manager.ts

@@ -0,0 +1,187 @@
+/**
+ * AudioPlaybackManager
+ *
+ * Manages streaming playback of PCM audio chunks via the Web Audio API.
+ * Receives raw PCM data (24 kHz, 16-bit signed little-endian, mono),
+ * converts to Float32 AudioBuffers, and schedules gap-free sequential
+ * playback using AudioBufferSourceNode.
+ *
+ * Works on all browsers including iOS Safari (no MediaSource dependency).
+ */
+export class AudioPlaybackManager {
+  private ctx: AudioContext | null = null;
+  private nextStartTime = 0;
+  private activeSources: AudioBufferSourceNode[] = [];
+  private finishedCallbacks: (() => void)[] = [];
+  private playing = false;
+  private streamEnded = false;
+  private pendingCount = 0;
+
+  // PCM format constants
+  private readonly sampleRate: number;
+
+  // Remainder byte from a previous chunk when the chunk had an odd byte count.
+  // Network chunks don't respect 2-byte sample boundaries, so we carry over
+  // the orphaned byte and prepend it to the next chunk.
+  private remainder: Uint8Array | null = null;
+
+  constructor(sampleRate = 24000) {
+    this.sampleRate = sampleRate;
+  }
+
+  /**
+   * Ensure AudioContext is created and running.
+   * Must be called after a user gesture on iOS Safari.
+   */
+  private ensureContext(): AudioContext {
+    if (!this.ctx) {
+      const w = typeof window !== "undefined" ? (window as any) : undefined;
+      if (!w) throw new Error("AudioPlaybackManager requires a browser environment");
+      const AudioCtx = w.AudioContext || w.webkitAudioContext;
+      this.ctx = new AudioCtx({ sampleRate: this.sampleRate }) as AudioContext;
+    }
+    const ctx = this.ctx!;
+    // Resume if suspended (autoplay policy)
+    if (ctx.state === "suspended") {
+      ctx.resume();
+    }
+    return ctx;
+  }
+
+  /**
+   * Enqueue a PCM chunk for playback.
+   * @param pcmData Raw PCM bytes (16-bit signed LE mono)
+   */
+  enqueue(pcmData: Uint8Array): void {
+    if (pcmData.length === 0) return;
+
+    // Prepend any remainder byte from the previous chunk
+    let data = pcmData;
+    if (this.remainder) {
+      const merged = new Uint8Array(this.remainder.length + pcmData.length);
+      merged.set(this.remainder);
+      merged.set(pcmData, this.remainder.length);
+      data = merged;
+      this.remainder = null;
+    }
+
+    // If odd byte count, save the trailing byte for next chunk
+    if (data.length % 2 !== 0) {
+      this.remainder = new Uint8Array([data[data.length - 1]]);
+      data = data.subarray(0, data.length - 1);
+    }
+
+    if (data.length === 0) return;
+
+    const ctx = this.ensureContext();
+    const float32 = this.pcmToFloat32(data);
+
+    const buffer = ctx.createBuffer(1, float32.length, this.sampleRate);
+    buffer.getChannelData(0).set(float32);
+
+    const source = ctx.createBufferSource();
+    source.buffer = buffer;
+    source.connect(ctx.destination);
+
+    // Schedule gap-free playback
+    const now = ctx.currentTime;
+    if (this.nextStartTime < now) {
+      this.nextStartTime = now;
+    }
+    source.start(this.nextStartTime);
+    this.nextStartTime += buffer.duration;
+
+    this.activeSources.push(source);
+    this.pendingCount++;
+    this.playing = true;
+
+    source.onended = () => {
+      const idx = this.activeSources.indexOf(source);
+      if (idx !== -1) this.activeSources.splice(idx, 1);
+      this.pendingCount--;
+      this.checkFinished();
+    };
+  }
+
+  /**
+   * Signal that no more chunks will arrive.
+   * The onFinished callback fires after all queued audio has played.
+   */
+  markStreamEnd(): void {
+    this.streamEnded = true;
+    this.checkFinished();
+  }
+
+  /**
+   * Immediately stop all playback and discard queued audio.
+   */
+  flush(): void {
+    for (const source of this.activeSources) {
+      try {
+        source.stop();
+        source.disconnect();
+      } catch {
+        // Ignore errors from already-stopped sources
+      }
+    }
+    this.activeSources = [];
+    this.pendingCount = 0;
+    this.nextStartTime = 0;
+    this.playing = false;
+    this.streamEnded = false;
+    this.finishedCallbacks = [];
+    this.remainder = null;
+  }
+
+  /**
+   * Whether audio is currently playing or queued.
+   */
+  isPlaying(): boolean {
+    return this.playing;
+  }
+
+  /**
+   * Register a callback for when all queued audio finishes playing.
+   */
+  onFinished(callback: () => void): void {
+    this.finishedCallbacks.push(callback);
+  }
+
+  /**
+   * Clean up AudioContext resources.
+   */
+  async destroy(): Promise<void> {
+    this.flush();
+    if (this.ctx) {
+      await this.ctx.close();
+      this.ctx = null;
+    }
+  }
+
+  private checkFinished(): void {
+    if (this.streamEnded && this.pendingCount <= 0 && this.playing) {
+      this.playing = false;
+      this.streamEnded = false;
+      const cbs = this.finishedCallbacks.slice();
+      this.finishedCallbacks = [];
+      for (const cb of cbs) cb();
+    }
+  }
+
+  /**
+   * Convert 16-bit signed LE PCM to Float32 samples in [-1, 1].
+   */
+  private pcmToFloat32(pcmData: Uint8Array): Float32Array {
+    // 2 bytes per sample (16-bit)
+    const numSamples = Math.floor(pcmData.length / 2);
+    const float32 = new Float32Array(numSamples);
+    const view = new DataView(pcmData.buffer, pcmData.byteOffset, pcmData.byteLength);
+
+    for (let i = 0; i < numSamples; i++) {
+      const int16 = view.getInt16(i * 2, true); // little-endian
+      float32[i] = int16 / 32768;
+    }
+
+    return float32;
+  }
+}

package/src/voice/runtype-voice-provider.ts

@@ -7,6 +7,8 @@ import type {
   VoiceStatus,
   VoiceConfig,
 } from "../types";
+import { AudioPlaybackManager } from "./audio-playback-manager";
+import { VoiceActivityDetector } from "./voice-activity-detector";
 
 export class RuntypeVoiceProvider implements VoiceProvider {
   type: "runtype" = "runtype";
@@ -20,15 +22,46 @@ export class RuntypeVoiceProvider implements VoiceProvider {
   private processingStartCallbacks: (() => void)[] = [];
   private audioChunks: Blob[] = [];
   private isProcessing = false;
+  private isSpeaking = false;
 
-  // Silence detection
-  private analyserNode: AnalyserNode | null = null;
+  // Voice activity detection (silence auto-stop + barge-in speech detection)
+  private vad = new VoiceActivityDetector();
   private mediaStream: MediaStream | null = null;
-  private silenceCheckInterval: ReturnType<typeof setInterval> | null = null;
-  private silenceStart: number | null = null;
+
+  // Cancellation / interruption support
+  private currentAudio: HTMLAudioElement | null = null;
+  private currentAudioUrl: string | null = null;
+  private currentRequestId: string | null = null;
+  private interruptionMode: "none" | "cancel" | "barge-in" = "none";
+
+  // Streaming audio playback (PCM chunks)
+  private playbackManager: AudioPlaybackManager | null = null;
 
   constructor(private config: VoiceConfig["runtype"]) {}
 
+  /** Returns the current interruption mode received from the server */
+  getInterruptionMode(): "none" | "cancel" | "barge-in" {
+    return this.interruptionMode;
+  }
+
+  /** Returns true if the barge-in mic stream is alive (hot mic between turns) */
+  isBargeInActive(): boolean {
+    return this.interruptionMode === "barge-in" && this.mediaStream !== null;
+  }
+
+  /** Tear down the barge-in mic pipeline — "hang up" the always-on mic */
+  async deactivateBargeIn(): Promise<void> {
+    this.vad.stop();
+    if (this.mediaStream) {
+      this.mediaStream.getTracks().forEach((track) => track.stop());
+      this.mediaStream = null;
+    }
+    if (this.audioContext) {
+      await this.audioContext.close();
+      this.audioContext = null;
+    }
+  }
+
   async connect() {
     if (this.ws && this.ws.readyState === WebSocket.OPEN) {
       return; // Already connected
@@ -113,6 +146,11 @@ export class RuntypeVoiceProvider implements VoiceProvider {
           { once: true }
         );
       });
+
+      // Send a ping immediately so the server replies with session_config
+      // (which includes interruptionMode). This ensures the client knows
+      // about barge-in mode before the first recording starts.
+      this.sendHeartbeat();
     } catch (error) {
       this.ws = null;
       this.errorCallbacks.forEach((cb) => cb(error as Error));
@@ -137,7 +175,17 @@ export class RuntypeVoiceProvider implements VoiceProvider {
       this.statusCallbacks.forEach((cb) => cb("error"));
     };
 
+    // Receive binary frames for streaming audio (set binaryType to arraybuffer)
+    this.ws.binaryType = "arraybuffer";
+
     this.ws.onmessage = (event) => {
+      // Binary frame = raw PCM audio chunk for streaming playback
+      if (event.data instanceof ArrayBuffer) {
+        this.handleAudioChunk(new Uint8Array(event.data));
+        return;
+      }
+
+      // Text frame = JSON control message
       try {
         const message = JSON.parse(event.data);
         this.handleWebSocketMessage(message);
@@ -151,15 +199,16 @@ export class RuntypeVoiceProvider implements VoiceProvider {
 
   private handleWebSocketMessage(message: any) {
     switch (message.type) {
-      case "voice_response":
-        // Play TTS audio if present
-        if (message.response.audio?.base64) {
-          this.playAudio(message.response.audio).catch((err) =>
-            this.errorCallbacks.forEach((cb) => cb(err instanceof Error ? err : new Error(String(err)))),
-          );
+      case "session_config":
+        // Server sends voice settings on session init
+        if (message.interruptionMode) {
+          this.interruptionMode = message.interruptionMode;
         }
-        // Use agentResponseText (the agent's reply) as the text result,
-        // falling back to transcript (user's STT input) for backwards compat
+        break;
+
+      case "voice_response":
+        // Deliver text result immediately
+        this.isProcessing = false;
         this.resultCallbacks.forEach((cb) =>
           cb({
             text: message.response.agentResponseText || message.response.transcript,
@@ -169,8 +218,39 @@ export class RuntypeVoiceProvider implements VoiceProvider {
             provider: "runtype",
           }),
         );
+
+        // Batch path: play TTS audio if present in the response (backward compat)
+        if (message.response.audio?.base64) {
+          this.isSpeaking = true;
+          this.statusCallbacks.forEach((cb) => cb("speaking"));
+          this.playAudio(message.response.audio).catch((err) =>
+            this.errorCallbacks.forEach((cb) => cb(err instanceof Error ? err : new Error(String(err)))),
+          );
+        } else if (!message.response.audio?.base64) {
+          // Streaming path: text-only voice_response — audio will arrive as
+          // binary chunks followed by audio_end. Transition to speaking state
+          // once the first audio chunk arrives (see handleAudioChunk).
+          // Stay in processing state until then.
+        }
+        break;
+
+      case "audio_end":
+        // Guard: discard late audio_end from a cancelled request
+        if (message.requestId && message.requestId !== this.currentRequestId) break;
+        // All PCM chunks have been sent — signal the playback manager
+        if (this.playbackManager) {
+          this.playbackManager.markStreamEnd();
+        } else {
+          // No audio chunks arrived — go idle
+          this.isSpeaking = false;
+          this.isProcessing = false;
+          this.statusCallbacks.forEach((cb) => cb("idle"));
+        }
+        break;
+
+      case "cancelled":
+        // Server acknowledged cancellation — discard any late-arriving responses
         this.isProcessing = false;
-        this.statusCallbacks.forEach((cb) => cb("idle"));
         break;
 
       case "error":
@@ -185,54 +265,119 @@ export class RuntypeVoiceProvider implements VoiceProvider {
     }
   }
 
+  /**
+   * Handle a binary audio chunk (raw PCM 24kHz 16-bit LE) for streaming playback.
+   */
+  private handleAudioChunk(pcmData: Uint8Array): void {
+    if (pcmData.length === 0) return;
+    if (!this.currentRequestId) return; // discard late chunks after cancel
+
+    // Lazily create playback manager on first chunk
+    if (!this.playbackManager) {
+      this.playbackManager = new AudioPlaybackManager(24000);
+      this.playbackManager.onFinished(() => {
+        this.isSpeaking = false;
+        this.playbackManager = null;
+        this.vad.stop(); // stop speech monitoring — audio ended naturally
+        this.statusCallbacks.forEach((cb) => cb("idle"));
+      });
+    }
+
+    // Transition to speaking on first chunk
+    if (!this.isSpeaking) {
+      this.isSpeaking = true;
+      this.statusCallbacks.forEach((cb) => cb("speaking"));
+      this.startBargeInMonitoring().catch(() => {}); // no-op if not barge-in mode
+    }
+
+    this.playbackManager.enqueue(pcmData);
+  }
+
+  /**
+   * Stop playback / cancel in-flight request and return to idle.
+   * This is the public "stop only" action — does NOT start recording.
+   */
+  stopPlayback(): void {
+    if (!this.isProcessing && !this.isSpeaking) return;
+    this.cancelCurrentPlayback();
+    this.statusCallbacks.forEach((cb) => cb("idle"));
+  }
+
+  /**
+   * Cancel the current playback and in-flight server request.
+   * Internal helper — does NOT fire status callbacks (caller decides next state).
+   */
+  private cancelCurrentPlayback(): void {
+    // Stop batch playback (Audio element)
+    if (this.currentAudio) {
+      this.currentAudio.pause();
+      this.currentAudio.src = "";
+      this.currentAudio = null;
+    }
+    if (this.currentAudioUrl) {
+      URL.revokeObjectURL(this.currentAudioUrl);
+      this.currentAudioUrl = null;
+    }
+
+    // Stop streaming playback (AudioPlaybackManager)
+    if (this.playbackManager) {
+      this.playbackManager.flush();
+      this.playbackManager = null;
+    }
+
+    // Tell server to abort the in-flight request
+    if (this.currentRequestId && this.ws && this.ws.readyState === WebSocket.OPEN) {
+      this.ws.send(
+        JSON.stringify({
+          type: "cancel",
+          requestId: this.currentRequestId,
+        }),
+      );
+    }
+
+    this.currentRequestId = null;
+    this.isProcessing = false;
+    this.isSpeaking = false;
+  }
+
   async startListening() {
     try {
-      if (this.isProcessing) {
-        throw new Error("Already processing audio");
+      if (this.isProcessing || this.isSpeaking) {
+        // If interruption is enabled, cancel current playback and proceed
+        if (this.interruptionMode !== "none") {
+          this.cancelCurrentPlayback();
+        } else {
+          // Mode is "none" — block mic while processing or speaking
+          return;
+        }
       }
 
-      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-      this.mediaStream = stream;
+      // Reuse existing mic stream in barge-in mode (mic stays hot)
+      if (!this.mediaStream) {
+        const constraints =
+          this.interruptionMode === "barge-in"
+            ? { audio: { echoCancellation: true, noiseSuppression: true } }
+            : { audio: true };
+        this.mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
+      }
       const w = this.w!;
-      this.audioContext = new (w.AudioContext || w.webkitAudioContext)();
-
-      // Set up silence detection via AnalyserNode
-      const audioCtx = this.audioContext!;
-      const source = audioCtx.createMediaStreamSource(stream);
-      this.analyserNode = audioCtx.createAnalyser();
-      this.analyserNode.fftSize = 2048;
-      source.connect(this.analyserNode);
+      if (!this.audioContext) {
+        this.audioContext = new (w.AudioContext || w.webkitAudioContext)();
+      }
+      const audioContext = this.audioContext!;
 
+      // VAD-based silence detection — fires once when user stops talking
       const pauseDuration = this.config?.pauseDuration ?? 2000;
       const silenceThreshold = this.config?.silenceThreshold ?? 0.01;
-      this.silenceStart = null;
-
-      const dataArray = new Float32Array(this.analyserNode.fftSize);
-      this.silenceCheckInterval = setInterval(() => {
-        if (!this.analyserNode) return;
-        this.analyserNode.getFloatTimeDomainData(dataArray);
-
-        // Compute RMS volume
-        let sum = 0;
-        for (let i = 0; i < dataArray.length; i++) {
-          sum += dataArray[i] * dataArray[i];
-        }
-        const rms = Math.sqrt(sum / dataArray.length);
-
-        if (rms < silenceThreshold) {
-          if (this.silenceStart === null) {
-            this.silenceStart = Date.now();
-          } else if (Date.now() - this.silenceStart >= pauseDuration) {
-            // Silence exceeded threshold — auto-stop
-            this.stopListening();
-          }
-        } else {
-          // Sound detected — reset silence timer
-          this.silenceStart = null;
-        }
-      }, 100);
+      this.vad.start(
+        audioContext,
+        this.mediaStream,
+        "silence",
+        { threshold: silenceThreshold, duration: pauseDuration },
+        () => this.stopListening(),
+      );
 
-      this.mediaRecorder = new MediaRecorder(stream);
+      this.mediaRecorder = new MediaRecorder(this.mediaStream);
       this.audioChunks = [];
 
       this.mediaRecorder.ondataavailable = (event) => {
@@ -265,32 +410,78 @@ export class RuntypeVoiceProvider implements VoiceProvider {
   }
 
   async stopListening() {
-    // Clean up silence detection
-    if (this.silenceCheckInterval) {
-      clearInterval(this.silenceCheckInterval);
-      this.silenceCheckInterval = null;
-    }
-    this.analyserNode = null;
-    this.silenceStart = null;
+    this.vad.stop();
 
     if (this.mediaRecorder) {
+      if (this.interruptionMode !== "barge-in") {
+        this.mediaRecorder.stream.getTracks().forEach((track) => track.stop());
+      }
       this.mediaRecorder.stop();
-      this.mediaRecorder.stream.getTracks().forEach((track) => track.stop());
       this.mediaRecorder = null;
     }
 
-    // Clean up media stream reference
-    if (this.mediaStream) {
-      this.mediaStream.getTracks().forEach((track) => track.stop());
-      this.mediaStream = null;
+    // Only tear down mic pipeline in non-barge-in modes
+    if (this.interruptionMode !== "barge-in") {
+      if (this.mediaStream) {
+        this.mediaStream.getTracks().forEach((track) => track.stop());
+        this.mediaStream = null;
+      }
+      if (this.audioContext) {
+        await this.audioContext.close();
+        this.audioContext = null;
+      }
     }
 
-    if (this.audioContext) {
-      await this.audioContext.close();
-      this.audioContext = null;
+    this.statusCallbacks.forEach((cb) => cb("idle"));
+  }
+
+  /**
+   * Start VAD in speech mode during agent playback — detects when the user
+   * starts talking so we can interrupt (barge-in). No-op in other modes.
+   * Acquires mic if needed (e.g., first response where stopListening tore it down).
+   */
+  private async startBargeInMonitoring(): Promise<void> {
+    if (this.interruptionMode !== "barge-in") return;
+
+    // Acquire mic pipeline if not already available (first response scenario)
+    const w = this.w;
+    if (!this.mediaStream && w) {
+      this.mediaStream = await navigator.mediaDevices.getUserMedia({
+        audio: { echoCancellation: true, noiseSuppression: true },
+      });
+    }
+    if (!this.audioContext && w) {
+      this.audioContext = new (w.AudioContext || w.webkitAudioContext)();
     }
+    if (!this.audioContext || !this.mediaStream) return;
+
+    const audioContext = this.audioContext!;
+    const speechThreshold = this.config?.silenceThreshold ?? 0.01;
+    const speechDebounce = 200; // 200ms sustained sound = real speech, not echo blip
+
+    this.vad.start(
+      audioContext,
+      this.mediaStream,
+      "speech",
+      { threshold: speechThreshold, duration: speechDebounce },
+      () => this.handleBargeIn(),
+    );
+  }
 
-    this.statusCallbacks.forEach((cb) => cb("idle"));
+  /**
+   * Handle a barge-in event: cancel playback and immediately start recording.
+   */
+  private handleBargeIn(): void {
+    this.cancelCurrentPlayback();
+    this.startListening().catch((err) => {
+      this.errorCallbacks.forEach((cb) =>
+        cb(err instanceof Error ? err : new Error(String(err))),
+      );
+    });
+  }
+
+  private generateRequestId(): string {
+    return "vreq_" + Math.random().toString(36).substring(2, 10) + Date.now().toString(36);
   }
 
   private async sendAudio(audioBlob: Blob) {
@@ -305,6 +496,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
     try {
       const base64Audio = await this.blobToBase64(audioBlob);
       const format = this.getFormatFromMimeType(audioBlob.type);
+      const requestId = this.generateRequestId();
+      this.currentRequestId = requestId;
 
       this.ws.send(
         JSON.stringify({
@@ -313,6 +506,7 @@ export class RuntypeVoiceProvider implements VoiceProvider {
           format,
           sampleRate: 16000,
           voiceId: this.config?.voiceId,
+          requestId,
         }),
       );
     } catch (error) {
@@ -357,7 +551,20 @@ export class RuntypeVoiceProvider implements VoiceProvider {
     const blob = new Blob([bytes], { type: mimeType });
     const url = URL.createObjectURL(blob);
     const audioEl = new Audio(url);
-    audioEl.onended = () => URL.revokeObjectURL(url);
+
+    // Store references so playback can be cancelled
+    this.currentAudio = audioEl;
+    this.currentAudioUrl = url;
+
+    audioEl.onended = () => {
+      URL.revokeObjectURL(url);
+      if (this.currentAudio === audioEl) {
+        this.currentAudio = null;
+        this.currentAudioUrl = null;
+        this.isSpeaking = false;
+        this.statusCallbacks.forEach((cb) => cb("idle"));
+      }
+    };
     await audioEl.play();
   }
 
@@ -378,8 +585,37 @@ export class RuntypeVoiceProvider implements VoiceProvider {
   }
 
   async disconnect(): Promise<void> {
+    // Stop any playing audio (batch)
+    if (this.currentAudio) {
+      this.currentAudio.pause();
+      this.currentAudio.src = "";
+      this.currentAudio = null;
+    }
+    if (this.currentAudioUrl) {
+      URL.revokeObjectURL(this.currentAudioUrl);
+      this.currentAudioUrl = null;
+    }
+    // Stop streaming playback
+    if (this.playbackManager) {
+      await this.playbackManager.destroy();
+      this.playbackManager = null;
+    }
+    this.currentRequestId = null;
+    this.isSpeaking = false;
+
+    this.vad.stop();
     await this.stopListening();
 
+    // Force mic teardown (barge-in mode skips this in stopListening)
+    if (this.mediaStream) {
+      this.mediaStream.getTracks().forEach((track) => track.stop());
+      this.mediaStream = null;
+    }
+    if (this.audioContext) {
+      await this.audioContext.close();
+      this.audioContext = null;
+    }
+
     if (this.ws) {
       try {
         this.ws.close();