ai 7.0.101 → 7.0.102

package/src/realtime/browser-realtime-audio.ts

@@ -10,6 +10,8 @@ export type BrowserRealtimeAudioOptions = {
   onAudio: (base64Audio: string) => void;
   onPlayingChange: (isPlaying: boolean) => void;
   onCapturingChange: (isCapturing: boolean) => void;
+  maxPlaybackSeconds?: number;
+  onError?: (error: Error) => void;
 };
 
 export class BrowserRealtimeAudio {
@@ -23,6 +25,9 @@ export class BrowserRealtimeAudio {
   private captureProcessor: ScriptProcessorNode | null = null;
   private captureSource: MediaStreamAudioSourceNode | null = null;
   private captureStream: MediaStream | null = null;
+  private ownsCaptureStream = true;
+  private captureCleanup: Array<() => void> = [];
+  private captureGeneration = 0;
 
   private playbackContext: AudioContext | null = null;
   private playbackQueue: Float32Array[] = [];
@@ -30,12 +35,17 @@ export class BrowserRealtimeAudio {
   private playbackStartTime = 0;
   private activeSources = new Set<AudioBufferSourceNode>();
   private isPlaying = false;
+  private playbackPaused = false;
+  private readonly maxPlaybackSeconds: number;
+  private readonly onError: ((error: Error) => void) | undefined;
 
   constructor(options: BrowserRealtimeAudioOptions) {
     this.captureSampleRate = options.captureSampleRate;
     this.playbackSampleRate = options.playbackSampleRate;
     this.onAudio = options.onAudio;
     this.onPlayingChange = options.onPlayingChange;
+    this.maxPlaybackSeconds = options.maxPlaybackSeconds ?? Infinity;
+    this.onError = options.onError;
     this.onCapturingChange = options.onCapturingChange;
   }
 
@@ -44,40 +54,101 @@ export class BrowserRealtimeAudio {
       this.playbackContext = new AudioContext({
         sampleRate: this.playbackSampleRate,
       });
+      const context = this.playbackContext;
+      context.onstatechange = () => {
+        if (this.playbackContext === context)
+          this.setPlaying(
+            context.state === 'running' && this.activeSources.size > 0,
+          );
+      };
     }
   }
 
-  startCapture(stream: MediaStream): void {
+  async resumePlayback(): Promise<void> {
+    this.ensurePlaybackContext();
+    const context = this.playbackContext;
+    if (this.playbackPaused) this.stopPlayback();
+    if (this.playbackContext !== context) return;
+    await context?.resume();
+    if (this.playbackContext !== context) return;
+    this.playbackPaused = false;
+    this.setPlaying(
+      this.playbackContext?.state === 'running' && this.activeSources.size > 0,
+    );
+  }
+
+  startCapture(stream: MediaStream, options?: { ownsStream?: boolean }): void {
+    const generation = this.captureGeneration + 1;
+    this.stopCapture();
+    if (generation !== this.captureGeneration) {
+      if (options?.ownsStream ?? true)
+        stream.getTracks().forEach(track => track.stop());
+      return;
+    }
+    this.captureStream = stream;
+    this.ownsCaptureStream = options?.ownsStream ?? true;
     const ctx = new AudioContext({ sampleRate: this.captureSampleRate });
     this.captureContext = ctx;
-    this.captureStream = stream;
+    void ctx.resume().catch(error => {
+      if (this.captureContext === ctx) this.onError?.(error);
+    });
 
     const source = ctx.createMediaStreamSource(stream);
     this.captureSource = source;
 
-    const processor = ctx.createScriptProcessor(4096, 1, 1);
+    const processor = ctx.createScriptProcessor(1024, 1, 1);
     this.captureProcessor = processor;
 
     processor.onaudioprocess = event => {
+      if (this.captureContext !== ctx) return;
       const inputData = event.inputBuffer.getChannelData(0);
       const samples = resampleAudio(
         new Float32Array(inputData),
         ctx.sampleRate,
         this.captureSampleRate,
       );
-      this.onAudio(encodeRealtimeAudio(samples));
+      try {
+        this.onAudio(encodeRealtimeAudio(samples));
+      } catch (error) {
+        this.onError?.(
+          error instanceof Error ? error : new Error(String(error)),
+        );
+      }
     };
 
     source.connect(processor);
     processor.connect(ctx.destination);
-    this.onCapturingChange(true);
+    const updateCapturing = () =>
+      this.onCapturingChange(
+        stream
+          .getAudioTracks()
+          .some(
+            track =>
+              track.enabled && !track.muted && track.readyState === 'live',
+          ),
+      );
+    for (const track of stream.getAudioTracks()) {
+      for (const event of ['ended', 'mute', 'unmute']) {
+        track.addEventListener(event, updateCapturing);
+        this.captureCleanup.push(() =>
+          track.removeEventListener(event, updateCapturing),
+        );
+      }
+    }
+    updateCapturing();
   }
 
   stopCapture(): void {
+    this.captureGeneration++;
+    if (this.captureProcessor != null)
+      this.captureProcessor.onaudioprocess = null;
+    for (const cleanup of this.captureCleanup) cleanup();
+    this.captureCleanup = [];
     this.captureProcessor?.disconnect();
     this.captureSource?.disconnect();
-    void this.captureContext?.close();
-    this.captureStream?.getTracks().forEach(track => track.stop());
+    void this.captureContext?.close().catch(() => {});
+    if (this.ownsCaptureStream)
+      this.captureStream?.getTracks().forEach(track => track.stop());
 
     this.captureProcessor = null;
     this.captureSource = null;
@@ -87,8 +158,27 @@ export class BrowserRealtimeAudio {
   }
 
   playAudio(base64Audio: string): void {
+    if (this.playbackPaused) return;
     this.ensurePlaybackContext();
-    this.playbackQueue.push(decodeRealtimeAudio(base64Audio));
+    const samples = decodeRealtimeAudio(base64Audio);
+    if (
+      Math.max(
+        0,
+        this.playbackTime - (this.playbackContext?.currentTime ?? 0),
+      ) +
+        samples.length / this.playbackSampleRate >
+      this.maxPlaybackSeconds
+    ) {
+      this.playbackPaused = true;
+      this.stopPlayback();
+      this.onError?.(
+        new Error(
+          'Realtime audio playback buffer is full; playback paused, call resumePlayback() to resume at the live edge',
+        ),
+      );
+      return;
+    }
+    this.playbackQueue.push(samples);
     this.schedulePlayback();
   }
 
@@ -97,7 +187,9 @@ export class BrowserRealtimeAudio {
 
     for (const source of this.activeSources) {
       try {
+        source.onended = null;
         source.stop();
+        source.disconnect();
       } catch {
         // Source may already be stopped by the browser.
       }
@@ -119,8 +211,11 @@ export class BrowserRealtimeAudio {
   dispose(): void {
     this.stopCapture();
     this.stopPlayback();
-    void this.playbackContext?.close();
+    if (this.playbackContext != null) this.playbackContext.onstatechange = null;
+    void this.playbackContext?.close().catch(() => {});
     this.playbackContext = null;
+    this.playbackTime = 0;
+    this.playbackPaused = false;
   }
 
   private setPlaying(isPlaying: boolean): void {
@@ -155,9 +250,11 @@ export class BrowserRealtimeAudio {
       this.playbackTime = startTime + buffer.duration;
 
       this.activeSources.add(source);
-      this.setPlaying(true);
+      this.setPlaying(ctx.state === 'running');
+      if (this.playbackContext !== ctx) return;
 
       source.onended = () => {
+        source.disconnect();
         this.activeSources.delete(source);
         if (this.playbackQueue.length === 0 && this.activeSources.size === 0) {
           this.setPlaying(false);

package/src/realtime/browser-realtime-live-websocket.ts

@@ -0,0 +1,247 @@
+import { UnsupportedFunctionalityError } from '@ai-sdk/provider';
+import type {
+  RealtimeClientEvent,
+  RealtimeModel,
+  RealtimeServerEvent,
+  RealtimeSessionConfig,
+} from '../types/realtime-model';
+import { BrowserRealtimeAudio } from './browser-realtime-audio';
+import { BrowserRealtimeTransport } from './browser-realtime-transport';
+
+/** Continuous PCM16 audio-chunk media for an application-owned WebSocket relay. */
+export class BrowserRealtimeLiveWebSocket {
+  private readonly transport: BrowserRealtimeTransport;
+  private readonly audio: BrowserRealtimeAudio;
+  private readonly config: RealtimeSessionConfig;
+  private generation = 0;
+  private ready = false;
+  private capturingStarted = false;
+  private stream?: MediaStream;
+  private pendingAudio = 0;
+  private captureGeneration = 0;
+  private captureEnabled = true;
+
+  constructor(
+    private readonly options: {
+      model: RealtimeModel;
+      sessionConfig?: Partial<RealtimeSessionConfig>;
+      sampleRate?: number;
+      maxPlaybackBufferSeconds?: number;
+      onEvent: (event: RealtimeServerEvent) => Promise<void>;
+      onError: (error: Error) => void;
+      onFatalError: (error: Error, drain?: Promise<void>) => void;
+      onClosing?: () => void;
+      onClose: (error?: Error) => void;
+      onCapturing: (value: boolean) => void;
+      onPlaying: (value: boolean) => void;
+    },
+  ) {
+    const input = options.sessionConfig?.inputAudioFormat;
+    const output = options.sessionConfig?.outputAudioFormat;
+    const sampleRate =
+      input?.rate ?? output?.rate ?? options.sampleRate ?? 24000;
+    const format = {
+      type: input?.type ?? output?.type ?? 'audio/pcm',
+      rate: sampleRate,
+    };
+    this.config = {
+      ...options.sessionConfig,
+      inputAudioFormat: {
+        ...format,
+        ...input,
+      },
+      outputAudioFormat: {
+        ...format,
+        ...output,
+      },
+    };
+    this.transport = new BrowserRealtimeTransport({
+      model: options.model,
+      onServerEvent: options.onEvent,
+      onError: options.onError,
+      onFatalError: options.onFatalError,
+      onClosing: () => {
+        const generation = this.generation;
+        options.onClosing?.();
+        if (generation === this.generation) this.stopCapture();
+      },
+      onClose: options.onClose,
+    });
+    this.audio = new BrowserRealtimeAudio({
+      captureSampleRate: this.config.inputAudioFormat?.rate ?? sampleRate,
+      playbackSampleRate: this.config.outputAudioFormat?.rate ?? sampleRate,
+      maxPlaybackSeconds: options.maxPlaybackBufferSeconds ?? 2,
+      onAudio: audio => this.sendAudio(audio),
+      onError: options.onError,
+      onCapturingChange: options.onCapturing,
+      onPlayingChange: options.onPlaying,
+    });
+  }
+
+  connect(options: {
+    url: string;
+    protocols?: string[];
+    stream?: MediaStream;
+    capture?: boolean;
+  }): void {
+    const generation = this.generation + 1;
+    this.dispose();
+    if (generation !== this.generation) return;
+    for (const format of [
+      this.config.inputAudioFormat,
+      this.config.outputAudioFormat,
+    ]) {
+      if (format?.type !== 'audio/pcm')
+        throw new UnsupportedFunctionalityError({
+          functionality:
+            'Continuous browser WebSocket audio supports PCM16 only',
+        });
+      if (
+        format.rate == null ||
+        !Number.isFinite(format.rate) ||
+        format.rate < 8000 ||
+        format.rate > 96000
+      )
+        throw new Error('Invalid realtime PCM sample rate');
+    }
+    this.stream = options.stream;
+    this.captureEnabled = options.capture !== false;
+    // Create/resume playback while connect is still in the caller's user gesture.
+    this.audio.ensurePlaybackContext();
+    void this.audio.resumePlayback().catch(error => {
+      if (generation === this.generation) this.options.onError(error);
+    });
+    this.transport.connect({
+      mode: 'relay',
+      url: options.url,
+      protocols: options.protocols,
+      onOpen: () => {
+        return this.transport.sendEvent({
+          type: this.options.model.capabilities?.startup ?? 'session-update',
+          config: this.config,
+        });
+      },
+    });
+  }
+
+  startCapture(): void {
+    this.ready = true;
+    if (!this.captureEnabled || this.capturingStarted) return;
+    void this.resumeCapture().catch(error => this.options.onError(error));
+  }
+
+  async resumeCapture(suppliedStream?: MediaStream): Promise<void> {
+    const generation = this.captureGeneration + 1;
+    this.stopCapture();
+    if (generation !== this.captureGeneration) return;
+    this.ready = true;
+    this.capturingStarted = true;
+    this.captureEnabled = true;
+    if (suppliedStream != null) this.stream = suppliedStream;
+    const supplied = this.stream;
+    const stream =
+      supplied ?? (await navigator.mediaDevices.getUserMedia({ audio: true }));
+    if (generation !== this.captureGeneration || !this.ready) {
+      if (supplied == null) stream.getTracks().forEach(track => track.stop());
+      return;
+    }
+    if (!stream.getAudioTracks().some(track => track.readyState === 'live')) {
+      if (supplied == null) stream.getTracks().forEach(track => track.stop());
+      throw new Error('Realtime requires a live audio track');
+    }
+    this.audio.startCapture(stream, { ownsStream: supplied == null });
+  }
+
+  private sendAudio(audio: string): void {
+    if (!this.ready || !this.transport.isOpen) return;
+    if (this.pendingAudio >= 8) {
+      this.transport.fail(new Error('Realtime audio send queue is full'));
+      return;
+    }
+    const generation = this.generation;
+    let sent: Promise<void>;
+    try {
+      sent = this.transport.sendEvent(
+        { type: 'input-audio-append', audio },
+        undefined,
+        true,
+      );
+    } catch (error) {
+      if (!this.ready || !this.transport.isOpen) return;
+      throw error;
+    }
+    this.pendingAudio++;
+    void sent.then(
+      () => {
+        if (generation === this.generation) this.pendingAudio--;
+      },
+      error => {
+        if (
+          generation === this.generation &&
+          this.ready &&
+          this.transport.isOpen
+        )
+          this.options.onFatalError(error);
+      },
+    );
+  }
+
+  sendEvent(
+    event: RealtimeClientEvent,
+    shouldSend?: () => boolean,
+  ): Promise<void> {
+    if (event.type === 'session-update') {
+      for (const key of ['inputAudioFormat', 'outputAudioFormat'] as const) {
+        const update = event.config[key];
+        if (
+          update != null &&
+          (update.type !== this.config[key]?.type ||
+            (update.rate != null && update.rate !== this.config[key]?.rate))
+        ) {
+          throw new Error(
+            'Changing Live WebSocket audio format requires reconnecting',
+          );
+        }
+      }
+    }
+    return this.transport.sendEvent(event, shouldSend);
+  }
+  playAudio(audio: string): void {
+    try {
+      this.audio.playAudio(audio);
+    } catch (error) {
+      this.options.onError(
+        error instanceof Error ? error : new Error(String(error)),
+      );
+    }
+  }
+  stopPlayback(): void {
+    this.audio.stopPlayback();
+  }
+  resumePlayback(): Promise<void> {
+    return this.audio.resumePlayback();
+  }
+
+  stopCapture(): void {
+    this.captureGeneration++;
+    this.ready = false;
+    this.capturingStarted = false;
+    this.captureEnabled = false;
+    this.audio.stopCapture();
+  }
+
+  finish(): Promise<void> {
+    return this.transport.finish();
+  }
+
+  dispose(): void {
+    this.generation++;
+    this.stopCapture();
+    this.ready = false;
+    this.capturingStarted = false;
+    this.pendingAudio = 0;
+    this.stream = undefined;
+    this.transport.disconnect();
+    this.audio.dispose();
+  }
+}