ai 7.0.101 → 7.0.103

package/src/realtime/realtime-command-tracker.ts

@@ -0,0 +1,81 @@
+import { generateId } from '@ai-sdk/provider-utils';
+import type {
+  RealtimeClientEvent,
+  RealtimeServerEvent,
+} from '../types/realtime-model';
+
+/** Attempt-local ACK correlation; unresolved commands are never evicted. */
+export class RealtimeCommandTracker {
+  private readonly pending = new Map<string, { muted?: boolean }>();
+  private readonly recent = new Set<string>();
+
+  constructor(
+    private readonly sendCommand: (event: RealtimeClientEvent) => Promise<void>,
+  ) {}
+
+  validateId(id: string): void {
+    if (this.pending.has(id) || this.recent.has(id))
+      throw new Error(
+        'Realtime command ID was already used; retry with a fresh eventId',
+      );
+    if (this.pending.size >= 512)
+      throw new Error('Too many pending realtime commands');
+  }
+
+  private complete(id: string): { muted?: boolean } | undefined {
+    const command = this.pending.get(id);
+    if (command == null) return;
+    this.pending.delete(id);
+    this.recent.add(id);
+    if (this.recent.size > 4096) {
+      const oldest = this.recent.values().next().value;
+      if (oldest != null) this.recent.delete(oldest);
+    }
+    return command;
+  }
+
+  send(event: RealtimeClientEvent): Promise<void> {
+    const muted =
+      event.type === 'input-audio-mute'
+        ? true
+        : event.type === 'input-audio-unmute'
+          ? false
+          : undefined;
+    if (
+      event.type === 'input-audio-mute' ||
+      event.type === 'input-audio-unmute'
+    )
+      event = { ...event, eventId: event.eventId ?? generateId() };
+    const id = 'eventId' in event ? event.eventId : undefined;
+    if (id != null) {
+      this.validateId(id);
+      this.pending.set(id, { muted });
+    }
+    const awaitsAcknowledgement =
+      event.type === 'context-append' ||
+      event.type === 'session-update' ||
+      muted != null;
+    try {
+      const sent = this.sendCommand(event);
+      void sent.then(
+        () => {
+          if (id != null && !awaitsAcknowledgement) this.complete(id);
+        },
+        () => {
+          if (id != null) this.complete(id);
+        },
+      );
+      return sent;
+    } catch (error) {
+      if (id != null) this.complete(id);
+      throw error;
+    }
+  }
+
+  receive(event: RealtimeServerEvent): { muted?: boolean } | undefined {
+    if (event.type === 'command-acknowledged' && event.clientEventId != null)
+      return this.complete(event.clientEventId);
+    if (event.type === 'error' && event.clientEventId != null)
+      this.complete(event.clientEventId);
+  }
+}

package/src/realtime/realtime-event-channel.ts

@@ -0,0 +1,170 @@
+import { safeParseJSON } from '@ai-sdk/provider-utils';
+import type {
+  RealtimeClientEvent,
+  RealtimeModel,
+  RealtimeServerEvent,
+} from '../types/realtime-model';
+
+/** One codec queue per connection; retired connections cannot deliver late work. */
+export class RealtimeEventChannel {
+  private active = true;
+  private finishing = false;
+  private readonly parse: (
+    raw: unknown,
+  ) => RealtimeServerEvent | RealtimeServerEvent[];
+  private incoming = Promise.resolve();
+  private outgoing = Promise.resolve();
+  private incomingCount = 0;
+  private incomingBytes = 0;
+  private outgoingCount = 0;
+
+  constructor(
+    private readonly options: {
+      model: RealtimeModel;
+      send: (data: unknown) => void;
+      onEvent: (event: RealtimeServerEvent) => void | Promise<void>;
+      onError: (error: Error) => void;
+      onFatalError?: (error: Error) => void;
+      maxPending?: number;
+      maxPendingBytes?: number;
+    },
+  ) {
+    this.parse =
+      options.model.createServerEventParser?.() ??
+      options.model.parseServerEvent.bind(options.model);
+  }
+
+  dispose(): void {
+    this.active = false;
+  }
+
+  /** Stop accepting messages, but deliver already received terminal events. */
+  async finish(): Promise<void> {
+    this.stopWriting();
+    await this.incoming;
+    this.dispose();
+  }
+
+  stopWriting(): void {
+    this.finishing = true;
+  }
+
+  send(event: RealtimeClientEvent, shouldSend?: () => boolean): Promise<void> {
+    if (!this.active || this.finishing)
+      throw new Error('Realtime connection is closed');
+    if (this.outgoingCount >= (this.options.maxPending ?? 512)) {
+      throw new Error('Realtime outgoing queue is full');
+    }
+    this.outgoingCount++;
+    const operation = this.outgoing.then(async () => {
+      if (!this.active || this.finishing)
+        throw new Error('Realtime connection is closed');
+      if (shouldSend?.() === false) return;
+      if (!this.active || this.finishing)
+        throw new Error('Realtime connection is closed');
+      const data = await this.options.model.serializeClientEvent(event);
+      if (!this.active || this.finishing)
+        throw new Error('Realtime connection is closed');
+      if (shouldSend?.() === false) return;
+      if (!this.active || this.finishing)
+        throw new Error('Realtime connection is closed');
+      if (data != null) this.options.send(data);
+    });
+    this.outgoing = operation
+      .catch(error => {
+        if (!this.finishing) {
+          // Capture owners escalate automatic audio failures; manual sends stay recoverable.
+          if (
+            event.type !== 'input-audio-append' &&
+            event.type !== 'session-close'
+          )
+            this.report(error);
+        }
+      })
+      .finally(() => {
+        this.outgoingCount--;
+      });
+    return operation;
+  }
+
+  receive(data: unknown): void {
+    if (!this.active || this.finishing) return;
+    const bytes =
+      typeof data === 'string'
+        ? data.length * 2
+        : data instanceof Blob
+          ? data.size
+          : ((data as ArrayBuffer).byteLength ?? 0);
+    if (
+      this.incomingCount >= (this.options.maxPending ?? 512) ||
+      this.incomingBytes + bytes >
+        (this.options.maxPendingBytes ?? 8 * 1024 * 1024)
+    ) {
+      this.finishing = true;
+      this.fatal(
+        new Error(
+          'Realtime incoming queue is full; input protocol continuity was lost',
+        ),
+      );
+      return;
+    }
+    this.incomingCount++;
+    this.incomingBytes += bytes;
+    this.incoming = this.incoming
+      .then(async () => {
+        if (!this.active) return;
+        const text =
+          typeof data === 'string'
+            ? data
+            : data instanceof Blob
+              ? await data.text()
+              : new TextDecoder().decode(data as ArrayBuffer);
+        const parsed = await safeParseJSON({ text });
+        if (!this.active) return;
+        if (!parsed.success)
+          throw new Error('Invalid JSON in realtime server message', {
+            cause: parsed.error,
+          });
+        const health = this.options.model.getHealthCheckResponse?.(
+          parsed.value,
+        );
+        if (!this.active) return;
+        if (health != null && !this.finishing) this.options.send(health);
+        if (!this.active) return;
+        const result = this.parse(parsed.value);
+        for (const event of Array.isArray(result) ? result : [result]) {
+          if (!this.active) return;
+          try {
+            await this.options.onEvent(event);
+          } catch (error) {
+            this.report(error);
+          }
+        }
+      })
+      .catch(error => {
+        this.finishing = true;
+        this.fatal(error instanceof Error ? error : new Error(String(error)));
+      })
+      .finally(() => {
+        this.incomingCount--;
+        this.incomingBytes -= bytes;
+      });
+  }
+
+  private fatal(error: Error): void {
+    if (!this.active) return;
+    if (this.options.onFatalError != null) this.options.onFatalError(error);
+    else this.report(error);
+  }
+
+  private report(error: unknown): void {
+    if (!this.active) return;
+    try {
+      this.options.onError(
+        error instanceof Error ? error : new Error(String(error)),
+      );
+    } catch {
+      /* Application callbacks cannot break the codec queue. */
+    }
+  }
+}

package/src/realtime/realtime-event-reducer.ts

@@ -1,5 +1,6 @@
 import { safeParseJSON } from '@ai-sdk/provider-utils';
 import type { RealtimeServerEvent } from '../types/realtime-model';
+import type { RealtimeSessionState } from './realtime-session-state';
 import type {
   DynamicToolUIPart,
   TextUIPart,
@@ -10,6 +11,7 @@ export type RealtimeStatus =
   | 'disconnected'
   | 'connecting'
   | 'connected'
+  | 'closing'
   | 'error';
 
 export interface RealtimeState {
@@ -18,6 +20,7 @@ export interface RealtimeState {
   events: RealtimeServerEvent[];
   isCapturing: boolean;
   isPlaying: boolean;
+  session?: RealtimeSessionState;
 }
 
 export type RealtimeReducerEffect =

package/src/realtime/realtime-session-state.ts

@@ -0,0 +1,65 @@
+import type { RealtimeServerEvent } from '../types/realtime-model';
+
+export type RealtimeSessionState = {
+  delegationMode?: 'client' | 'provider';
+  sessionId?: string;
+  transcripts: Extract<RealtimeServerEvent, { type: 'transcript-fragment' }>[];
+  delegations: Extract<RealtimeServerEvent, { type: 'delegation-created' }>[];
+  usage?: { seconds: number };
+  finalization: 'pending' | 'confirmed' | 'unconfirmed';
+  terminationReason?: string;
+  isInputMuted: boolean;
+};
+
+export function createSessionState(): RealtimeSessionState {
+  return {
+    transcripts: [],
+    delegations: [],
+    isInputMuted: false,
+    finalization: 'pending',
+  };
+}
+
+export function reduceSessionState(
+  state: RealtimeSessionState,
+  event: RealtimeServerEvent,
+  limit: number,
+  muted?: boolean,
+): RealtimeSessionState {
+  switch (event.type) {
+    case 'session-started':
+      return {
+        ...state,
+        sessionId: event.sessionId,
+        delegationMode: event.delegationMode,
+      };
+    case 'session-usage':
+      return { ...state, usage: event.usage };
+    case 'session-closed':
+      return {
+        ...state,
+        sessionId: event.sessionId ?? state.sessionId,
+        usage: event.usage,
+        terminationReason: event.reason,
+        finalization: 'confirmed',
+      };
+    case 'transcript-fragment':
+      return {
+        ...state,
+        transcripts: [...state.transcripts, event].slice(-limit),
+      };
+    case 'delegation-created':
+      return state.delegations.some(
+        item => item.delegationId === event.delegationId,
+      )
+        ? state
+        : {
+            ...state,
+            delegations: [...state.delegations, event].slice(-limit),
+          };
+    case 'command-acknowledged':
+      return muted == null ? state : { ...state, isInputMuted: muted };
+    default:
+      return state;
+  }
+}