@ai-sdk/provider 4.0.14 → 4.0.15

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @ai-sdk/provider
 
+## 4.0.15
+
+### Patch Changes
+
+- 5c0054d: Add optional browser-direct WebRTC for experimental client-delegated Live conversations alongside the existing WebSocket path. Exchange SDP through an application endpoint with `api.session`, configure server-owned data-channel permissions, and preserve committed React session ownership. Capture follows the selected sender track, borrowed tracks remain caller-owned, and disconnect recovery and finalization stay bounded. Applications continue to handle client delegation and submit context; Live session updates and Responses delegation remain unsupported.
+
+  Serialize microphone sender changes and close the peer if detachment fails, without stopping borrowed tracks. Validate nonempty SDP setup answers with a bounded response body, and document the same-origin broker authentication contract.
+
+- 39535af: Add experimental OpenAI Live provider support through the unified `openai.experimental_realtime` factory for server WebSocket sessions with client delegation. Applications own their agents and tools, receive continuous audio/transcript events and delegation metadata, and return context through validated channels. Support immutable startup options, microphone mute controls, graceful session close, and cumulative voice usage. Responses delegation and Live session updates reject before sending.
+
+  Route known Live model IDs to Live, allow an OpenAI-specific `api` override for early-access models, and preserve legacy Realtime defaults for unknown IDs. Token minting follows the same selection rules and rejects Live before requesting unsupported credentials. Extend the realtime v4 specification with optional server WebSocket configuration, per-connection raw-event parsers, and model-wide startup/finalization capabilities. This provider layer supplies connection settings and protocol mapping for server adapters; browser lifecycle and UI integration belong to the core runtime and framework hooks.
+
+  Preserve Realtime client event IDs for session updates and audio appends, and correlate server errors with the originating client event.
+
 ## 4.0.14
 
 ### Patch Changes

package/dist/index.d.ts

@@ -6997,12 +6997,33 @@ type RealtimeModelV4FunctionCallOutput = {
 type RealtimeModelV4ClientEvent = {
     type: 'session-update';
     config: RealtimeModelV4SessionConfig;
+    eventId?: string;
+} | {
+    type: 'session-start';
+    config: RealtimeModelV4SessionConfig;
+    eventId?: string;
+} | {
+    type: 'session-close';
+    eventId?: string;
+} | {
+    type: 'input-audio-mute';
+    eventId?: string;
+} | {
+    type: 'input-audio-unmute';
+    eventId?: string;
+} | {
+    type: 'context-append';
+    content: string;
+    delegationId: string | null;
+    eventId?: string;
+    providerOptions?: SharedV4ProviderOptions;
 } | {
     type: 'input-audio-append';
     /**
      * Base64-encoded audio chunk to append to the input buffer.
      */
     audio: string;
+    eventId?: string;
 } | {
     type: 'input-audio-commit';
 } | {
@@ -7043,6 +7064,51 @@ type RealtimeModelV4ClientEvent = {
  * event data for debugging and provider-specific access.
  */
 type RealtimeModelV4ServerEvent = {
+    type: 'session-started';
+    sessionId: string;
+    delegationMode?: 'client' | 'provider';
+    raw: unknown;
+} | {
+    type: 'session-closed';
+    sessionId?: string;
+    usage: {
+        seconds: number;
+    };
+    reason: string;
+    raw: unknown;
+} | {
+    type: 'session-usage';
+    /** Cumulative duration snapshot, not an increment. */
+    usage: {
+        seconds: number;
+    };
+    contextWindowUsageRatio?: number;
+    raw: unknown;
+} | {
+    type: 'audio-chunk';
+    delta: string;
+    raw: unknown;
+} | {
+    type: 'transcript-fragment';
+    speaker: 'user' | 'assistant';
+    delta: string;
+    startMs: number;
+    endMs: number;
+    raw: unknown;
+} | {
+    type: 'delegation-created';
+    delegationId: string;
+    target?: 'client' | 'provider';
+    offsetMs?: number;
+    responseId?: string;
+    raw: unknown;
+} | {
+    type: 'command-acknowledged';
+    /** Provider-native command name that was acknowledged. */
+    command: string;
+    clientEventId?: string;
+    raw: unknown;
+} | {
     type: 'session-created';
     sessionId?: string;
     raw: unknown;
@@ -7173,6 +7239,7 @@ type RealtimeModelV4ServerEvent = {
     type: 'error';
     message: string;
     code?: string;
+    clientEventId?: string;
     raw: unknown;
 } | {
     type: 'custom';
@@ -7203,6 +7270,38 @@ type RealtimeModelV4 = {
      * Provider-specific model ID (e.g. 'gpt-4o-realtime', 'grok-3').
      */
     readonly modelId: string;
+    /** Conversation semantics and supported transports, when declared. */
+    readonly capabilities?: {
+        conversation: 'continuous' | 'turn-based';
+        transports: readonly ('websocket' | 'webrtc')[];
+        /** Omission preserves the legacy client-secret WebSocket connection. */
+        connections?: readonly ('client-secret-websocket' | 'server-websocket' | 'webrtc')[];
+        /** Omission preserves session-update startup. WebRTC setup may start the session. */
+        startup?: 'session-start' | 'session-update';
+        /** Omission preserves transport-close finalization. */
+        finalization?: 'session-close' | 'transport-close';
+    };
+    /** Server-only connection settings. Headers can contain long-lived credentials. */
+    getServerWebSocketConfig?(): {
+        url: string;
+        headers: Record<string, string>;
+    } | PromiseLike<{
+        url: string;
+        headers: Record<string, string>;
+    }>;
+    /** Provider-specific setup for the WebRTC event channel. */
+    getWebRTCConfig?(): {
+        dataChannelLabel: string;
+    };
+    /** Server-side SDP exchange. Return only the answer and session ID to the client. */
+    doCreateWebRTCSession?(options: {
+        sdp: string;
+        sessionConfig?: RealtimeModelV4SessionConfig;
+        abortSignal?: AbortSignal;
+    }): PromiseLike<{
+        sessionId: string;
+        sdp: string;
+    }>;
     /**
      * Server-side: Creates an ephemeral client secret for authenticating
      * browser-side WebSocket connections. The secret is short-lived and
@@ -7210,13 +7309,13 @@ type RealtimeModelV4 = {
      *
      * Naming: "do" prefix to prevent accidental direct usage by the user.
      */
-    doCreateClientSecret(options: RealtimeModelV4ClientSecretOptions): PromiseLike<RealtimeModelV4ClientSecretResult>;
+    doCreateClientSecret?(options: RealtimeModelV4ClientSecretOptions): PromiseLike<RealtimeModelV4ClientSecretResult>;
     /**
      * Browser-side: Returns the WebSocket URL and subprotocols to use
      * when connecting. Each provider has its own authentication mechanism
      * (e.g. OpenAI uses subprotocol headers, xAI may use query params).
      */
-    getWebSocketConfig(options: {
+    getWebSocketConfig?(options: {
         token: string;
         url: string;
     }): {
@@ -7233,6 +7332,8 @@ type RealtimeModelV4 = {
      * text, and turn-complete data in one message).
      */
     parseServerEvent(raw: unknown): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[];
+    /** Create a raw-event parser per connection and discard it on disconnect. */
+    createServerEventParser?(): (raw: unknown) => RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[];
     /**
      * Browser-side: Serializes a normalized client event into the
      * provider's native JSON format for sending over the WebSocket.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/provider",
-  "version": "4.0.14",
+  "version": "4.0.15",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,

package/src/realtime-model/v4/realtime-model-v4-client-event.ts

@@ -1,3 +1,4 @@
+import type { SharedV4ProviderOptions } from '../../shared/v4/shared-v4-provider-options';
 import type { RealtimeModelV4ConversationItem } from './realtime-model-v4-conversation-item';
 import type { RealtimeModelV4SessionConfig } from './realtime-model-v4-session-config';
 
@@ -12,6 +13,31 @@ export type RealtimeModelV4ClientEvent =
   | {
       type: 'session-update';
       config: RealtimeModelV4SessionConfig;
+      eventId?: string;
+    }
+  | {
+      type: 'session-start';
+      config: RealtimeModelV4SessionConfig;
+      eventId?: string;
+    }
+  | {
+      type: 'session-close';
+      eventId?: string;
+    }
+  | {
+      type: 'input-audio-mute';
+      eventId?: string;
+    }
+  | {
+      type: 'input-audio-unmute';
+      eventId?: string;
+    }
+  | {
+      type: 'context-append';
+      content: string;
+      delegationId: string | null;
+      eventId?: string;
+      providerOptions?: SharedV4ProviderOptions;
     }
 
   // ── Input audio buffer ─────────────────────────────────────────────
@@ -22,6 +48,7 @@ export type RealtimeModelV4ClientEvent =
        * Base64-encoded audio chunk to append to the input buffer.
        */
       audio: string;
+      eventId?: string;
     }
   | {
       type: 'input-audio-commit';

package/src/realtime-model/v4/realtime-model-v4-server-event.ts

@@ -6,8 +6,55 @@
  * event data for debugging and provider-specific access.
  */
 export type RealtimeModelV4ServerEvent =
+  | {
+      type: 'session-started';
+      sessionId: string;
+      delegationMode?: 'client' | 'provider';
+      raw: unknown;
+    }
+  | {
+      type: 'session-closed';
+      sessionId?: string;
+      usage: { seconds: number };
+      reason: string;
+      raw: unknown;
+    }
+  | {
+      type: 'session-usage';
+      /** Cumulative duration snapshot, not an increment. */
+      usage: { seconds: number };
+      contextWindowUsageRatio?: number;
+      raw: unknown;
+    }
+  | {
+      type: 'audio-chunk';
+      delta: string;
+      raw: unknown;
+    }
+  | {
+      type: 'transcript-fragment';
+      speaker: 'user' | 'assistant';
+      delta: string;
+      startMs: number;
+      endMs: number;
+      raw: unknown;
+    }
+  | {
+      type: 'delegation-created';
+      delegationId: string;
+      target?: 'client' | 'provider';
+      offsetMs?: number;
+      responseId?: string;
+      raw: unknown;
+    }
+  | {
+      type: 'command-acknowledged';
+      /** Provider-native command name that was acknowledged. */
+      command: string;
+      clientEventId?: string;
+      raw: unknown;
+    }
   // ── Session lifecycle ──────────────────────────────────────────────
-
   | {
       type: 'session-created';
       sessionId?: string;
@@ -184,6 +231,7 @@ export type RealtimeModelV4ServerEvent =
       type: 'error';
       message: string;
       code?: string;
+      clientEventId?: string;
       raw: unknown;
     }
 

package/src/realtime-model/v4/realtime-model-v4.ts

@@ -29,6 +29,37 @@ export type RealtimeModelV4 = {
    */
   readonly modelId: string;
 
+  /** Conversation semantics and supported transports, when declared. */
+  readonly capabilities?: {
+    conversation: 'continuous' | 'turn-based';
+    transports: readonly ('websocket' | 'webrtc')[];
+    /** Omission preserves the legacy client-secret WebSocket connection. */
+    connections?: readonly (
+      | 'client-secret-websocket'
+      | 'server-websocket'
+      | 'webrtc'
+    )[];
+    /** Omission preserves session-update startup. WebRTC setup may start the session. */
+    startup?: 'session-start' | 'session-update';
+    /** Omission preserves transport-close finalization. */
+    finalization?: 'session-close' | 'transport-close';
+  };
+
+  /** Server-only connection settings. Headers can contain long-lived credentials. */
+  getServerWebSocketConfig?():
+    | { url: string; headers: Record<string, string> }
+    | PromiseLike<{ url: string; headers: Record<string, string> }>;
+
+  /** Provider-specific setup for the WebRTC event channel. */
+  getWebRTCConfig?(): { dataChannelLabel: string };
+
+  /** Server-side SDP exchange. Return only the answer and session ID to the client. */
+  doCreateWebRTCSession?(options: {
+    sdp: string;
+    sessionConfig?: RealtimeModelV4SessionConfig;
+    abortSignal?: AbortSignal;
+  }): PromiseLike<{ sessionId: string; sdp: string }>;
+
   /**
    * Server-side: Creates an ephemeral client secret for authenticating
    * browser-side WebSocket connections. The secret is short-lived and
@@ -36,7 +67,7 @@ export type RealtimeModelV4 = {
    *
    * Naming: "do" prefix to prevent accidental direct usage by the user.
    */
-  doCreateClientSecret(
+  doCreateClientSecret?(
     options: RealtimeModelV4ClientSecretOptions,
   ): PromiseLike<RealtimeModelV4ClientSecretResult>;
 
@@ -45,7 +76,7 @@ export type RealtimeModelV4 = {
    * when connecting. Each provider has its own authentication mechanism
    * (e.g. OpenAI uses subprotocol headers, xAI may use query params).
    */
-  getWebSocketConfig(options: { token: string; url: string }): {
+  getWebSocketConfig?(options: { token: string; url: string }): {
     url: string;
     protocols?: string[];
   };
@@ -63,6 +94,11 @@ export type RealtimeModelV4 = {
     raw: unknown,
   ): RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[];
 
+  /** Create a raw-event parser per connection and discard it on disconnect. */
+  createServerEventParser?(): (
+    raw: unknown,
+  ) => RealtimeModelV4ServerEvent | RealtimeModelV4ServerEvent[];
+
   /**
    * Browser-side: Serializes a normalized client event into the
    * provider's native JSON format for sending over the WebSocket.