@ai-sdk/openai 4.0.66 → 4.0.67

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @ai-sdk/openai
 
+## 4.0.67
+
+### Patch Changes
+
+- 5c0054d: Add optional browser-direct WebRTC for experimental client-delegated Live conversations alongside the existing WebSocket path. Exchange SDP through an application endpoint with `api.session`, configure server-owned data-channel permissions, and preserve committed React session ownership. Capture follows the selected sender track, borrowed tracks remain caller-owned, and disconnect recovery and finalization stay bounded. Applications continue to handle client delegation and submit context; Live session updates and Responses delegation remain unsupported.
+
+  Serialize microphone sender changes and close the peer if detachment fails, without stopping borrowed tracks. Validate nonempty SDP setup answers with a bounded response body, and document the same-origin broker authentication contract.
+
+- 39535af: Add experimental OpenAI Live provider support through the unified `openai.experimental_realtime` factory for server WebSocket sessions with client delegation. Applications own their agents and tools, receive continuous audio/transcript events and delegation metadata, and return context through validated channels. Support immutable startup options, microphone mute controls, graceful session close, and cumulative voice usage. Responses delegation and Live session updates reject before sending.
+
+  Route known Live model IDs to Live, allow an OpenAI-specific `api` override for early-access models, and preserve legacy Realtime defaults for unknown IDs. Token minting follows the same selection rules and rejects Live before requesting unsupported credentials. Extend the realtime v4 specification with optional server WebSocket configuration, per-connection raw-event parsers, and model-wide startup/finalization capabilities. This provider layer supplies connection settings and protocol mapping for server adapters; browser lifecycle and UI integration belong to the core runtime and framework hooks.
+
+  Preserve Realtime client event IDs for session updates and audio appends, and correlate server errors with the originating client event.
+
+- Updated dependencies [5c0054d]
+- Updated dependencies [39535af]
+  - @ai-sdk/provider@4.0.15
+  - @ai-sdk/provider-utils@5.0.41
+
 ## 4.0.66
 
 ### Patch Changes

package/README.md

@@ -43,4 +43,29 @@ const { text } = await generateText({
 
 ## Documentation
 
-Please check out the **[OpenAI provider documentation](https://ai-sdk.dev/providers/ai-sdk-providers/openai)** for more information.
+### Experimental realtime models
+
+Use the same factory for OpenAI Realtime and Live models:
+
+```ts
+openai.experimental_realtime('gpt-realtime');
+openai.experimental_realtime('gpt-live-1');
+openai.experimental_realtime('not-yet-released', { api: 'live' });
+```
+
+Live supports browser WSS sessions through an application-owned relay and
+`experimental_useRealtime`, with client delegation: your application owns the
+agent and tools. Optional browser-direct WebRTC is also available. Start with the
+[OpenAI provider documentation](https://ai-sdk.dev/providers/ai-sdk-providers/openai#realtime-models)
+for connection settings, startup options, and event mapping.
+
+The [`experimental_useRealtime` reference](https://ai-sdk.dev/docs/reference/ai-sdk-ui/use-realtime#continuous-conversations)
+covers the JSON/PCM16 relay runtime, capture/playback controls, and application-handled
+client delegation, plus `api.session` WebRTC setup and server-owned data-channel
+permissions. Microphone capture requires browser permission; the application handles
+client delegation and submits context on either transport. The
+[Realtime guide](https://ai-sdk.dev/docs/ai-sdk-core/realtime) covers legacy token-based,
+turn-based sessions.
+
+Runtime and provider authors can consult the repository's
+[realtime integration notes](../../architecture/realtime-provider-integration.md).

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import * as _ai_sdk_provider from '@ai-sdk/provider';
-import { JSONValue, ProviderV4, LanguageModelV4, EmbeddingModelV4, ImageModelV4, TranscriptionModelV4, Experimental_SpeechTranslationModelV4, SpeechModelV4, Experimental_RealtimeFactoryV4, FilesV4, SkillsV4, Experimental_BatchV4, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientSecretOptions, Experimental_RealtimeModelV4ClientSecretResult, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4SessionConfig, JSONObject, Experimental_SpeechTranslationModelV4StreamOptions } from '@ai-sdk/provider';
+import { JSONValue, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientSecretOptions, Experimental_RealtimeModelV4ClientSecretResult, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4SessionConfig, Experimental_RealtimeFactoryV4, ProviderV4, LanguageModelV4, EmbeddingModelV4, ImageModelV4, TranscriptionModelV4, Experimental_SpeechTranslationModelV4, SpeechModelV4, FilesV4, SkillsV4, Experimental_BatchV4, JSONObject, Experimental_SpeechTranslationModelV4StreamOptions } from '@ai-sdk/provider';
 import * as _ai_sdk_provider_utils from '@ai-sdk/provider-utils';
 import { InferSchema, FetchFunction, WebSocketConstructor, WORKFLOW_SERIALIZE, WORKFLOW_DESERIALIZE } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
@@ -1375,6 +1375,128 @@ declare const openaiTools: {
     }, {}>;
 };
 
+type OpenAIRealtimeModelLiveId = 'gpt-live-1' | (string & {});
+declare const openaiRealtimeModelLiveOptionsSchema: z.ZodObject<{
+    client: z.ZodOptional<z.ZodObject<{
+        dataChannel: z.ZodObject<{
+            allowedClientEvents: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<"all">, z.ZodArray<z.ZodString>]>>;
+            allowedServerEvents: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<"all">, z.ZodArray<z.ZodObject<{
+                type: z.ZodString;
+                responseEvent: z.ZodOptional<z.ZodString>;
+            }, z.core.$strict>>]>>;
+        }, z.core.$strict>;
+    }, z.core.$strict>>;
+    delegation: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+        type: z.ZodLiteral<"client">;
+    }, z.core.$strict>>>;
+    input: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"message">;
+        role: z.ZodEnum<{
+            developer: "developer";
+            user: "user";
+        }>;
+        content: z.ZodTuple<[z.ZodObject<{
+            type: z.ZodLiteral<"input_text">;
+            text: z.ZodString;
+        }, z.core.$strict>], null>;
+    }, z.core.$strict>, z.ZodObject<{
+        type: z.ZodLiteral<"message">;
+        role: z.ZodLiteral<"assistant">;
+        content: z.ZodTuple<[z.ZodObject<{
+            type: z.ZodEnum<{
+                text: "text";
+                output_text: "output_text";
+            }>;
+            text: z.ZodString;
+        }, z.core.$strict>], null>;
+    }, z.core.$strict>]>>>;
+    store: z.ZodOptional<z.ZodBoolean>;
+    voice: z.ZodOptional<z.ZodObject<{
+        id: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+/** Experimental Live options under sessionConfig.providerOptions.openai. */
+type OpenAIRealtimeModelLiveOptions = z.infer<typeof openaiRealtimeModelLiveOptionsSchema>;
+
+type OpenAIRealtimeModelConfig = {
+    provider: string;
+    baseURL: string;
+    headers: () => Record<string, string | undefined>;
+    fetch?: FetchFunction;
+};
+declare class OpenAIRealtimeModel implements Experimental_RealtimeModelV4 {
+    readonly specificationVersion: "v4";
+    readonly provider: string;
+    readonly modelId: string;
+    private readonly config;
+    constructor(modelId: string, config: OpenAIRealtimeModelConfig);
+    doCreateClientSecret(options: Experimental_RealtimeModelV4ClientSecretOptions): Promise<Experimental_RealtimeModelV4ClientSecretResult>;
+    getWebSocketConfig(options: {
+        token: string;
+        url: string;
+    }): {
+        url: string;
+        protocols?: string[];
+    };
+    parseServerEvent(raw: unknown): Experimental_RealtimeModelV4ServerEvent;
+    serializeClientEvent(event: Experimental_RealtimeModelV4ClientEvent): unknown;
+    buildSessionConfig(config: Experimental_RealtimeModelV4SessionConfig): Record<string, unknown>;
+}
+
+type OpenAIRealtimeModelLiveConfig = OpenAIRealtimeModelConfig;
+declare class OpenAIRealtimeModelLive implements Experimental_RealtimeModelV4 {
+    readonly modelId: OpenAIRealtimeModelLiveId;
+    private readonly config;
+    readonly specificationVersion: "v4";
+    readonly capabilities: {
+        readonly conversation: "continuous";
+        readonly transports: readonly ["websocket", "webrtc"];
+        readonly connections: readonly ["server-websocket", "webrtc"];
+        readonly startup: "session-start";
+        readonly finalization: "session-close";
+    };
+    constructor(modelId: OpenAIRealtimeModelLiveId, config: OpenAIRealtimeModelLiveConfig);
+    get provider(): string;
+    getWebRTCConfig(): {
+        dataChannelLabel: string;
+    };
+    getServerWebSocketConfig(): {
+        url: string;
+        headers: Record<string, string>;
+    };
+    doCreateWebRTCSession({ sdp, sessionConfig, abortSignal, }: {
+        sdp: string;
+        sessionConfig?: Experimental_RealtimeModelV4SessionConfig;
+        abortSignal?: AbortSignal;
+    }): Promise<{
+        sessionId: string;
+        sdp: string;
+    }>;
+    parseServerEvent(raw: unknown): _ai_sdk_provider.Experimental_RealtimeModelV4ServerEvent[];
+    createServerEventParser(): (raw: unknown) => _ai_sdk_provider.Experimental_RealtimeModelV4ServerEvent[];
+    serializeClientEvent(event: Experimental_RealtimeModelV4ClientEvent): unknown;
+    buildSessionConfig(config: Experimental_RealtimeModelV4SessionConfig): Record<string, unknown>;
+}
+
+declare const knownLiveModelIds: readonly ["gpt-live-1"];
+type OpenAIRealtimeOptions = {
+    /** Overrides model ID routing, including for early-access models. */
+    api?: 'live' | 'realtime';
+};
+interface OpenAIRealtimeFactory extends Experimental_RealtimeFactoryV4 {
+    (modelId: string, options: {
+        api: 'live';
+    }): OpenAIRealtimeModelLive;
+    (modelId: string, options: {
+        api: 'realtime';
+    }): OpenAIRealtimeModel;
+    (modelId: (typeof knownLiveModelIds)[number], options?: {
+        api?: undefined;
+    }): OpenAIRealtimeModelLive;
+    (modelId: string, options?: OpenAIRealtimeOptions): Experimental_RealtimeModelV4;
+    getToken(options: Parameters<Experimental_RealtimeFactoryV4['getToken']>[0] & OpenAIRealtimeOptions): ReturnType<Experimental_RealtimeFactoryV4['getToken']>;
+}
+
 type OpenAIResponsesModelId = 'gpt-3.5-turbo-0125' | 'gpt-3.5-turbo-1106' | 'gpt-3.5-turbo' | 'gpt-4.1-2025-04-14' | 'gpt-4.1-mini-2025-04-14' | 'gpt-4.1-mini' | 'gpt-4.1-nano-2025-04-14' | 'gpt-4.1-nano' | 'gpt-4.1' | 'gpt-4o-2024-05-13' | 'gpt-4o-2024-08-06' | 'gpt-4o-2024-11-20' | 'gpt-4o-mini-2024-07-18' | 'gpt-4o-mini' | 'gpt-4o' | 'gpt-5.1' | 'gpt-5.1-2025-11-13' | 'gpt-5.1-chat-latest' | 'gpt-5.1-codex-mini' | 'gpt-5.1-codex' | 'gpt-5.1-codex-max' | 'gpt-5.2' | 'gpt-5.2-2025-12-11' | 'gpt-5.2-chat-latest' | 'gpt-5.2-pro' | 'gpt-5.2-pro-2025-12-11' | 'gpt-5.2-codex' | 'gpt-5.3-chat-latest' | 'gpt-5.3-codex' | 'gpt-5.4' | 'gpt-5.4-2026-03-05' | 'gpt-5.4-mini' | 'gpt-5.4-mini-2026-03-17' | 'gpt-5.4-nano' | 'gpt-5.4-nano-2026-03-17' | 'gpt-5.4-pro' | 'gpt-5.4-pro-2026-03-05' | 'gpt-5.5' | 'gpt-5.5-2026-04-23' | 'gpt-5.6' | 'gpt-5.6-luna' | 'gpt-5.6-sol' | 'gpt-5.6-terra' | 'gpt-6-astra' | 'gpt-5-2025-08-07' | 'gpt-5-chat-latest' | 'gpt-5-codex' | 'gpt-5-mini-2025-08-07' | 'gpt-5-mini' | 'gpt-5-nano-2025-08-07' | 'gpt-5-nano' | 'gpt-5-pro-2025-10-06' | 'gpt-5-pro' | 'gpt-5' | 'o1-2024-12-17' | 'o1' | 'o3-2025-04-16' | 'o3-mini-2025-01-31' | 'o3-mini' | 'o3' | 'o4-mini' | 'o4-mini-2025-04-16' | (string & {});
 declare const openaiLanguageModelResponsesOptionsSchema: _ai_sdk_provider_utils.LazySchema<{
     conversation?: string | null | undefined;
@@ -1513,7 +1635,7 @@ interface OpenAIProvider extends ProviderV4 {
      * Creates an experimental realtime model for bidirectional audio/text
      * communication over WebSocket.
      */
-    experimental_realtime: Experimental_RealtimeFactoryV4;
+    experimental_realtime: OpenAIRealtimeFactory;
     /**
      * Returns a FilesV4 interface for uploading files to OpenAI.
      */
@@ -1578,31 +1700,6 @@ declare function createOpenAI(options?: OpenAIProviderSettings): OpenAIProvider;
  */
 declare const openai: OpenAIProvider;
 
-type OpenAIRealtimeModelConfig = {
-    provider: string;
-    baseURL: string;
-    headers: () => Record<string, string | undefined>;
-    fetch?: FetchFunction;
-};
-declare class OpenAIRealtimeModel implements Experimental_RealtimeModelV4 {
-    readonly specificationVersion: "v4";
-    readonly provider: string;
-    readonly modelId: string;
-    private readonly config;
-    constructor(modelId: string, config: OpenAIRealtimeModelConfig);
-    doCreateClientSecret(options: Experimental_RealtimeModelV4ClientSecretOptions): Promise<Experimental_RealtimeModelV4ClientSecretResult>;
-    getWebSocketConfig(options: {
-        token: string;
-        url: string;
-    }): {
-        url: string;
-        protocols?: string[];
-    };
-    parseServerEvent(raw: unknown): Experimental_RealtimeModelV4ServerEvent;
-    serializeClientEvent(event: Experimental_RealtimeModelV4ClientEvent): unknown;
-    buildSessionConfig(config: Experimental_RealtimeModelV4SessionConfig): Record<string, unknown>;
-}
-
 type OpenAIToolOptions = {
     allowedCallers?: Array<'direct' | 'programmatic'>;
     /**
@@ -1807,4 +1904,4 @@ type OpenaiResponsesSourceDocumentProviderMetadata = {
 
 declare const VERSION: string;
 
-export { OpenAIRealtimeModel as Experimental_OpenAIRealtimeModel, type OpenAIRealtimeModelConfig as Experimental_OpenAIRealtimeModelConfig, OpenAISpeechTranslationModel as Experimental_OpenAISpeechTranslationModel, type OpenAISpeechTranslationModelId as Experimental_OpenAISpeechTranslationModelId, type OpenAISpeechTranslationModelOptions as Experimental_OpenAISpeechTranslationModelOptions, OpenAISpeechTranslationModel as Experimental_OpenAITranslationModel, type OpenAISpeechTranslationModelId as Experimental_OpenAITranslationModelId, type OpenAISpeechTranslationModelOptions as Experimental_OpenAITranslationModelOptions, type OpenAILanguageModelChatOptions as OpenAIChatLanguageModelOptions, type OpenAIComputerAction, type OpenAIComputerSafetyCheck, type OpenAIEmbeddingModelOptions, type OpenAIFilesOptions, type OpenAIImageModelEditOptions, type OpenAIImageModelGenerationOptions, type OpenAIImageModelOptions, type OpenAILanguageModelChatOptions, type OpenAILanguageModelCompletionOptions, type OpenAILanguageModelResponsesOptions, type OpenAIProvider, type OpenAIProviderSettings, type OpenAILanguageModelResponsesOptions as OpenAIResponsesProviderOptions, type OpenAISpeechModelOptions, type OpenAIToolOptions, type OpenAITranscriptionModelOptions, type OpenaiResponsesCompactionProviderMetadata, type OpenaiResponsesProviderMetadata, type OpenaiResponsesReasoningProviderMetadata, type OpenaiResponsesSourceDocumentProviderMetadata, type OpenaiResponsesTextProviderMetadata, type OpenaiResponsesToolCallProviderMetadata, VERSION, createOpenAI, openai };
+export { type OpenAIRealtimeFactory as Experimental_OpenAIRealtimeFactory, OpenAIRealtimeModel as Experimental_OpenAIRealtimeModel, type OpenAIRealtimeModelConfig as Experimental_OpenAIRealtimeModelConfig, OpenAIRealtimeModelLive as Experimental_OpenAIRealtimeModelLive, type OpenAIRealtimeModelLiveConfig as Experimental_OpenAIRealtimeModelLiveConfig, type OpenAIRealtimeModelLiveId as Experimental_OpenAIRealtimeModelLiveId, type OpenAIRealtimeModelLiveOptions as Experimental_OpenAIRealtimeModelLiveOptions, type OpenAIRealtimeOptions as Experimental_OpenAIRealtimeOptions, OpenAISpeechTranslationModel as Experimental_OpenAISpeechTranslationModel, type OpenAISpeechTranslationModelId as Experimental_OpenAISpeechTranslationModelId, type OpenAISpeechTranslationModelOptions as Experimental_OpenAISpeechTranslationModelOptions, OpenAISpeechTranslationModel as Experimental_OpenAITranslationModel, type OpenAISpeechTranslationModelId as Experimental_OpenAITranslationModelId, type OpenAISpeechTranslationModelOptions as Experimental_OpenAITranslationModelOptions, type OpenAILanguageModelChatOptions as OpenAIChatLanguageModelOptions, type OpenAIComputerAction, type OpenAIComputerSafetyCheck, type OpenAIEmbeddingModelOptions, type OpenAIFilesOptions, type OpenAIImageModelEditOptions, type OpenAIImageModelGenerationOptions, type OpenAIImageModelOptions, type OpenAILanguageModelChatOptions, type OpenAILanguageModelCompletionOptions, type OpenAILanguageModelResponsesOptions, type OpenAIProvider, type OpenAIProviderSettings, type OpenAILanguageModelResponsesOptions as OpenAIResponsesProviderOptions, type OpenAISpeechModelOptions, type OpenAIToolOptions, type OpenAITranscriptionModelOptions, type OpenaiResponsesCompactionProviderMetadata, type OpenaiResponsesProviderMetadata, type OpenaiResponsesReasoningProviderMetadata, type OpenaiResponsesSourceDocumentProviderMetadata, type OpenaiResponsesTextProviderMetadata, type OpenaiResponsesToolCallProviderMetadata, VERSION, createOpenAI, openai };