@ai-sdk/openai 4.0.66 → 4.0.68

package/src/live/openai-realtime-model-live.ts

@@ -0,0 +1,107 @@
+import {
+  type Experimental_RealtimeModelV4 as RealtimeModelV4,
+  type Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  type Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import {
+  createJsonResponseHandler,
+  postJsonToApi,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import { openaiFailedResponseHandler } from '../openai-error';
+import {
+  createOpenAILiveServerEventParser,
+  parseOpenAILiveServerEvent,
+  serializeOpenAILiveClientEvent,
+} from './openai-live-event-mapper';
+import type { OpenAIRealtimeModelLiveId } from './openai-realtime-model-live-options';
+import { buildOpenAILiveSessionConfig } from './openai-live-session-config';
+import type { OpenAIRealtimeModelConfig } from '../realtime/openai-realtime-model';
+
+export type OpenAIRealtimeModelLiveConfig = OpenAIRealtimeModelConfig;
+
+const webRTCSessionSchema = z.object({
+  session: z.object({ id: z.string().min(1) }),
+  transport: z.object({ type: z.literal('webrtc'), sdp: z.string().min(1) }),
+});
+
+export class OpenAIRealtimeModelLive implements RealtimeModelV4 {
+  readonly specificationVersion = 'v4' as const;
+  readonly capabilities = {
+    conversation: 'continuous',
+    transports: ['websocket', 'webrtc'],
+    connections: ['server-websocket', 'webrtc'],
+    startup: 'session-start',
+    finalization: 'session-close',
+  } as const;
+
+  constructor(
+    readonly modelId: OpenAIRealtimeModelLiveId,
+    private readonly config: OpenAIRealtimeModelLiveConfig,
+  ) {}
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  getWebRTCConfig(): { dataChannelLabel: string } {
+    return { dataChannelLabel: 'oai-events' };
+  }
+
+  getServerWebSocketConfig(): { url: string; headers: Record<string, string> } {
+    const url = new URL(`${this.config.baseURL}/live/sessions`);
+    url.protocol = url.protocol === 'http:' ? 'ws:' : 'wss:';
+    const headers: Record<string, string> = {};
+    for (const [key, value] of Object.entries(this.config.headers())) {
+      if (value !== undefined) headers[key] = value;
+    }
+    return { url: url.toString(), headers };
+  }
+
+  async doCreateWebRTCSession({
+    sdp,
+    sessionConfig = {},
+    abortSignal,
+  }: {
+    sdp: string;
+    sessionConfig?: RealtimeModelV4SessionConfig;
+    abortSignal?: AbortSignal;
+  }): Promise<{ sessionId: string; sdp: string }> {
+    const session = buildOpenAILiveSessionConfig(
+      sessionConfig,
+      this.modelId,
+      'webrtc',
+    );
+    const { value } = await postJsonToApi({
+      url: `${this.config.baseURL}/live/sessions`,
+      headers: this.config.headers(),
+      body: {
+        session,
+        transport: { type: 'webrtc', sdp: z.string().min(1).parse(sdp) },
+      },
+      failedResponseHandler: openaiFailedResponseHandler,
+      successfulResponseHandler: createJsonResponseHandler(webRTCSessionSchema),
+      abortSignal,
+      fetch: this.config.fetch,
+    });
+    return { sessionId: value.session.id, sdp: value.transport.sdp };
+  }
+
+  parseServerEvent(raw: unknown) {
+    return parseOpenAILiveServerEvent(raw);
+  }
+
+  createServerEventParser() {
+    return createOpenAILiveServerEventParser();
+  }
+
+  serializeClientEvent(event: RealtimeModelV4ClientEvent): unknown {
+    return serializeOpenAILiveClientEvent(event, this.modelId);
+  }
+
+  buildSessionConfig(
+    config: RealtimeModelV4SessionConfig,
+  ): Record<string, unknown> {
+    return buildOpenAILiveSessionConfig(config, this.modelId);
+  }
+}

package/src/openai-provider.ts

@@ -5,8 +5,6 @@ import type {
   ImageModelV4,
   LanguageModelV4,
   ProviderV4,
-  Experimental_RealtimeFactoryV4 as RealtimeFactoryV4,
-  Experimental_RealtimeFactoryV4GetTokenOptions as RealtimeFactoryV4GetTokenOptions,
   SpeechModelV4,
   SkillsV4,
   TranscriptionModelV4,
@@ -33,7 +31,10 @@ import type { OpenAIImageModelId } from './image/openai-image-model-options';
 import { openaiTools } from './openai-tools';
 import { OpenAIBatch } from './openai-batch';
 import { OpenAIResponsesLanguageModel } from './responses/openai-responses-language-model';
-import { OpenAIRealtimeModel } from './realtime/openai-realtime-model';
+import {
+  createOpenAIRealtimeFactory,
+  type OpenAIRealtimeFactory,
+} from './realtime/openai-realtime-factory';
 import type { OpenAIResponsesModelId } from './responses/openai-responses-language-model-options';
 import { OpenAISpeechModel } from './speech/openai-speech-model';
 import type { OpenAISpeechModelId } from './speech/openai-speech-model-options';
@@ -125,7 +126,7 @@ export interface OpenAIProvider extends ProviderV4 {
    * Creates an experimental realtime model for bidirectional audio/text
    * communication over WebSocket.
    */
-  experimental_realtime: RealtimeFactoryV4;
+  experimental_realtime: OpenAIRealtimeFactory;
 
   /**
    * Returns a FilesV4 interface for uploading files to OpenAI.
@@ -337,33 +338,6 @@ export function createOpenAI(
       },
     });
 
-  const createRealtimeModel = (modelId: string) =>
-    new OpenAIRealtimeModel(modelId, {
-      provider: `${providerName}.realtime`,
-      baseURL,
-      headers: getHeaders,
-      fetch: options.fetch,
-    });
-
-  const experimentalRealtimeFactory = Object.assign(
-    (modelId: string) => createRealtimeModel(modelId),
-    {
-      getToken: async (tokenOptions: RealtimeFactoryV4GetTokenOptions) => {
-        const model = createRealtimeModel(tokenOptions.model);
-        const secret = await model.doCreateClientSecret({
-          sessionConfig: tokenOptions.sessionConfig,
-          expiresAfterSeconds: tokenOptions.expiresAfterSeconds,
-        });
-
-        return {
-          token: secret.token,
-          url: secret.url,
-          expiresAt: secret.expiresAt,
-        };
-      },
-    },
-  ) as RealtimeFactoryV4;
-
   const provider = function (modelId: OpenAIResponsesModelId) {
     return createLanguageModel(modelId);
   };
@@ -393,7 +367,12 @@ export function createOpenAI(
   provider.skills = createSkills;
   provider.experimental_batch = createBatch;
 
-  provider.experimental_realtime = experimentalRealtimeFactory;
+  provider.experimental_realtime = createOpenAIRealtimeFactory({
+    provider: providerName,
+    baseURL,
+    headers: getHeaders,
+    fetch: options.fetch,
+  });
 
   provider.tools = openaiTools;
 

package/src/realtime/openai-realtime-event-mapper.ts

@@ -9,7 +9,11 @@ type OpenAIRealtimeWireEvent = {
   session?: { id?: string };
   item?: { id?: string } & Record<string, unknown>;
   response?: { id?: string; status?: string };
-  error?: { message?: string; code?: string };
+  error?: {
+    message?: string | null;
+    code?: string | null;
+    event_id?: string | null;
+  } | null;
   item_id: string;
   previous_item_id?: string;
   response_id: string;
@@ -217,6 +221,7 @@ export function parseOpenAIRealtimeServerEvent(
         type: 'error',
         message: event.error?.message ?? event.message ?? 'Unknown error',
         code: event.error?.code ?? event.code,
+        clientEventId: event.error?.event_id ?? undefined,
         raw,
       };
 
@@ -238,12 +243,14 @@ export function serializeOpenAIRealtimeClientEvent(
       return {
         type: 'session.update',
         session: buildOpenAISessionConfig(event.config, modelId),
+        ...(event.eventId != null ? { event_id: event.eventId } : {}),
       };
 
     case 'input-audio-append':
       return {
         type: 'input_audio_buffer.append',
         audio: event.audio,
+        ...(event.eventId != null ? { event_id: event.eventId } : {}),
       };
 
     case 'input-audio-commit':

package/src/realtime/openai-realtime-factory.ts

@@ -0,0 +1,88 @@
+import {
+  InvalidArgumentError,
+  UnsupportedFunctionalityError,
+  type Experimental_RealtimeFactoryV4 as RealtimeFactoryV4,
+  type Experimental_RealtimeModelV4 as RealtimeModelV4,
+} from '@ai-sdk/provider';
+import { OpenAIRealtimeModelLive } from '../live/openai-realtime-model-live';
+import {
+  OpenAIRealtimeModel,
+  type OpenAIRealtimeModelConfig,
+} from './openai-realtime-model';
+
+const knownLiveModelIds = ['gpt-live-1'] as const;
+
+export type OpenAIRealtimeOptions = {
+  /** Overrides model ID routing, including for early-access models. */
+  api?: 'live' | 'realtime';
+};
+
+export interface OpenAIRealtimeFactory extends RealtimeFactoryV4 {
+  (modelId: string, options: { api: 'live' }): OpenAIRealtimeModelLive;
+  (modelId: string, options: { api: 'realtime' }): OpenAIRealtimeModel;
+  (
+    modelId: (typeof knownLiveModelIds)[number],
+    options?: { api?: undefined },
+  ): OpenAIRealtimeModelLive;
+  (modelId: string, options?: OpenAIRealtimeOptions): RealtimeModelV4;
+
+  getToken(
+    options: Parameters<RealtimeFactoryV4['getToken']>[0] &
+      OpenAIRealtimeOptions,
+  ): ReturnType<RealtimeFactoryV4['getToken']>;
+}
+
+function resolveRealtimeApi(
+  modelId: string,
+  { api }: OpenAIRealtimeOptions = {},
+): 'live' | 'realtime' {
+  if (api !== undefined) {
+    if (api !== 'live' && api !== 'realtime') {
+      throw new InvalidArgumentError({
+        argument: 'api',
+        message: 'OpenAI realtime api must be "live" or "realtime".',
+      });
+    }
+    return api;
+  }
+
+  return knownLiveModelIds.some(knownModelId => knownModelId === modelId)
+    ? 'live'
+    : 'realtime';
+}
+
+export function createOpenAIRealtimeFactory(
+  config: OpenAIRealtimeModelConfig,
+): OpenAIRealtimeFactory {
+  const createModel = (modelId: string, options?: OpenAIRealtimeOptions) => {
+    const api = resolveRealtimeApi(modelId, options);
+    const modelConfig = { ...config, provider: `${config.provider}.${api}` };
+    return api === 'live'
+      ? new OpenAIRealtimeModelLive(modelId, modelConfig)
+      : new OpenAIRealtimeModel(modelId, modelConfig);
+  };
+
+  return Object.assign(createModel, {
+    getToken: async (
+      options: Parameters<OpenAIRealtimeFactory['getToken']>[0],
+    ) => {
+      const model = createModel(options.model, options);
+      if (model instanceof OpenAIRealtimeModelLive) {
+        throw new UnsupportedFunctionalityError({
+          functionality:
+            'Short-lived OpenAI credentials for the Live API. Use server WebSocket setup via getServerWebSocketConfig() with a server-side API key instead.',
+        });
+      }
+
+      const secret = await model.doCreateClientSecret({
+        sessionConfig: options.sessionConfig,
+        expiresAfterSeconds: options.expiresAfterSeconds,
+      });
+      return {
+        token: secret.token,
+        url: secret.url,
+        expiresAt: secret.expiresAt,
+      };
+    },
+  }) as OpenAIRealtimeFactory;
+}