@ai-sdk/openai 4.0.108 → 4.0.110

package/src/realtime/openai-realtime-event-mapper.ts

@@ -2,13 +2,48 @@ import type {
   Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
+  Experimental_RealtimeModelV4Usage as RealtimeModelV4Usage,
 } from '@ai-sdk/provider';
 
+type OpenAIRealtimeSessionOptions = {
+  intent?: RealtimeModelV4SessionIntent;
+};
+
+type OpenAIRealtimeTokenDetails = {
+  audio_tokens?: number;
+  text_tokens?: number;
+};
+
+type OpenAIRealtimeInputTokenDetails = OpenAIRealtimeTokenDetails & {
+  cached_tokens?: number;
+  cached_tokens_details?: OpenAIRealtimeTokenDetails;
+};
+
+type OpenAIRealtimeResponseUsage = {
+  input_tokens?: number;
+  output_tokens?: number;
+  input_token_details?: OpenAIRealtimeInputTokenDetails;
+  output_token_details?: OpenAIRealtimeTokenDetails;
+};
+
+type OpenAIRealtimeTranscriptionUsage = {
+  input_tokens?: number;
+  type?: string;
+  seconds?: number;
+  output_tokens?: number;
+  input_token_details?: OpenAIRealtimeTokenDetails;
+};
+
 type OpenAIRealtimeWireEvent = {
   type: string;
   session?: { id?: string };
   item?: { id?: string } & Record<string, unknown>;
-  response?: { id?: string; status?: string };
+  response?: {
+    id?: string;
+    status?: string;
+    usage?: OpenAIRealtimeResponseUsage;
+  };
   error?: {
     message?: string | null;
     code?: string | null;
@@ -25,6 +60,7 @@ type OpenAIRealtimeWireEvent = {
   arguments: string;
   message?: string;
   code?: string;
+  usage?: OpenAIRealtimeTranscriptionUsage;
 };
 
 /**
@@ -80,13 +116,16 @@ export function parseOpenAIRealtimeServerEvent(
         raw,
       };
 
-    case 'conversation.item.input_audio_transcription.completed':
+    case 'conversation.item.input_audio_transcription.completed': {
+      const usage = mapTranscriptionUsage(event.usage);
       return {
         type: 'input-transcription-completed',
         itemId: event.item_id,
         transcript: event.transcript ?? '',
+        ...(usage != null ? { usage } : {}),
         raw,
       };
+    }
 
     // ── Response lifecycle ──────────────────────────────────────────
     case 'response.created':
@@ -96,13 +135,16 @@ export function parseOpenAIRealtimeServerEvent(
         raw,
       };
 
-    case 'response.done':
+    case 'response.done': {
+      const usage = mapResponseUsage(event.response?.usage);
       return {
         type: 'response-done',
         responseId: event.response?.id ?? event.response_id,
         status: event.response?.status ?? 'completed',
+        ...(usage != null ? { usage } : {}),
         raw,
       };
+    }
 
     // ── Output item lifecycle ───────────────────────────────────────
     case 'response.output_item.added':
@@ -231,18 +273,86 @@ export function parseOpenAIRealtimeServerEvent(
   }
 }
 
+/**
+ * Maps OpenAI realtime `response.done` usage to normalized usage.
+ *
+ * Input buckets are reported gross (cache-inclusive) with the cached portion
+ * surfaced separately, so consumers can apply their own cached-vs-uncached
+ * billing split without re-reading `raw`.
+ */
+function mapResponseUsage(
+  usage: OpenAIRealtimeResponseUsage | undefined,
+): RealtimeModelV4Usage | undefined {
+  if (usage == null) return undefined;
+
+  const input = usage.input_token_details;
+  const output = usage.output_token_details;
+  const cached = input?.cached_tokens_details;
+
+  return compactUsage({
+    inputTokens: usage.input_tokens,
+    outputTokens: usage.output_tokens,
+    cachedInputTokens: input?.cached_tokens,
+    inputAudioTokens: input?.audio_tokens,
+    inputTextTokens: input?.text_tokens,
+    outputAudioTokens: output?.audio_tokens,
+    outputTextTokens: output?.text_tokens,
+    cachedInputAudioTokens: cached?.audio_tokens,
+    cachedInputTextTokens: cached?.text_tokens,
+  });
+}
+
+/**
+ * Maps OpenAI realtime transcription-completed usage to normalized usage.
+ * Duration-billed transcription reports `seconds`; token-billed reports tokens.
+ */
+function mapTranscriptionUsage(
+  usage: OpenAIRealtimeTranscriptionUsage | undefined,
+): RealtimeModelV4Usage | undefined {
+  if (usage == null) return undefined;
+
+  if (usage.type === 'duration') {
+    return usage.seconds != null ? { seconds: usage.seconds } : undefined;
+  }
+
+  const input = usage.input_token_details;
+  return compactUsage({
+    inputTokens: usage.input_tokens,
+    outputTokens: usage.output_tokens,
+    inputAudioTokens: input?.audio_tokens,
+    inputTextTokens: input?.text_tokens,
+    outputTextTokens: usage.output_tokens,
+  });
+}
+
+/** Drop `undefined` fields; return `undefined` when nothing is observable. */
+function compactUsage(
+  usage: RealtimeModelV4Usage,
+): RealtimeModelV4Usage | undefined {
+  const result: RealtimeModelV4Usage = {};
+  for (const [key, value] of Object.entries(usage)) {
+    if (value != null) {
+      result[key as keyof RealtimeModelV4Usage] = value;
+    }
+  }
+  return Object.keys(result).length > 0 ? result : undefined;
+}
+
 /**
  * Serializes a normalized client event into OpenAI's Realtime API format.
  */
 export function serializeOpenAIRealtimeClientEvent(
   event: RealtimeModelV4ClientEvent,
   modelId: string,
+  options?: OpenAIRealtimeSessionOptions,
 ): unknown {
+  assertSupportedOpenAIIntent(options?.intent);
+
   switch (event.type) {
     case 'session-update':
       return {
         type: 'session.update',
-        session: buildOpenAISessionConfig(event.config, modelId),
+        session: buildOpenAISessionConfig(event.config, modelId, options),
         ...(event.eventId != null ? { event_id: event.eventId } : {}),
       };
 
@@ -332,23 +442,43 @@ export function serializeOpenAIRealtimeClientEvent(
 export function buildOpenAISessionConfig(
   config: RealtimeModelV4SessionConfig,
   modelId: string,
+  options?: OpenAIRealtimeSessionOptions,
 ): Record<string, unknown> {
+  const intent = options?.intent ?? 'conversation';
+  assertSupportedOpenAIIntent(intent);
+  if (intent === 'transcription') {
+    for (const key of [
+      'instructions',
+      'voice',
+      'outputModalities',
+      'outputAudioFormat',
+      'outputAudioTranscription',
+      'tools',
+    ] as const) {
+      if (config[key] != null)
+        throw new Error(
+          `OpenAI realtime transcription sessions do not support '${key}'.`,
+        );
+    }
+  }
+
   const session: Record<string, unknown> = {
-    type: 'realtime',
-    model: modelId,
+    type: intent === 'transcription' ? 'transcription' : 'realtime',
+    ...(intent === 'conversation' ? { model: modelId } : {}),
   };
 
-  if (config.instructions != null) {
+  if (intent === 'conversation' && config.instructions != null) {
     session.instructions = config.instructions;
   }
 
-  if (config.outputModalities != null) {
+  if (intent === 'conversation' && config.outputModalities != null) {
     session.output_modalities = config.outputModalities;
   }
 
   const audio: Record<string, unknown> = {};
 
   if (
+    intent === 'transcription' ||
     config.inputAudioFormat != null ||
     config.inputAudioTranscription != null ||
     config.turnDetection != null
@@ -387,7 +517,17 @@ export function buildOpenAISessionConfig(
       }
     }
 
-    if (config.inputAudioTranscription != null) {
+    if (intent === 'transcription') {
+      input.transcription = {
+        model: config.inputAudioTranscription?.model ?? modelId,
+        ...(config.inputAudioTranscription?.language != null
+          ? { language: config.inputAudioTranscription.language }
+          : {}),
+        ...(config.inputAudioTranscription?.prompt != null
+          ? { prompt: config.inputAudioTranscription.prompt }
+          : {}),
+      };
+    } else if (config.inputAudioTranscription != null) {
       input.transcription = {
         model: config.inputAudioTranscription.model ?? 'gpt-realtime-whisper',
         ...(config.inputAudioTranscription.language != null
@@ -402,7 +542,10 @@ export function buildOpenAISessionConfig(
     audio.input = input;
   }
 
-  if (config.outputAudioFormat != null || config.voice != null) {
+  if (
+    intent === 'conversation' &&
+    (config.outputAudioFormat != null || config.voice != null)
+  ) {
     const output: Record<string, unknown> = {};
 
     if (config.outputAudioFormat != null) {
@@ -425,7 +568,11 @@ export function buildOpenAISessionConfig(
     session.audio = audio;
   }
 
-  if (config.tools != null && config.tools.length > 0) {
+  if (
+    intent === 'conversation' &&
+    config.tools != null &&
+    config.tools.length > 0
+  ) {
     session.tools = config.tools.map(tool => ({
       type: tool.type,
       name: tool.name,
@@ -441,3 +588,17 @@ export function buildOpenAISessionConfig(
 
   return session;
 }
+
+function assertSupportedOpenAIIntent(
+  intent: RealtimeModelV4SessionIntent | undefined,
+): void {
+  if (
+    intent != null &&
+    intent !== 'conversation' &&
+    intent !== 'transcription'
+  ) {
+    throw new Error(
+      `OpenAI realtime session intent '${intent}' is not supported.`,
+    );
+  }
+}

package/src/realtime/openai-realtime-model.ts

@@ -1,6 +1,8 @@
 import {
   type Resolvable,
   resolve,
+  resolveWith,
+  removeUndefinedEntries,
   type FetchFunction,
 } from '@ai-sdk/provider-utils';
 import type {
@@ -8,8 +10,10 @@ import type {
   Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
   Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
   Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerConnection as RealtimeModelV4ServerConnection,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
 } from '@ai-sdk/provider';
 import {
   buildOpenAISessionConfig,
@@ -26,6 +30,11 @@ export type OpenAIRealtimeModelConfig = {
 
 export class OpenAIRealtimeModel implements RealtimeModelV4 {
   readonly specificationVersion = 'v4' as const;
+  readonly capabilities = {
+    conversation: 'turn-based',
+    transports: ['websocket'],
+    connections: ['client-secret-websocket', 'server-websocket'],
+  } as const;
   readonly provider: string;
   readonly modelId: string;
 
@@ -99,17 +108,62 @@ export class OpenAIRealtimeModel implements RealtimeModelV4 {
     };
   }
 
+  getServerWebSocketConfig(options?: {
+    intent?: RealtimeModelV4SessionIntent;
+  }):
+    | RealtimeModelV4ServerConnection
+    | PromiseLike<RealtimeModelV4ServerConnection> {
+    return resolveWith(this.config.headers, headers => ({
+      url: buildServerUrl(
+        this.config.baseURL,
+        this.modelId,
+        options?.intent ?? 'conversation',
+      ),
+      headers: removeUndefinedEntries(headers),
+    }));
+  }
+
   parseServerEvent(raw: unknown): RealtimeModelV4ServerEvent {
     return parseOpenAIRealtimeServerEvent(raw);
   }
 
-  serializeClientEvent(event: RealtimeModelV4ClientEvent): unknown {
-    return serializeOpenAIRealtimeClientEvent(event, this.modelId);
+  serializeClientEvent(
+    event: RealtimeModelV4ClientEvent,
+    options?: { intent?: RealtimeModelV4SessionIntent },
+  ): unknown {
+    return serializeOpenAIRealtimeClientEvent(event, this.modelId, options);
   }
 
   buildSessionConfig(
     config: RealtimeModelV4SessionConfig,
+    options?: { intent?: RealtimeModelV4SessionIntent },
   ): Record<string, unknown> {
-    return buildOpenAISessionConfig(config, this.modelId);
+    return buildOpenAISessionConfig(config, this.modelId, options);
+  }
+}
+
+/**
+ * Builds the upstream realtime WebSocket URL for a server-initiated connection.
+ * Transcription keeps the model in `session.update`; a `?model=` here would
+ * open a voice session instead.
+ */
+function buildServerUrl(
+  baseURL: string,
+  modelId: string,
+  intent: RealtimeModelV4SessionIntent,
+): string {
+  const url = new URL(baseURL);
+  const protocol = url.protocol === 'http:' ? 'ws:' : 'wss:';
+  const base = `${protocol}//${url.host}${url.pathname.replace(/\/$/, '')}`;
+
+  switch (intent) {
+    case 'transcription':
+      return `${base}/realtime?intent=transcription`;
+    case 'conversation':
+      return `${base}/realtime?model=${encodeURIComponent(modelId)}`;
+    default:
+      throw new Error(
+        `OpenAI realtime session intent '${intent}' is not supported.`,
+      );
   }
 }

package/src/responses/map-openai-responses-finish-reason.ts

@@ -3,15 +3,21 @@ import type { LanguageModelV4FinishReason } from '@ai-sdk/provider';
 export function mapOpenAIResponseFinishReason({
   finishReason,
   hasFunctionCall,
+  hasRefusal = false,
 }: {
   finishReason: string | null | undefined;
   // flag that checks if there have been client-side tool calls (not executed by openai)
   hasFunctionCall: boolean;
+  // flag that checks if the model refused to answer:
+  hasRefusal?: boolean;
 }): LanguageModelV4FinishReason['unified'] {
   switch (finishReason) {
     case undefined:
     case null:
-      return hasFunctionCall ? 'tool-calls' : 'stop';
+      // A client-side function call must win over a refusal so that the tools
+      // are still executed. The refusal text remains part of the content.
+      if (hasFunctionCall) return 'tool-calls';
+      return hasRefusal ? 'content-filter' : 'stop';
     case 'max_output_tokens':
       return 'length';
     case 'content_filter':

package/src/responses/openai-responses-api.ts

@@ -647,7 +647,24 @@ export type OpenAIResponsesTool =
     }
   | {
       type: 'code_interpreter';
-      container: string | { type: 'auto'; file_ids: string[] | undefined };
+      container:
+        | string
+        | {
+            type: 'auto';
+            file_ids: string[] | undefined;
+            memory_limit?: '1g' | '4g' | '16g' | '64g';
+            network_policy?:
+              | { type: 'disabled' }
+              | {
+                  type: 'allowlist';
+                  allowed_domains: string[];
+                  domain_secrets?: Array<{
+                    domain: string;
+                    name: string;
+                    value: string;
+                  }>;
+                };
+          };
     }
   | {
       type: 'file_search';
@@ -712,6 +729,7 @@ export type OpenAIResponsesTool =
         | undefined;
       server_description: string | undefined;
       server_url: string | undefined;
+      tunnel_id: string | undefined;
     }
   | {
       type: 'custom';
@@ -851,6 +869,8 @@ const openaiResponsesModeledChunkTypes = new Set([
   'response.output_item.done',
   'response.output_text.annotation.added',
   'response.output_text.delta',
+  'response.refusal.delta',
+  'response.refusal.done',
   'response.reasoning_summary_part.added',
   'response.reasoning_summary_part.done',
   'response.reasoning_summary_text.delta',
@@ -930,6 +950,20 @@ export const openaiResponsesChunkSchema = lazySchema(() =>
           )
           .nullish(),
       }),
+      z.object({
+        type: z.literal('response.refusal.delta'),
+        item_id: z.string(),
+        output_index: z.number().nullish(),
+        content_index: z.number().nullish(),
+        delta: z.string(),
+      }),
+      z.object({
+        type: z.literal('response.refusal.done'),
+        item_id: z.string(),
+        output_index: z.number().nullish(),
+        content_index: z.number().nullish(),
+        refusal: z.string(),
+      }),
       z.object({
         type: z.enum(['response.completed', 'response.incomplete']),
         response: z.object({
@@ -1530,55 +1564,62 @@ export const openaiResponsesResponseSchema = lazySchema(() =>
               role: z.literal('assistant'),
               id: z.string(),
               phase: z.enum(['commentary', 'final_answer']).nullish(),
+              // the API returns either text or a refusal as content parts:
               content: z.array(
-                z.object({
-                  type: z.literal('output_text'),
-                  text: z.string(),
-                  logprobs: z
-                    .array(
-                      z.object({
-                        token: z.string(),
-                        logprob: z.number(),
-                        top_logprobs: z.array(
-                          z.object({
-                            token: z.string(),
-                            logprob: z.number(),
-                          }),
-                        ),
-                      }),
-                    )
-                    .nullish(),
-                  annotations: z.array(
-                    z.discriminatedUnion('type', [
-                      z.object({
-                        type: z.literal('url_citation'),
-                        start_index: z.number(),
-                        end_index: z.number(),
-                        url: z.string(),
-                        title: z.string(),
-                      }),
-                      z.object({
-                        type: z.literal('file_citation'),
-                        file_id: z.string(),
-                        filename: z.string(),
-                        index: z.number(),
-                      }),
-                      z.object({
-                        type: z.literal('container_file_citation'),
-                        container_id: z.string(),
-                        file_id: z.string(),
-                        filename: z.string(),
-                        start_index: z.number(),
-                        end_index: z.number(),
-                      }),
-                      z.object({
-                        type: z.literal('file_path'),
-                        file_id: z.string(),
-                        index: z.number(),
-                      }),
-                    ]),
-                  ),
-                }),
+                z.discriminatedUnion('type', [
+                  z.object({
+                    type: z.literal('output_text'),
+                    text: z.string(),
+                    logprobs: z
+                      .array(
+                        z.object({
+                          token: z.string(),
+                          logprob: z.number(),
+                          top_logprobs: z.array(
+                            z.object({
+                              token: z.string(),
+                              logprob: z.number(),
+                            }),
+                          ),
+                        }),
+                      )
+                      .nullish(),
+                    annotations: z.array(
+                      z.discriminatedUnion('type', [
+                        z.object({
+                          type: z.literal('url_citation'),
+                          start_index: z.number(),
+                          end_index: z.number(),
+                          url: z.string(),
+                          title: z.string(),
+                        }),
+                        z.object({
+                          type: z.literal('file_citation'),
+                          file_id: z.string(),
+                          filename: z.string(),
+                          index: z.number(),
+                        }),
+                        z.object({
+                          type: z.literal('container_file_citation'),
+                          container_id: z.string(),
+                          file_id: z.string(),
+                          filename: z.string(),
+                          start_index: z.number(),
+                          end_index: z.number(),
+                        }),
+                        z.object({
+                          type: z.literal('file_path'),
+                          file_id: z.string(),
+                          index: z.number(),
+                        }),
+                      ]),
+                    ),
+                  }),
+                  z.object({
+                    type: z.literal('refusal'),
+                    refusal: z.string(),
+                  }),
+                ]),
               ),
             }),
             z.object({

package/src/responses/openai-responses-language-model.ts

@@ -923,6 +923,9 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
     let hasFunctionCall = false;
     const hostedToolSearchCallIds: string[] = [];
 
+    // flag that checks if the model refused to answer:
+    let hasRefusal = false;
+
     let structuredOutputText: string | undefined;
 
     // map response content to content array
@@ -1092,11 +1095,32 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
             part.phase === 'final_answer'
           ) {
             structuredOutputText = part.content
-              .map(contentPart => contentPart.text)
+              .map(contentPart =>
+                contentPart.type === 'output_text'
+                  ? contentPart.text
+                  : contentPart.refusal,
+              )
               .join('');
           }
 
           for (const contentPart of part.content) {
+            if (contentPart.type === 'refusal') {
+              hasRefusal = true;
+
+              content.push({
+                type: 'text',
+                text: contentPart.refusal,
+                providerMetadata: {
+                  [providerOptionsName]: {
+                    itemId: part.id,
+                    ...(part.phase != null && { phase: part.phase }),
+                  } satisfies ResponsesTextProviderMetadata,
+                },
+              });
+
+              continue;
+            }
+
             if (
               options.providerOptions?.[providerOptionsName]?.logprobs &&
               contentPart.logprobs
@@ -1529,6 +1553,7 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
         unified: mapOpenAIResponseFinishReason({
           finishReason: response.incomplete_details?.reason,
           hasFunctionCall,
+          hasRefusal,
         }),
         raw: response.incomplete_details?.reason ?? undefined,
       },
@@ -1658,6 +1683,9 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
     // flag that checks if there have been client-side tool calls (not executed by openai)
     let hasFunctionCall = false;
 
+    // flag that checks if the model refused to answer:
+    let hasRefusal = false;
+
     const activeReasoning: Record<
       string,
       {
@@ -2772,6 +2800,20 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
               ) {
                 logprobs.push(value.logprobs);
               }
+            } else if (isRefusalDeltaChunk(value)) {
+              hasRefusal = true;
+
+              const itemId = resolveOutputItemId({
+                itemId: value.item_id,
+                outputIndex: value.output_index,
+              });
+              controller.enqueue({
+                type: 'text-delta',
+                id: itemId,
+                delta: value.delta,
+              });
+            } else if (isRefusalDoneChunk(value)) {
+              hasRefusal = true;
             } else if (value.type === 'response.reasoning_summary_part.added') {
               const itemId = resolveOutputItemId({
                 itemId: value.item_id,
@@ -2872,6 +2914,7 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV4 {
                   unified: mapOpenAIResponseFinishReason({
                     finishReason: value.response.incomplete_details?.reason,
                     hasFunctionCall,
+                    hasRefusal,
                   }),
                   raw: value.response.incomplete_details?.reason ?? undefined,
                 };
@@ -3014,6 +3057,18 @@ function isTextDeltaChunk(
   return chunk.type === 'response.output_text.delta';
 }
 
+function isRefusalDeltaChunk(
+  chunk: OpenAIResponsesChunk,
+): chunk is OpenAIResponsesChunk & { type: 'response.refusal.delta' } {
+  return chunk.type === 'response.refusal.delta';
+}
+
+function isRefusalDoneChunk(
+  chunk: OpenAIResponsesChunk,
+): chunk is OpenAIResponsesChunk & { type: 'response.refusal.done' } {
+  return chunk.type === 'response.refusal.done';
+}
+
 function isOpenAIChatCompletionChunk(value: unknown): boolean {
   const chunk = asRecord(value);