@ai-sdk/openai 4.0.107 → 4.0.109

package/src/internal/index.ts

@@ -13,10 +13,12 @@ export * from '../speech/openai-speech-model-options';
 export * from '../responses/openai-responses-language-model';
 export * from '../responses/openai-responses-provider-metadata';
 export * from '../tool/apply-patch';
+export * from '../tool/custom';
 export * from '../tool/code-interpreter';
 export * from '../tool/file-search';
 export * from '../tool/image-generation';
 export * from '../tool/programmatic-tool-calling';
+export * from '../tool/shell';
 export * from '../tool/tool-search';
 export * from '../tool/web-search';
 export * from '../tool/web-search-preview';

package/src/live/openai-realtime-model-live.ts

@@ -2,6 +2,7 @@ import {
   type Experimental_RealtimeModelV4 as RealtimeModelV4,
   type Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
   type Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  type Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
 } from '@ai-sdk/provider';
 import {
   resolveWith,
@@ -54,7 +55,10 @@ export class OpenAIRealtimeModelLive implements RealtimeModelV4 {
     return { dataChannelLabel: 'oai-events' };
   }
 
-  getServerWebSocketConfig() {
+  getServerWebSocketConfig(options?: {
+    intent?: RealtimeModelV4SessionIntent;
+  }) {
+    this.assertConversationIntent(options);
     return resolveWith(this.config.headers, configHeaders => {
       const url = new URL(`${this.config.baseURL}/live/sessions`);
       url.protocol = url.protocol === 'http:' ? 'ws:' : 'wss:';
@@ -103,13 +107,28 @@ export class OpenAIRealtimeModelLive implements RealtimeModelV4 {
     return createOpenAILiveServerEventParser();
   }
 
-  serializeClientEvent(event: RealtimeModelV4ClientEvent): unknown {
+  private assertConversationIntent(options?: {
+    intent?: RealtimeModelV4SessionIntent;
+  }): void {
+    if (options?.intent != null && options.intent !== 'conversation')
+      throw new Error(
+        `OpenAI Live session intent '${options.intent}' is not supported.`,
+      );
+  }
+
+  serializeClientEvent(
+    event: RealtimeModelV4ClientEvent,
+    options?: { intent?: RealtimeModelV4SessionIntent },
+  ): unknown {
+    this.assertConversationIntent(options);
     return serializeOpenAILiveClientEvent(event, this.modelId);
   }
 
   buildSessionConfig(
     config: RealtimeModelV4SessionConfig,
+    options?: { intent?: RealtimeModelV4SessionIntent },
   ): Record<string, unknown> {
+    this.assertConversationIntent(options);
     return buildOpenAILiveSessionConfig(config, this.modelId);
   }
 }

package/src/openai-model-catalog.ts

@@ -461,6 +461,8 @@ function createGptImageEntry(
 
 const openaiImageModelEntries: Readonly<Record<string, OpenAIImageModelEntry>> =
   {
+    'gpt-image-2.6-flare': createGptImageEntry(customImageSizes),
+    'gpt-image-2.6-sunburst': createGptImageEntry(customImageSizes),
     'gpt-image-2.5-flare': createGptImageEntry(customImageSizes),
     'gpt-image-2.5-sunburst': createGptImageEntry(customImageSizes),
     'gpt-image-2': createGptImageEntry(gptImageSizes),
@@ -757,7 +759,7 @@ const realtimeModelEntry = {
     realtime: {
       conversation: 'turn-based',
       transports: ['websocket'],
-      connections: ['client-secret-websocket'],
+      connections: ['client-secret-websocket', 'server-websocket'],
     },
   },
   requestConfig: {

package/src/openai-tools.ts

@@ -126,16 +126,17 @@ export const openaiTools = {
 
   /**
    * MCP (Model Context Protocol) allows models to call tools exposed by
-   * remote MCP servers or service connectors.
+   * remote MCP servers, service connectors, or Secure MCP tunnels.
    *
    * @param serverLabel - Label to identify the MCP server.
    * @param allowedTools - Allowed tool names or filter object.
-   * @param authorization - OAuth access token for the MCP server/connector.
+   * @param authorization - OAuth access token for the MCP server or connector (including tunneled servers).
    * @param connectorId - Identifier for a service connector.
    * @param headers - Optional headers to include in MCP requests.
-   * // param requireApproval - Approval policy ('always'|'never'|filter object). (Removed - always 'never')
+   * @param requireApproval - Approval policy ('always'|'never'|filter object).
    * @param serverDescription - Optional description of the server.
    * @param serverUrl - URL for the MCP server.
+   * @param tunnelId - Identifier for an OpenAI Secure MCP tunnel.
    */
   mcp,
 

package/src/realtime/openai-realtime-event-mapper.ts

@@ -2,13 +2,48 @@ import type {
   Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
+  Experimental_RealtimeModelV4Usage as RealtimeModelV4Usage,
 } from '@ai-sdk/provider';
 
+type OpenAIRealtimeSessionOptions = {
+  intent?: RealtimeModelV4SessionIntent;
+};
+
+type OpenAIRealtimeTokenDetails = {
+  audio_tokens?: number;
+  text_tokens?: number;
+};
+
+type OpenAIRealtimeInputTokenDetails = OpenAIRealtimeTokenDetails & {
+  cached_tokens?: number;
+  cached_tokens_details?: OpenAIRealtimeTokenDetails;
+};
+
+type OpenAIRealtimeResponseUsage = {
+  input_tokens?: number;
+  output_tokens?: number;
+  input_token_details?: OpenAIRealtimeInputTokenDetails;
+  output_token_details?: OpenAIRealtimeTokenDetails;
+};
+
+type OpenAIRealtimeTranscriptionUsage = {
+  input_tokens?: number;
+  type?: string;
+  seconds?: number;
+  output_tokens?: number;
+  input_token_details?: OpenAIRealtimeTokenDetails;
+};
+
 type OpenAIRealtimeWireEvent = {
   type: string;
   session?: { id?: string };
   item?: { id?: string } & Record<string, unknown>;
-  response?: { id?: string; status?: string };
+  response?: {
+    id?: string;
+    status?: string;
+    usage?: OpenAIRealtimeResponseUsage;
+  };
   error?: {
     message?: string | null;
     code?: string | null;
@@ -25,6 +60,7 @@ type OpenAIRealtimeWireEvent = {
   arguments: string;
   message?: string;
   code?: string;
+  usage?: OpenAIRealtimeTranscriptionUsage;
 };
 
 /**
@@ -80,13 +116,16 @@ export function parseOpenAIRealtimeServerEvent(
         raw,
       };
 
-    case 'conversation.item.input_audio_transcription.completed':
+    case 'conversation.item.input_audio_transcription.completed': {
+      const usage = mapTranscriptionUsage(event.usage);
       return {
         type: 'input-transcription-completed',
         itemId: event.item_id,
         transcript: event.transcript ?? '',
+        ...(usage != null ? { usage } : {}),
         raw,
       };
+    }
 
     // ── Response lifecycle ──────────────────────────────────────────
     case 'response.created':
@@ -96,13 +135,16 @@ export function parseOpenAIRealtimeServerEvent(
         raw,
       };
 
-    case 'response.done':
+    case 'response.done': {
+      const usage = mapResponseUsage(event.response?.usage);
       return {
         type: 'response-done',
         responseId: event.response?.id ?? event.response_id,
         status: event.response?.status ?? 'completed',
+        ...(usage != null ? { usage } : {}),
         raw,
       };
+    }
 
     // ── Output item lifecycle ───────────────────────────────────────
     case 'response.output_item.added':
@@ -231,18 +273,86 @@ export function parseOpenAIRealtimeServerEvent(
   }
 }
 
+/**
+ * Maps OpenAI realtime `response.done` usage to normalized usage.
+ *
+ * Input buckets are reported gross (cache-inclusive) with the cached portion
+ * surfaced separately, so consumers can apply their own cached-vs-uncached
+ * billing split without re-reading `raw`.
+ */
+function mapResponseUsage(
+  usage: OpenAIRealtimeResponseUsage | undefined,
+): RealtimeModelV4Usage | undefined {
+  if (usage == null) return undefined;
+
+  const input = usage.input_token_details;
+  const output = usage.output_token_details;
+  const cached = input?.cached_tokens_details;
+
+  return compactUsage({
+    inputTokens: usage.input_tokens,
+    outputTokens: usage.output_tokens,
+    cachedInputTokens: input?.cached_tokens,
+    inputAudioTokens: input?.audio_tokens,
+    inputTextTokens: input?.text_tokens,
+    outputAudioTokens: output?.audio_tokens,
+    outputTextTokens: output?.text_tokens,
+    cachedInputAudioTokens: cached?.audio_tokens,
+    cachedInputTextTokens: cached?.text_tokens,
+  });
+}
+
+/**
+ * Maps OpenAI realtime transcription-completed usage to normalized usage.
+ * Duration-billed transcription reports `seconds`; token-billed reports tokens.
+ */
+function mapTranscriptionUsage(
+  usage: OpenAIRealtimeTranscriptionUsage | undefined,
+): RealtimeModelV4Usage | undefined {
+  if (usage == null) return undefined;
+
+  if (usage.type === 'duration') {
+    return usage.seconds != null ? { seconds: usage.seconds } : undefined;
+  }
+
+  const input = usage.input_token_details;
+  return compactUsage({
+    inputTokens: usage.input_tokens,
+    outputTokens: usage.output_tokens,
+    inputAudioTokens: input?.audio_tokens,
+    inputTextTokens: input?.text_tokens,
+    outputTextTokens: usage.output_tokens,
+  });
+}
+
+/** Drop `undefined` fields; return `undefined` when nothing is observable. */
+function compactUsage(
+  usage: RealtimeModelV4Usage,
+): RealtimeModelV4Usage | undefined {
+  const result: RealtimeModelV4Usage = {};
+  for (const [key, value] of Object.entries(usage)) {
+    if (value != null) {
+      result[key as keyof RealtimeModelV4Usage] = value;
+    }
+  }
+  return Object.keys(result).length > 0 ? result : undefined;
+}
+
 /**
  * Serializes a normalized client event into OpenAI's Realtime API format.
  */
 export function serializeOpenAIRealtimeClientEvent(
   event: RealtimeModelV4ClientEvent,
   modelId: string,
+  options?: OpenAIRealtimeSessionOptions,
 ): unknown {
+  assertSupportedOpenAIIntent(options?.intent);
+
   switch (event.type) {
     case 'session-update':
       return {
         type: 'session.update',
-        session: buildOpenAISessionConfig(event.config, modelId),
+        session: buildOpenAISessionConfig(event.config, modelId, options),
         ...(event.eventId != null ? { event_id: event.eventId } : {}),
       };
 
@@ -332,23 +442,43 @@ export function serializeOpenAIRealtimeClientEvent(
 export function buildOpenAISessionConfig(
   config: RealtimeModelV4SessionConfig,
   modelId: string,
+  options?: OpenAIRealtimeSessionOptions,
 ): Record<string, unknown> {
+  const intent = options?.intent ?? 'conversation';
+  assertSupportedOpenAIIntent(intent);
+  if (intent === 'transcription') {
+    for (const key of [
+      'instructions',
+      'voice',
+      'outputModalities',
+      'outputAudioFormat',
+      'outputAudioTranscription',
+      'tools',
+    ] as const) {
+      if (config[key] != null)
+        throw new Error(
+          `OpenAI realtime transcription sessions do not support '${key}'.`,
+        );
+    }
+  }
+
   const session: Record<string, unknown> = {
-    type: 'realtime',
-    model: modelId,
+    type: intent === 'transcription' ? 'transcription' : 'realtime',
+    ...(intent === 'conversation' ? { model: modelId } : {}),
   };
 
-  if (config.instructions != null) {
+  if (intent === 'conversation' && config.instructions != null) {
     session.instructions = config.instructions;
   }
 
-  if (config.outputModalities != null) {
+  if (intent === 'conversation' && config.outputModalities != null) {
     session.output_modalities = config.outputModalities;
   }
 
   const audio: Record<string, unknown> = {};
 
   if (
+    intent === 'transcription' ||
     config.inputAudioFormat != null ||
     config.inputAudioTranscription != null ||
     config.turnDetection != null
@@ -387,7 +517,17 @@ export function buildOpenAISessionConfig(
       }
     }
 
-    if (config.inputAudioTranscription != null) {
+    if (intent === 'transcription') {
+      input.transcription = {
+        model: config.inputAudioTranscription?.model ?? modelId,
+        ...(config.inputAudioTranscription?.language != null
+          ? { language: config.inputAudioTranscription.language }
+          : {}),
+        ...(config.inputAudioTranscription?.prompt != null
+          ? { prompt: config.inputAudioTranscription.prompt }
+          : {}),
+      };
+    } else if (config.inputAudioTranscription != null) {
       input.transcription = {
         model: config.inputAudioTranscription.model ?? 'gpt-realtime-whisper',
         ...(config.inputAudioTranscription.language != null
@@ -402,7 +542,10 @@ export function buildOpenAISessionConfig(
     audio.input = input;
   }
 
-  if (config.outputAudioFormat != null || config.voice != null) {
+  if (
+    intent === 'conversation' &&
+    (config.outputAudioFormat != null || config.voice != null)
+  ) {
     const output: Record<string, unknown> = {};
 
     if (config.outputAudioFormat != null) {
@@ -425,7 +568,11 @@ export function buildOpenAISessionConfig(
     session.audio = audio;
   }
 
-  if (config.tools != null && config.tools.length > 0) {
+  if (
+    intent === 'conversation' &&
+    config.tools != null &&
+    config.tools.length > 0
+  ) {
     session.tools = config.tools.map(tool => ({
       type: tool.type,
       name: tool.name,
@@ -441,3 +588,17 @@ export function buildOpenAISessionConfig(
 
   return session;
 }
+
+function assertSupportedOpenAIIntent(
+  intent: RealtimeModelV4SessionIntent | undefined,
+): void {
+  if (
+    intent != null &&
+    intent !== 'conversation' &&
+    intent !== 'transcription'
+  ) {
+    throw new Error(
+      `OpenAI realtime session intent '${intent}' is not supported.`,
+    );
+  }
+}

package/src/realtime/openai-realtime-model.ts

@@ -1,6 +1,8 @@
 import {
   type Resolvable,
   resolve,
+  resolveWith,
+  removeUndefinedEntries,
   type FetchFunction,
 } from '@ai-sdk/provider-utils';
 import type {
@@ -8,8 +10,10 @@ import type {
   Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
   Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
   Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerConnection as RealtimeModelV4ServerConnection,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
 } from '@ai-sdk/provider';
 import {
   buildOpenAISessionConfig,
@@ -26,6 +30,11 @@ export type OpenAIRealtimeModelConfig = {
 
 export class OpenAIRealtimeModel implements RealtimeModelV4 {
   readonly specificationVersion = 'v4' as const;
+  readonly capabilities = {
+    conversation: 'turn-based',
+    transports: ['websocket'],
+    connections: ['client-secret-websocket', 'server-websocket'],
+  } as const;
   readonly provider: string;
   readonly modelId: string;
 
@@ -99,17 +108,62 @@ export class OpenAIRealtimeModel implements RealtimeModelV4 {
     };
   }
 
+  getServerWebSocketConfig(options?: {
+    intent?: RealtimeModelV4SessionIntent;
+  }):
+    | RealtimeModelV4ServerConnection
+    | PromiseLike<RealtimeModelV4ServerConnection> {
+    return resolveWith(this.config.headers, headers => ({
+      url: buildServerUrl(
+        this.config.baseURL,
+        this.modelId,
+        options?.intent ?? 'conversation',
+      ),
+      headers: removeUndefinedEntries(headers),
+    }));
+  }
+
   parseServerEvent(raw: unknown): RealtimeModelV4ServerEvent {
     return parseOpenAIRealtimeServerEvent(raw);
   }
 
-  serializeClientEvent(event: RealtimeModelV4ClientEvent): unknown {
-    return serializeOpenAIRealtimeClientEvent(event, this.modelId);
+  serializeClientEvent(
+    event: RealtimeModelV4ClientEvent,
+    options?: { intent?: RealtimeModelV4SessionIntent },
+  ): unknown {
+    return serializeOpenAIRealtimeClientEvent(event, this.modelId, options);
   }
 
   buildSessionConfig(
     config: RealtimeModelV4SessionConfig,
+    options?: { intent?: RealtimeModelV4SessionIntent },
   ): Record<string, unknown> {
-    return buildOpenAISessionConfig(config, this.modelId);
+    return buildOpenAISessionConfig(config, this.modelId, options);
+  }
+}
+
+/**
+ * Builds the upstream realtime WebSocket URL for a server-initiated connection.
+ * Transcription keeps the model in `session.update`; a `?model=` here would
+ * open a voice session instead.
+ */
+function buildServerUrl(
+  baseURL: string,
+  modelId: string,
+  intent: RealtimeModelV4SessionIntent,
+): string {
+  const url = new URL(baseURL);
+  const protocol = url.protocol === 'http:' ? 'ws:' : 'wss:';
+  const base = `${protocol}//${url.host}${url.pathname.replace(/\/$/, '')}`;
+
+  switch (intent) {
+    case 'transcription':
+      return `${base}/realtime?intent=transcription`;
+    case 'conversation':
+      return `${base}/realtime?model=${encodeURIComponent(modelId)}`;
+    default:
+      throw new Error(
+        `OpenAI realtime session intent '${intent}' is not supported.`,
+      );
   }
 }

package/src/responses/openai-responses-api.ts

@@ -647,7 +647,24 @@ export type OpenAIResponsesTool =
     }
   | {
       type: 'code_interpreter';
-      container: string | { type: 'auto'; file_ids: string[] | undefined };
+      container:
+        | string
+        | {
+            type: 'auto';
+            file_ids: string[] | undefined;
+            memory_limit?: '1g' | '4g' | '16g' | '64g';
+            network_policy?:
+              | { type: 'disabled' }
+              | {
+                  type: 'allowlist';
+                  allowed_domains: string[];
+                  domain_secrets?: Array<{
+                    domain: string;
+                    name: string;
+                    value: string;
+                  }>;
+                };
+          };
     }
   | {
       type: 'file_search';
@@ -712,6 +729,7 @@ export type OpenAIResponsesTool =
         | undefined;
       server_description: string | undefined;
       server_url: string | undefined;
+      tunnel_id: string | undefined;
     }
   | {
       type: 'custom';

package/src/responses/openai-responses-prepare-tools.ts

@@ -333,7 +333,23 @@ export async function prepareResponsesTools({
                   ? { type: 'auto', file_ids: undefined }
                   : typeof args.container === 'string'
                     ? args.container
-                    : { type: 'auto', file_ids: args.container.fileIds },
+                    : {
+                        type: 'auto',
+                        file_ids: args.container.fileIds,
+                        memory_limit: args.container.memoryLimit,
+                        network_policy:
+                          args.container.networkPolicy == null
+                            ? undefined
+                            : args.container.networkPolicy.type === 'disabled'
+                              ? { type: 'disabled' }
+                              : {
+                                  type: 'allowlist',
+                                  allowed_domains:
+                                    args.container.networkPolicy.allowedDomains,
+                                  domain_secrets:
+                                    args.container.networkPolicy.domainSecrets,
+                                },
+                      },
             });
             break;
           }
@@ -407,6 +423,7 @@ export async function prepareResponsesTools({
               require_approval: requireApprovalParam ?? 'never',
               server_description: args.serverDescription,
               server_url: args.serverUrl,
+              tunnel_id: args.tunnelId,
             });
 
             break;

package/src/tool/code-interpreter.ts

@@ -37,6 +37,25 @@ export const codeInterpreterArgsSchema = lazySchema(() =>
           z.string(),
           z.object({
             fileIds: z.array(z.string()).optional(),
+            memoryLimit: z.enum(['1g', '4g', '16g', '64g']).optional(),
+            networkPolicy: z
+              .discriminatedUnion('type', [
+                z.object({ type: z.literal('disabled') }),
+                z.object({
+                  type: z.literal('allowlist'),
+                  allowedDomains: z.array(z.string()),
+                  domainSecrets: z
+                    .array(
+                      z.object({
+                        domain: z.string(),
+                        name: z.string(),
+                        value: z.string(),
+                      }),
+                    )
+                    .optional(),
+                }),
+              ])
+              .optional(),
           }),
         ])
         .optional(),
@@ -48,9 +67,26 @@ type CodeInterpreterArgs = {
   /**
    * The code interpreter container.
    * Can be a container ID
-   * or an object that specifies uploaded file IDs to make available to your code.
+   * or an object that configures uploaded file IDs, the memory limit,
+   * and the network policy for a new container.
    */
-  container?: string | { fileIds?: string[] };
+  container?:
+    | string
+    | {
+        fileIds?: string[];
+        memoryLimit?: '1g' | '4g' | '16g' | '64g';
+        networkPolicy?:
+          | { type: 'disabled' }
+          | {
+              type: 'allowlist';
+              allowedDomains: string[];
+              domainSecrets?: Array<{
+                domain: string;
+                name: string;
+                value: string;
+              }>;
+            };
+      };
 };
 
 export const codeInterpreterToolFactory = createProviderExecutedToolFactory<

package/src/tool/mcp.ts

@@ -51,10 +51,16 @@ export const mcpArgsSchema = lazySchema(() =>
           .optional(),
         serverDescription: z.string().optional(),
         serverUrl: z.string().optional(),
+        tunnelId: z.string().optional(),
       })
       .refine(
-        v => v.serverUrl != null || v.connectorId != null,
-        'One of serverUrl or connectorId must be provided.',
+        v => v.serverUrl != null || v.connectorId != null || v.tunnelId != null,
+        'One of serverUrl, connectorId, or tunnelId must be provided.',
+      )
+      .refine(
+        v =>
+          v.tunnelId == null || (v.serverUrl == null && v.connectorId == null),
+        'tunnelId cannot be combined with serverUrl or connectorId.',
       ),
   ),
 );
@@ -74,7 +80,7 @@ export const mcpOutputSchema = lazySchema(() =>
   ),
 );
 
-type McpArgs = {
+type McpCommonArgs = {
   /** A label for this MCP server, used to identify it in tool calls. */
   serverLabel: string;
   /** List of allowed tool names or a filter object. */
@@ -84,10 +90,8 @@ type McpArgs = {
         readOnly?: boolean;
         toolNames?: string[];
       };
-  /** OAuth access token usable with the remote MCP server or connector. */
+  /** OAuth access token usable with the remote MCP server or connector (including tunneled servers). */
   authorization?: string;
-  /** Identifier for a service connector. */
-  connectorId?: string;
   /** Optional HTTP headers to send to the MCP server. */
   headers?: Record<string, string>;
   /**
@@ -103,10 +107,25 @@ type McpArgs = {
       };
   /** Optional description of the MCP server. */
   serverDescription?: string;
-  /** URL for the MCP server. One of serverUrl or connectorId must be provided. */
-  serverUrl?: string;
 };
 
+type McpArgs = McpCommonArgs &
+  (
+    | {
+        /** URL for the MCP server. */
+        serverUrl?: string;
+        /** Identifier for a service connector. */
+        connectorId?: string;
+        tunnelId?: never;
+      }
+    | {
+        /** Identifier for an OpenAI Secure MCP tunnel. */
+        tunnelId: string;
+        serverUrl?: never;
+        connectorId?: never;
+      }
+  );
+
 export const mcpToolFactory = createProviderExecutedToolFactory<
   {},
   {