@ai-sdk/openai 4.0.0-canary.68 → 4.0.0-canary.69

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @ai-sdk/openai
 
+## 4.0.0-canary.69
+
+### Patch Changes
+
+- ce769dd: feat(provider): add experimental Realtime API support for voice conversations
+
+  Adds first-class support for realtime (speech-to-speech) APIs:
+
+  - `Experimental_RealtimeModelV4` spec in `@ai-sdk/provider` with normalized event types and factory
+  - OpenAI, Google, and xAI realtime provider implementations
+  - `openai.experimental_realtime()` / `google.experimental_realtime()` / `xai.experimental_realtime()` work in both server and browser
+  - `.getToken()` static method on each provider for server-side ephemeral token creation
+  - `experimental_getRealtimeToolDefinitions` helper for provider session tool definitions
+  - `experimental_useRealtime` hook in `@ai-sdk/react` returning `UIMessage[]` (aligned with `useChat`), with `onToolCall` and `addToolOutput` for client-driven tool execution
+  - `inputAudioTranscription` session config for showing transcribed user audio messages when supported by the provider
+
+- Updated dependencies [ce769dd]
+  - @ai-sdk/provider@4.0.0-canary.18
+  - @ai-sdk/provider-utils@5.0.0-canary.46
+
 ## 4.0.0-canary.68
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import * as _ai_sdk_provider from '@ai-sdk/provider';
-import { JSONValue, ProviderV4, LanguageModelV4, EmbeddingModelV4, ImageModelV4, TranscriptionModelV4, SpeechModelV4, FilesV4, SkillsV4 } from '@ai-sdk/provider';
+import { JSONValue, ProviderV4, LanguageModelV4, EmbeddingModelV4, ImageModelV4, TranscriptionModelV4, SpeechModelV4, Experimental_RealtimeFactoryV4, FilesV4, SkillsV4, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientSecretOptions, Experimental_RealtimeModelV4ClientSecretResult, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4SessionConfig } from '@ai-sdk/provider';
 import * as _ai_sdk_provider_utils from '@ai-sdk/provider-utils';
 import { InferSchema, FetchFunction } from '@ai-sdk/provider-utils';
 
@@ -1150,6 +1150,11 @@ interface OpenAIProvider extends ProviderV4 {
      * Creates a model for speech generation.
      */
     speech(modelId: OpenAISpeechModelId): SpeechModelV4;
+    /**
+     * Creates an experimental realtime model for bidirectional audio/text
+     * communication over WebSocket.
+     */
+    experimental_realtime: Experimental_RealtimeFactoryV4;
     /**
      * Returns a FilesV4 interface for uploading files to OpenAI.
      */
@@ -1203,6 +1208,31 @@ declare function createOpenAI(options?: OpenAIProviderSettings): OpenAIProvider;
  */
 declare const openai: OpenAIProvider;
 
+type OpenAIRealtimeModelConfig = {
+    provider: string;
+    baseURL: string;
+    headers: () => Record<string, string | undefined>;
+    fetch?: FetchFunction;
+};
+declare class OpenAIRealtimeModel implements Experimental_RealtimeModelV4 {
+    readonly specificationVersion: "v4";
+    readonly provider: string;
+    readonly modelId: string;
+    private readonly config;
+    constructor(modelId: string, config: OpenAIRealtimeModelConfig);
+    doCreateClientSecret(options: Experimental_RealtimeModelV4ClientSecretOptions): Promise<Experimental_RealtimeModelV4ClientSecretResult>;
+    getWebSocketConfig(options: {
+        token: string;
+        url: string;
+    }): {
+        url: string;
+        protocols?: string[];
+    };
+    parseServerEvent(raw: unknown): Experimental_RealtimeModelV4ServerEvent;
+    serializeClientEvent(event: Experimental_RealtimeModelV4ClientEvent): unknown;
+    buildSessionConfig(config: Experimental_RealtimeModelV4SessionConfig): Record<string, unknown>;
+}
+
 declare const openaiFilesOptionsSchema: _ai_sdk_provider_utils.LazySchema<{
     purpose?: string | undefined;
     expiresAfter?: number | undefined;
@@ -1263,4 +1293,4 @@ type OpenaiResponsesSourceDocumentProviderMetadata = {
 
 declare const VERSION: string;
 
-export { type OpenAILanguageModelChatOptions as OpenAIChatLanguageModelOptions, type OpenAIEmbeddingModelOptions, type OpenAIFilesOptions, type OpenAIImageModelEditOptions, type OpenAIImageModelGenerationOptions, type OpenAIImageModelOptions, type OpenAILanguageModelChatOptions, type OpenAILanguageModelCompletionOptions, type OpenAILanguageModelResponsesOptions, type OpenAIProvider, type OpenAIProviderSettings, type OpenAILanguageModelResponsesOptions as OpenAIResponsesProviderOptions, type OpenAISpeechModelOptions, type OpenAITranscriptionModelOptions, type OpenaiResponsesCompactionProviderMetadata, type OpenaiResponsesProviderMetadata, type OpenaiResponsesReasoningProviderMetadata, type OpenaiResponsesSourceDocumentProviderMetadata, type OpenaiResponsesTextProviderMetadata, VERSION, createOpenAI, openai };
+export { OpenAIRealtimeModel as Experimental_OpenAIRealtimeModel, type OpenAIRealtimeModelConfig as Experimental_OpenAIRealtimeModelConfig, type OpenAILanguageModelChatOptions as OpenAIChatLanguageModelOptions, type OpenAIEmbeddingModelOptions, type OpenAIFilesOptions, type OpenAIImageModelEditOptions, type OpenAIImageModelGenerationOptions, type OpenAIImageModelOptions, type OpenAILanguageModelChatOptions, type OpenAILanguageModelCompletionOptions, type OpenAILanguageModelResponsesOptions, type OpenAIProvider, type OpenAIProviderSettings, type OpenAILanguageModelResponsesOptions as OpenAIResponsesProviderOptions, type OpenAISpeechModelOptions, type OpenAITranscriptionModelOptions, type OpenaiResponsesCompactionProviderMetadata, type OpenaiResponsesProviderMetadata, type OpenaiResponsesReasoningProviderMetadata, type OpenaiResponsesSourceDocumentProviderMetadata, type OpenaiResponsesTextProviderMetadata, VERSION, createOpenAI, openai };

package/dist/index.js

@@ -2869,6 +2869,395 @@ var openaiTools = {
   toolSearch
 };
 
+// src/realtime/openai-realtime-event-mapper.ts
+function parseOpenAIRealtimeServerEvent(raw) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s;
+  const event = raw;
+  const type = event.type;
+  switch (type) {
+    // ── Session lifecycle ──────────────────────────────────────────
+    case "session.created":
+      return {
+        type: "session-created",
+        sessionId: (_a = event.session) == null ? void 0 : _a.id,
+        raw
+      };
+    case "session.updated":
+      return { type: "session-updated", raw };
+    // ── Input audio buffer ─────────────────────────────────────────
+    case "input_audio_buffer.speech_started":
+      return {
+        type: "speech-started",
+        itemId: event.item_id,
+        raw
+      };
+    case "input_audio_buffer.speech_stopped":
+      return {
+        type: "speech-stopped",
+        itemId: event.item_id,
+        raw
+      };
+    case "input_audio_buffer.committed":
+      return {
+        type: "audio-committed",
+        itemId: event.item_id,
+        previousItemId: event.previous_item_id,
+        raw
+      };
+    // ── Conversation items ─────────────────────────────────────────
+    case "conversation.item.added":
+      return {
+        type: "conversation-item-added",
+        itemId: (_c = (_b = event.item) == null ? void 0 : _b.id) != null ? _c : event.item_id,
+        item: event.item,
+        raw
+      };
+    case "conversation.item.input_audio_transcription.completed":
+      return {
+        type: "input-transcription-completed",
+        itemId: event.item_id,
+        transcript: (_d = event.transcript) != null ? _d : "",
+        raw
+      };
+    // ── Response lifecycle ──────────────────────────────────────────
+    case "response.created":
+      return {
+        type: "response-created",
+        responseId: (_f = (_e = event.response) == null ? void 0 : _e.id) != null ? _f : event.response_id,
+        raw
+      };
+    case "response.done":
+      return {
+        type: "response-done",
+        responseId: (_h = (_g = event.response) == null ? void 0 : _g.id) != null ? _h : event.response_id,
+        status: (_j = (_i = event.response) == null ? void 0 : _i.status) != null ? _j : "completed",
+        raw
+      };
+    // ── Output item lifecycle ───────────────────────────────────────
+    case "response.output_item.added":
+      return {
+        type: "output-item-added",
+        responseId: event.response_id,
+        itemId: (_l = (_k = event.item) == null ? void 0 : _k.id) != null ? _l : event.item_id,
+        raw
+      };
+    case "response.output_item.done":
+      return {
+        type: "output-item-done",
+        responseId: event.response_id,
+        itemId: (_n = (_m = event.item) == null ? void 0 : _m.id) != null ? _n : event.item_id,
+        raw
+      };
+    case "response.content_part.added":
+      return {
+        type: "content-part-added",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw
+      };
+    case "response.content_part.done":
+      return {
+        type: "content-part-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw
+      };
+    // ── Audio output ────────────────────────────────────────────────
+    case "response.output_audio.delta":
+      return {
+        type: "audio-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.output_audio.done":
+      return {
+        type: "audio-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        raw
+      };
+    // ── Audio transcript output ─────────────────────────────────────
+    case "response.output_audio_transcript.delta":
+      return {
+        type: "audio-transcript-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.output_audio_transcript.done":
+      return {
+        type: "audio-transcript-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        transcript: event.transcript,
+        raw
+      };
+    // ── Text output ─────────────────────────────────────────────────
+    case "response.output_text.delta":
+      return {
+        type: "text-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.output_text.done":
+      return {
+        type: "text-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        text: event.text,
+        raw
+      };
+    // ── Function calling ────────────────────────────────────────────
+    case "response.function_call_arguments.delta":
+      return {
+        type: "function-call-arguments-delta",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        callId: event.call_id,
+        delta: event.delta,
+        raw
+      };
+    case "response.function_call_arguments.done":
+      return {
+        type: "function-call-arguments-done",
+        responseId: event.response_id,
+        itemId: event.item_id,
+        callId: event.call_id,
+        name: event.name,
+        arguments: event.arguments,
+        raw
+      };
+    // ── Error ───────────────────────────────────────────────────────
+    case "error":
+      return {
+        type: "error",
+        message: (_q = (_p = (_o = event.error) == null ? void 0 : _o.message) != null ? _p : event.message) != null ? _q : "Unknown error",
+        code: (_s = (_r = event.error) == null ? void 0 : _r.code) != null ? _s : event.code,
+        raw
+      };
+    // ── Pass-through ────────────────────────────────────────────────
+    default:
+      return { type: "custom", rawType: type, raw };
+  }
+}
+function serializeOpenAIRealtimeClientEvent(event, modelId) {
+  switch (event.type) {
+    case "session-update":
+      return {
+        type: "session.update",
+        session: buildOpenAISessionConfig(event.config, modelId)
+      };
+    case "input-audio-append":
+      return {
+        type: "input_audio_buffer.append",
+        audio: event.audio
+      };
+    case "input-audio-commit":
+      return { type: "input_audio_buffer.commit" };
+    case "input-audio-clear":
+      return { type: "input_audio_buffer.clear" };
+    case "conversation-item-create": {
+      const item = event.item;
+      switch (item.type) {
+        case "text-message":
+          return {
+            type: "conversation.item.create",
+            item: {
+              type: "message",
+              role: item.role,
+              content: [{ type: "input_text", text: item.text }]
+            }
+          };
+        case "audio-message":
+          return {
+            type: "conversation.item.create",
+            item: {
+              type: "message",
+              role: item.role,
+              content: [{ type: "input_audio", audio: item.audio }]
+            }
+          };
+        case "function-call-output":
+          return {
+            type: "conversation.item.create",
+            item: {
+              type: "function_call_output",
+              call_id: item.callId,
+              output: item.output
+            }
+          };
+      }
+      break;
+    }
+    case "conversation-item-truncate":
+      return {
+        type: "conversation.item.truncate",
+        item_id: event.itemId,
+        content_index: event.contentIndex,
+        audio_end_ms: event.audioEndMs
+      };
+    case "response-create":
+      return {
+        type: "response.create",
+        ...event.options != null ? {
+          response: {
+            ...event.options.modalities != null ? { output_modalities: event.options.modalities } : {},
+            ...event.options.instructions != null ? { instructions: event.options.instructions } : {},
+            ...event.options.metadata != null ? { metadata: event.options.metadata } : {}
+          }
+        } : {}
+      };
+    case "response-cancel":
+      return { type: "response.cancel" };
+  }
+}
+function buildOpenAISessionConfig(config, modelId) {
+  var _a;
+  const session = {
+    type: "realtime",
+    model: modelId
+  };
+  if (config.instructions != null) {
+    session.instructions = config.instructions;
+  }
+  if (config.outputModalities != null) {
+    session.output_modalities = config.outputModalities;
+  }
+  const audio = {};
+  if (config.inputAudioFormat != null || config.inputAudioTranscription != null || config.turnDetection != null) {
+    const input = {};
+    if (config.inputAudioFormat != null) {
+      input.format = {
+        type: config.inputAudioFormat.type,
+        ...config.inputAudioFormat.rate != null ? { rate: config.inputAudioFormat.rate } : {}
+      };
+    }
+    if (config.turnDetection != null) {
+      if (config.turnDetection.type === "disabled") {
+        input.turn_detection = null;
+      } else {
+        const td = {
+          type: config.turnDetection.type === "server-vad" ? "server_vad" : "semantic_vad"
+        };
+        if (config.turnDetection.threshold != null) {
+          td.threshold = config.turnDetection.threshold;
+        }
+        if (config.turnDetection.silenceDurationMs != null) {
+          td.silence_duration_ms = config.turnDetection.silenceDurationMs;
+        }
+        if (config.turnDetection.prefixPaddingMs != null) {
+          td.prefix_padding_ms = config.turnDetection.prefixPaddingMs;
+        }
+        input.turn_detection = td;
+      }
+    }
+    if (config.inputAudioTranscription != null) {
+      input.transcription = {
+        model: (_a = config.inputAudioTranscription.model) != null ? _a : "gpt-realtime-whisper",
+        ...config.inputAudioTranscription.language != null ? { language: config.inputAudioTranscription.language } : {},
+        ...config.inputAudioTranscription.prompt != null ? { prompt: config.inputAudioTranscription.prompt } : {}
+      };
+    }
+    audio.input = input;
+  }
+  if (config.outputAudioFormat != null || config.voice != null) {
+    const output = {};
+    if (config.outputAudioFormat != null) {
+      output.format = {
+        type: config.outputAudioFormat.type,
+        ...config.outputAudioFormat.rate != null ? { rate: config.outputAudioFormat.rate } : {}
+      };
+    }
+    if (config.voice != null) {
+      output.voice = config.voice;
+    }
+    audio.output = output;
+  }
+  if (Object.keys(audio).length > 0) {
+    session.audio = audio;
+  }
+  if (config.tools != null && config.tools.length > 0) {
+    session.tools = config.tools.map((tool) => ({
+      type: tool.type,
+      name: tool.name,
+      description: tool.description,
+      parameters: tool.parameters
+    }));
+    session.tool_choice = "auto";
+  }
+  if (config.providerOptions != null) {
+    Object.assign(session, config.providerOptions);
+  }
+  return session;
+}
+
+// src/realtime/openai-realtime-model.ts
+var OpenAIRealtimeModel = class {
+  constructor(modelId, config) {
+    this.specificationVersion = "v4";
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+  async doCreateClientSecret(options) {
+    var _a;
+    const fetchFn = (_a = this.config.fetch) != null ? _a : fetch;
+    const url = `${this.config.baseURL}/realtime/client_secrets`;
+    const session = options.sessionConfig != null ? buildOpenAISessionConfig(options.sessionConfig, this.modelId) : { type: "realtime", model: this.modelId };
+    const response = await fetchFn(url, {
+      method: "POST",
+      headers: {
+        ...this.config.headers(),
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        session,
+        ...options.expiresAfterSeconds != null ? {
+          // `anchor` is required by the client secrets endpoint; without it
+          // the request fails with "Missing required parameter:
+          // 'expires_after.anchor'".
+          expires_after: {
+            anchor: "created_at",
+            seconds: options.expiresAfterSeconds
+          }
+        } : {}
+      })
+    });
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `OpenAI realtime client secret request failed: ${response.status} ${text}`
+      );
+    }
+    const data = await response.json();
+    return {
+      token: data.value,
+      url: `wss://${new URL(this.config.baseURL).host}/v1/realtime?model=${encodeURIComponent(this.modelId)}`,
+      expiresAt: data.expires_at
+    };
+  }
+  getWebSocketConfig(options) {
+    return {
+      url: options.url,
+      protocols: ["realtime", `openai-insecure-api-key.${options.token}`]
+    };
+  }
+  parseServerEvent(raw) {
+    return parseOpenAIRealtimeServerEvent(raw);
+  }
+  serializeClientEvent(event) {
+    return serializeOpenAIRealtimeClientEvent(event, this.modelId);
+  }
+  buildSessionConfig(config) {
+    return buildOpenAISessionConfig(config, this.modelId);
+  }
+};
+
 // src/responses/openai-responses-language-model.ts
 import {
   APICallError
@@ -7322,7 +7711,7 @@ var OpenAISkills = class {
 };
 
 // src/version.ts
-var VERSION = true ? "4.0.0-canary.68" : "0.0.0-test";
+var VERSION = true ? "4.0.0-canary.69" : "0.0.0-test";
 
 // src/openai-provider.ts
 function createOpenAI(options = {}) {
@@ -7413,6 +7802,29 @@ function createOpenAI(options = {}) {
       fileIdPrefixes: ["file-"]
     });
   };
+  const createRealtimeModel = (modelId) => new OpenAIRealtimeModel(modelId, {
+    provider: `${providerName}.realtime`,
+    baseURL,
+    headers: getHeaders,
+    fetch: options.fetch
+  });
+  const experimentalRealtimeFactory = Object.assign(
+    (modelId) => createRealtimeModel(modelId),
+    {
+      getToken: async (tokenOptions) => {
+        const model = createRealtimeModel(tokenOptions.model);
+        const secret = await model.doCreateClientSecret({
+          sessionConfig: tokenOptions.sessionConfig,
+          expiresAfterSeconds: tokenOptions.expiresAfterSeconds
+        });
+        return {
+          token: secret.token,
+          url: secret.url,
+          expiresAt: secret.expiresAt
+        };
+      }
+    }
+  );
   const provider = function(modelId) {
     return createLanguageModel(modelId);
   };
@@ -7433,11 +7845,13 @@ function createOpenAI(options = {}) {
   provider.speechModel = createSpeechModel;
   provider.files = createFiles;
   provider.skills = createSkills;
+  provider.experimental_realtime = experimentalRealtimeFactory;
   provider.tools = openaiTools;
   return provider;
 }
 var openai = createOpenAI();
 export {
+  OpenAIRealtimeModel as Experimental_OpenAIRealtimeModel,
   VERSION,
   createOpenAI,
   openai