@ai-sdk/openai 4.0.6 → 4.0.8

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @ai-sdk/openai
 
+## 4.0.8
+
+### Patch Changes
+
+- b51ed36: Send inline image file parts in OpenAI chat requests as data URLs instead of bare base64 strings.
+
+## 4.0.7
+
+### Patch Changes
+
+- 5c5c0f5: Add experimental streaming transcription support for transcription models, including OpenAI `gpt-realtime-whisper` and xAI WebSocket STT.
+- Updated dependencies [5c5c0f5]
+  - @ai-sdk/provider@4.0.2
+  - @ai-sdk/provider-utils@5.0.5
+
 ## 4.0.6
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 import * as _ai_sdk_provider from '@ai-sdk/provider';
 import { JSONValue, ProviderV4, LanguageModelV4, EmbeddingModelV4, ImageModelV4, TranscriptionModelV4, SpeechModelV4, Experimental_RealtimeFactoryV4, FilesV4, SkillsV4, Experimental_RealtimeModelV4, Experimental_RealtimeModelV4ClientSecretOptions, Experimental_RealtimeModelV4ClientSecretResult, Experimental_RealtimeModelV4ServerEvent, Experimental_RealtimeModelV4ClientEvent, Experimental_RealtimeModelV4SessionConfig } from '@ai-sdk/provider';
 import * as _ai_sdk_provider_utils from '@ai-sdk/provider-utils';
-import { InferSchema, FetchFunction } from '@ai-sdk/provider-utils';
+import { InferSchema, FetchFunction, WebSocketConstructor } from '@ai-sdk/provider-utils';
 
 type OpenAIChatModelId = 'o1' | 'o1-2024-12-17' | 'o3-mini' | 'o3-mini-2025-01-31' | 'o3' | 'o3-2025-04-16' | 'o4-mini' | 'o4-mini-2025-04-16' | 'gpt-4.1' | 'gpt-4.1-2025-04-14' | 'gpt-4.1-mini' | 'gpt-4.1-mini-2025-04-14' | 'gpt-4.1-nano' | 'gpt-4.1-nano-2025-04-14' | 'gpt-4o' | 'gpt-4o-2024-05-13' | 'gpt-4o-2024-08-06' | 'gpt-4o-2024-11-20' | 'gpt-4o-audio-preview' | 'gpt-4o-audio-preview-2024-12-17' | 'gpt-4o-audio-preview-2025-06-03' | 'gpt-4o-mini' | 'gpt-4o-mini-2024-07-18' | 'gpt-4o-mini-audio-preview' | 'gpt-4o-mini-audio-preview-2024-12-17' | 'gpt-4o-search-preview' | 'gpt-4o-search-preview-2025-03-11' | 'gpt-4o-mini-search-preview' | 'gpt-4o-mini-search-preview-2025-03-11' | 'gpt-3.5-turbo-0125' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-1106' | 'gpt-3.5-turbo-16k' | 'gpt-5' | 'gpt-5-2025-08-07' | 'gpt-5-mini' | 'gpt-5-mini-2025-08-07' | 'gpt-5-nano' | 'gpt-5-nano-2025-08-07' | 'gpt-5-chat-latest' | 'gpt-5.1' | 'gpt-5.1-2025-11-13' | 'gpt-5.1-chat-latest' | 'gpt-5.2' | 'gpt-5.2-2025-12-11' | 'gpt-5.2-chat-latest' | 'gpt-5.2-pro' | 'gpt-5.2-pro-2025-12-11' | 'gpt-5.3-chat-latest' | 'gpt-5.4' | 'gpt-5.4-2026-03-05' | 'gpt-5.4-mini' | 'gpt-5.4-mini-2026-03-17' | 'gpt-5.4-nano' | 'gpt-5.4-nano-2026-03-17' | 'gpt-5.4-pro' | 'gpt-5.4-pro-2026-03-05' | 'gpt-5.5' | 'gpt-5.5-2026-04-23' | (string & {});
 declare const openaiLanguageModelChatOptions: _ai_sdk_provider_utils.LazySchema<{
@@ -1103,13 +1103,17 @@ declare const openaiSpeechModelOptionsSchema: _ai_sdk_provider_utils.LazySchema<
 }>;
 type OpenAISpeechModelOptions = InferSchema<typeof openaiSpeechModelOptionsSchema>;
 
-type OpenAITranscriptionModelId = 'whisper-1' | 'gpt-4o-mini-transcribe' | 'gpt-4o-mini-transcribe-2025-03-20' | 'gpt-4o-mini-transcribe-2025-12-15' | 'gpt-4o-transcribe' | 'gpt-4o-transcribe-diarize' | (string & {});
+type OpenAITranscriptionModelId = 'whisper-1' | 'gpt-4o-mini-transcribe' | 'gpt-4o-mini-transcribe-2025-03-20' | 'gpt-4o-mini-transcribe-2025-12-15' | 'gpt-4o-transcribe' | 'gpt-4o-transcribe-diarize' | 'gpt-realtime-whisper' | (string & {});
 declare const openAITranscriptionModelOptions: _ai_sdk_provider_utils.LazySchema<{
     include?: string[] | undefined;
     language?: string | undefined;
     prompt?: string | undefined;
     temperature?: number | undefined;
     timestampGranularities?: ("word" | "segment")[] | undefined;
+    streaming?: {
+        delay?: "minimal" | "low" | "medium" | "high" | "xhigh" | undefined;
+        include?: string[] | undefined;
+    } | undefined;
 }>;
 type OpenAITranscriptionModelOptions = InferSchema<typeof openAITranscriptionModelOptions>;
 
@@ -1211,6 +1215,11 @@ interface OpenAIProviderSettings {
      * or to provide a custom fetch implementation for e.g. testing.
      */
     fetch?: FetchFunction;
+    /**
+     * Custom WebSocket implementation. This is useful for testing or for
+     * runtimes that need a WebSocket constructor with header support.
+     */
+    webSocket?: WebSocketConstructor;
 }
 /**
  * Create an OpenAI provider instance.

package/dist/index.js

@@ -304,7 +304,7 @@ function convertToOpenAIChatMessages({
                       return {
                         type: "image_url",
                         image_url: {
-                          url: part.data.type === "url" ? part.data.url.toString() : convertToBase64(part.data.data),
+                          url: part.data.type === "url" ? part.data.url.toString() : `data:${resolveFullMediaType({ part })};base64,${convertToBase64(part.data.data)}`,
                           detail: (_b = (_a2 = part.providerOptions) == null ? void 0 : _a2.openai) == null ? void 0 : _b.imageDetail
                         }
                       };
@@ -7645,14 +7645,22 @@ var OpenAISpeechModel = class _OpenAISpeechModel {
 };
 
 // src/transcription/openai-transcription-model.ts
+import {
+  UnsupportedFunctionalityError as UnsupportedFunctionalityError6
+} from "@ai-sdk/provider";
 import {
   combineHeaders as combineHeaders8,
   convertBase64ToUint8Array as convertBase64ToUint8Array2,
+  convertToBase64 as convertToBase643,
   createJsonResponseHandler as createJsonResponseHandler7,
+  getWebSocketConstructor,
   mediaTypeToExtension,
   parseProviderOptions as parseProviderOptions9,
   postFormDataToApi as postFormDataToApi3,
+  readWebSocketMessageText,
+  safeParseJSON,
   serializeModelOptions as serializeModelOptions7,
+  toWebSocketUrl,
   WORKFLOW_DESERIALIZE as WORKFLOW_DESERIALIZE7,
   WORKFLOW_SERIALIZE as WORKFLOW_SERIALIZE7
 } from "@ai-sdk/provider-utils";
@@ -7721,12 +7729,28 @@ var openAITranscriptionModelOptions = lazySchema26(
        * The timestamp granularities to populate for this transcription.
        * @default ['segment']
        */
-      timestampGranularities: z28.array(z28.enum(["word", "segment"])).default(["segment"]).optional()
+      timestampGranularities: z28.array(z28.enum(["word", "segment"])).default(["segment"]).optional(),
+      /**
+       * Options for streaming transcription models such as `gpt-realtime-whisper`.
+       */
+      streaming: z28.object({
+        /**
+         * Latency/accuracy tradeoff for realtime transcription.
+         */
+        delay: z28.enum(["minimal", "low", "medium", "high", "xhigh"]).optional(),
+        /**
+         * Additional fields to include in realtime transcription events.
+         */
+        include: z28.array(z28.string()).optional()
+      }).optional()
     })
   )
 );
 
 // src/transcription/openai-transcription-model.ts
+function isRealtimeTranscriptionModelId(modelId) {
+  return modelId === "gpt-realtime-whisper" || modelId.startsWith("gpt-realtime-whisper-");
+}
 var languageMap = {
   afrikaans: "af",
   arabic: "ar",
@@ -7863,6 +7887,11 @@ var OpenAITranscriptionModel = class _OpenAITranscriptionModel {
   }
   async doGenerate(options) {
     var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
+    if (isRealtimeTranscriptionModelId(this.modelId)) {
+      throw new UnsupportedFunctionalityError6({
+        functionality: `non-streaming transcription with ${this.modelId}`
+      });
+    }
     const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
     const { formData, warnings } = await this.getArgs(options);
     const {
@@ -7906,7 +7935,235 @@ var OpenAITranscriptionModel = class _OpenAITranscriptionModel {
       }
     };
   }
+  async doStream(options) {
+    var _a, _b, _c, _d, _e, _f, _g;
+    if (!isRealtimeTranscriptionModelId(this.modelId)) {
+      throw new UnsupportedFunctionalityError6({
+        functionality: `streaming transcription with ${this.modelId}`
+      });
+    }
+    const currentDate = (_c = (_b = (_a = this.config._internal) == null ? void 0 : _a.currentDate) == null ? void 0 : _b.call(_a)) != null ? _c : /* @__PURE__ */ new Date();
+    const openAIOptions = await parseProviderOptions9({
+      provider: "openai",
+      providerOptions: options.providerOptions,
+      schema: openAITranscriptionModelOptions
+    });
+    const warnings = [];
+    const rawOpenAIOptions = (_e = (_d = options.providerOptions) == null ? void 0 : _d.openai) != null ? _e : {};
+    for (const option of [
+      "include",
+      "prompt",
+      "temperature",
+      "timestampGranularities"
+    ]) {
+      if (rawOpenAIOptions[option] != null) {
+        warnings.push({
+          type: "unsupported",
+          feature: `providerOptions.openai.${option}`,
+          details: `OpenAI streaming transcription does not support ${option}.`
+        });
+      }
+    }
+    const headers = combineHeaders8((_g = (_f = this.config).headers) == null ? void 0 : _g.call(_f), options.headers);
+    const sessionUpdate = buildOpenAIRealtimeTranscriptionSession({
+      modelId: this.modelId,
+      inputAudioFormat: options.inputAudioFormat,
+      providerOptions: openAIOptions
+    });
+    return {
+      request: { body: sessionUpdate },
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId
+      },
+      stream: createOpenAIRealtimeTranscriptionStream({
+        webSocket: this.config.webSocket,
+        url: toWebSocketUrl(
+          this.config.url({
+            path: "/realtime?intent=transcription",
+            modelId: this.modelId
+          })
+        ),
+        headers,
+        sessionUpdate,
+        language: openAIOptions == null ? void 0 : openAIOptions.language,
+        warnings,
+        audio: options.audio,
+        abortSignal: options.abortSignal,
+        includeRawChunks: options.includeRawChunks
+      })
+    };
+  }
 };
+function createOpenAIRealtimeTranscriptionStream({
+  webSocket,
+  url,
+  headers,
+  sessionUpdate,
+  language,
+  warnings,
+  audio,
+  abortSignal,
+  includeRawChunks
+}) {
+  let finished = false;
+  let cleanup = () => {
+  };
+  return new ReadableStream({
+    start: (controller) => {
+      const WebSocketConstructor = getWebSocketConstructor(webSocket);
+      const ws = new WebSocketConstructor(
+        url,
+        getOpenAIRealtimeProtocols(headers),
+        { headers }
+      );
+      let audioReader;
+      cleanup = (closeCode) => {
+        abortSignal == null ? void 0 : abortSignal.removeEventListener("abort", abort);
+        void (audioReader == null ? void 0 : audioReader.cancel().catch(() => {
+        }));
+        try {
+          ws.close(closeCode);
+        } catch (e) {
+        }
+      };
+      const finishWithError = (error) => {
+        if (finished) return;
+        finished = true;
+        cleanup();
+        controller.error(error);
+      };
+      const finish = (text, id) => {
+        if (finished) return;
+        finished = true;
+        if (id != null) {
+          controller.enqueue({ type: "transcript-final", id, text });
+        }
+        controller.enqueue({
+          type: "finish",
+          text,
+          segments: [],
+          language
+        });
+        controller.close();
+        cleanup(1e3);
+      };
+      const abort = () => {
+        var _a;
+        finishWithError((_a = abortSignal == null ? void 0 : abortSignal.reason) != null ? _a : new Error("Aborted"));
+      };
+      if (abortSignal == null ? void 0 : abortSignal.aborted) {
+        abort();
+        return;
+      }
+      abortSignal == null ? void 0 : abortSignal.addEventListener("abort", abort, { once: true });
+      const sendAudio = async () => {
+        audioReader = audio.getReader();
+        try {
+          while (true) {
+            const { done, value } = await audioReader.read();
+            if (done || finished) break;
+            ws.send(
+              JSON.stringify({
+                type: "input_audio_buffer.append",
+                audio: convertToBase643(value)
+              })
+            );
+          }
+        } finally {
+          audioReader.releaseLock();
+        }
+        if (!finished) {
+          ws.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
+        }
+      };
+      ws.onopen = () => {
+        controller.enqueue({ type: "stream-start", warnings });
+        ws.send(JSON.stringify(sessionUpdate));
+        void sendAudio().catch(finishWithError);
+      };
+      ws.onmessage = (event) => {
+        void readWebSocketMessageText(event.data).then(async (text) => {
+          var _a, _b, _c, _d;
+          const parsed = await safeParseJSON({ text });
+          if (!parsed.success) return;
+          const raw = parsed.value;
+          if (includeRawChunks) {
+            controller.enqueue({ type: "raw", rawValue: raw });
+          }
+          switch (raw.type) {
+            case "conversation.item.input_audio_transcription.delta": {
+              controller.enqueue({
+                type: "transcript-delta",
+                id: raw.item_id,
+                delta: (_a = raw.delta) != null ? _a : ""
+              });
+              break;
+            }
+            case "conversation.item.input_audio_transcription.completed": {
+              finish((_b = raw.transcript) != null ? _b : "", raw.item_id);
+              break;
+            }
+            case "error": {
+              finishWithError(
+                new Error((_d = (_c = raw.error) == null ? void 0 : _c.message) != null ? _d : "OpenAI realtime error")
+              );
+              break;
+            }
+          }
+        }).catch(finishWithError);
+      };
+      ws.onerror = () => {
+        finishWithError(new Error("OpenAI realtime transcription error"));
+      };
+      ws.onclose = () => {
+        if (finished) return;
+        finished = true;
+        cleanup();
+        controller.close();
+      };
+    },
+    cancel: () => {
+      if (finished) return;
+      finished = true;
+      cleanup();
+    }
+  });
+}
+function buildOpenAIRealtimeTranscriptionSession({
+  modelId,
+  inputAudioFormat,
+  providerOptions
+}) {
+  var _a, _b;
+  return {
+    type: "session.update",
+    session: {
+      type: "transcription",
+      audio: {
+        input: {
+          format: {
+            type: inputAudioFormat.type,
+            ...inputAudioFormat.rate != null ? { rate: inputAudioFormat.rate } : {}
+          },
+          transcription: {
+            model: modelId,
+            ...(providerOptions == null ? void 0 : providerOptions.language) != null ? { language: providerOptions.language } : {},
+            ...((_a = providerOptions == null ? void 0 : providerOptions.streaming) == null ? void 0 : _a.delay) != null ? { delay: providerOptions.streaming.delay } : {}
+          },
+          turn_detection: null
+        }
+      },
+      ...((_b = providerOptions == null ? void 0 : providerOptions.streaming) == null ? void 0 : _b.include) != null ? { include: providerOptions.streaming.include } : {}
+    }
+  };
+}
+function getOpenAIRealtimeProtocols(headers) {
+  var _a;
+  const authorization = (_a = headers.Authorization) != null ? _a : headers.authorization;
+  const token = (authorization == null ? void 0 : authorization.startsWith("Bearer ")) ? authorization.slice("Bearer ".length) : void 0;
+  return token == null ? ["realtime"] : ["realtime", `openai-insecure-api-key.${token}`];
+}
 
 // src/skills/openai-skills.ts
 import {
@@ -7993,7 +8250,7 @@ var OpenAISkills = class {
 };
 
 // src/version.ts
-var VERSION = true ? "4.0.6" : "0.0.0-test";
+var VERSION = true ? "4.0.8" : "0.0.0-test";
 
 // src/openai-provider.ts
 function createOpenAI(options = {}) {
@@ -8046,7 +8303,8 @@ function createOpenAI(options = {}) {
     provider: `${providerName}.transcription`,
     url: ({ path }) => `${baseURL}${path}`,
     headers: getHeaders,
-    fetch: options.fetch
+    fetch: options.fetch,
+    webSocket: options.webSocket
   });
   const createSpeechModel = (modelId) => new OpenAISpeechModel(modelId, {
     provider: `${providerName}.speech`,