@ai-sdk/openai 4.0.21 → 4.0.22

package/docs/03-openai.mdx

@@ -2903,6 +2903,50 @@ The following provider options are available:
 | `gpt-4o-transcribe`      | <Check />     | <Cross /> | <Cross /> | <Cross /> | <Cross /> |
 | `gpt-realtime-whisper`   | <Cross />     | <Check /> | <Cross /> | <Cross /> | <Cross /> |
 
+## Translation Models
+
+<Note type="warning">Speech translation is an experimental feature.</Note>
+
+You can create models that translate live speech over the OpenAI Realtime
+translations WebSocket using the `.translation()` factory method. Translation
+models are streaming-only and are used with
+[`experimental_streamTranslate`](/docs/reference/ai-sdk-core/stream-translate).
+
+The first argument is the model id e.g. `gpt-realtime-translate`.
+
+```ts
+const model = openai.translation('gpt-realtime-translate');
+```
+
+```ts
+import { experimental_streamTranslate as streamTranslate } from 'ai';
+import { openai } from '@ai-sdk/openai';
+
+const result = streamTranslate({
+  model: openai.translation('gpt-realtime-translate'),
+  audio: audioStream, // ReadableStream<Uint8Array | string>
+  inputAudioFormat: { type: 'audio/pcm', rate: 24000 },
+  targetLanguage: 'es',
+});
+
+for await (const part of result.fullStream) {
+  if (part.type === 'output-text-delta') {
+    process.stdout.write(part.delta);
+  }
+}
+```
+
+The OpenAI Realtime translation API auto-detects the source language
+(`sourceLanguage` is not supported), accepts 24kHz 16-bit PCM input, and always
+outputs 24kHz 16-bit PCM audio (`outputAudioFormat` is not supported).
+Unsupported source and output settings surface as call warnings.
+
+### Model Capabilities
+
+| Model                    | Translated Audio | Translated Text | Source Transcript |
+| ------------------------ | ---------------- | --------------- | ----------------- |
+| `gpt-realtime-translate` | <Check />        | <Check />       | <Check />         |
+
 ## Speech Models
 
 You can create models that call the [OpenAI speech API](https://platform.openai.com/docs/api-reference/audio/speech)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/openai",
-  "version": "4.0.21",
+  "version": "4.0.22",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -36,7 +36,7 @@
   },
   "dependencies": {
     "@ai-sdk/provider": "4.0.4",
-    "@ai-sdk/provider-utils": "5.0.13"
+    "@ai-sdk/provider-utils": "5.0.14"
   },
   "devDependencies": {
     "@types/node": "22.19.19",

package/src/index.ts

@@ -22,6 +22,11 @@ export type { OpenAILanguageModelCompletionOptions } from './completion/openai-c
 export type { OpenAIEmbeddingModelOptions } from './embedding/openai-embedding-model-options';
 export type { OpenAISpeechModelOptions } from './speech/openai-speech-model-options';
 export type { OpenAITranscriptionModelOptions } from './transcription/openai-transcription-model-options';
+export { OpenAITranslationModel as Experimental_OpenAITranslationModel } from './translation/openai-translation-model';
+export type {
+  OpenAITranslationModelId as Experimental_OpenAITranslationModelId,
+  OpenAITranslationModelOptions as Experimental_OpenAITranslationModelOptions,
+} from './translation/openai-translation-model-options';
 export type { OpenAIFilesOptions } from './files/openai-files-options';
 export type {
   OpenAIComputerAction,

package/src/openai-provider.ts

@@ -9,6 +9,7 @@ import type {
   SpeechModelV4,
   SkillsV4,
   TranscriptionModelV4,
+  Experimental_SpeechTranslationModelV4 as SpeechTranslationModelV4,
 } from '@ai-sdk/provider';
 import {
   loadApiKey,
@@ -36,6 +37,8 @@ import { OpenAISpeechModel } from './speech/openai-speech-model';
 import type { OpenAISpeechModelId } from './speech/openai-speech-model-options';
 import { OpenAITranscriptionModel } from './transcription/openai-transcription-model';
 import type { OpenAITranscriptionModelId } from './transcription/openai-transcription-model-options';
+import { OpenAITranslationModel } from './translation/openai-translation-model';
+import type { OpenAITranslationModelId } from './translation/openai-translation-model-options';
 import { OpenAISkills } from './skills/openai-skills';
 import { VERSION } from './version';
 
@@ -97,6 +100,18 @@ export interface OpenAIProvider extends ProviderV4 {
    */
   transcription(modelId: OpenAITranscriptionModelId): TranscriptionModelV4;
 
+  /**
+   * Creates an experimental model for streaming speech translation.
+   */
+  translation(modelId: OpenAITranslationModelId): SpeechTranslationModelV4;
+
+  /**
+   * Creates an experimental model for streaming speech translation.
+   */
+  speechTranslationModel(
+    modelId: OpenAITranslationModelId,
+  ): SpeechTranslationModelV4;
+
   /**
    * Creates a model for speech generation.
    */
@@ -242,6 +257,15 @@ export function createOpenAI(
       webSocket: options.webSocket,
     });
 
+  const createTranslationModel = (modelId: OpenAITranslationModelId) =>
+    new OpenAITranslationModel(modelId, {
+      provider: `${providerName}.translation`,
+      url: ({ path }) => `${baseURL}${path}`,
+      headers: getHeaders,
+      fetch: options.fetch,
+      webSocket: options.webSocket,
+    });
+
   const createSpeechModel = (modelId: OpenAISpeechModelId) =>
     new OpenAISpeechModel(modelId, {
       provider: `${providerName}.speech`,
@@ -334,6 +358,9 @@ export function createOpenAI(
   provider.transcription = createTranscriptionModel;
   provider.transcriptionModel = createTranscriptionModel;
 
+  provider.translation = createTranslationModel;
+  provider.speechTranslationModel = createTranslationModel;
+
   provider.speech = createSpeechModel;
   provider.speechModel = createSpeechModel;
   provider.files = createFiles;

package/src/translation/openai-translation-model-options.ts

@@ -0,0 +1,16 @@
+import {
+  lazySchema,
+  zodSchema,
+  type InferSchema,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+
+export type OpenAITranslationModelId = 'gpt-realtime-translate' | (string & {});
+
+export const openAITranslationModelOptions = lazySchema(() =>
+  zodSchema(z.object({})),
+);
+
+export type OpenAITranslationModelOptions = InferSchema<
+  typeof openAITranslationModelOptions
+>;

package/src/translation/openai-translation-model.ts

@@ -0,0 +1,397 @@
+import {
+  InvalidArgumentError,
+  type Experimental_SpeechTranslationModelV4 as TranslationModelV4,
+  type Experimental_SpeechTranslationModelV4StreamOptions as SpeechTranslationModelV4StreamOptions,
+  type Experimental_SpeechTranslationModelV4StreamPart as SpeechTranslationModelV4StreamPart,
+  type SharedV4Warning,
+} from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  connectToWebSocket,
+  convertToBase64,
+  parseProviderOptions,
+  safeParseJSON,
+  serializeModelOptions,
+  toWebSocketUrl,
+  WORKFLOW_DESERIALIZE,
+  WORKFLOW_SERIALIZE,
+  waitForWebSocketBufferDrain,
+  type WebSocketConnection,
+  type WebSocketLike,
+} from '@ai-sdk/provider-utils';
+import type { OpenAIConfig } from '../openai-config';
+import {
+  openAITranslationModelOptions,
+  type OpenAITranslationModelId,
+} from './openai-translation-model-options';
+
+type OpenAIRealtimeTranslationEvent = {
+  type?: string;
+  delta?: string;
+  error?: { message?: string };
+};
+
+interface OpenAITranslationModelConfig extends OpenAIConfig {
+  _internal?: {
+    currentDate?: () => Date;
+  };
+}
+
+export class OpenAITranslationModel implements TranslationModelV4 {
+  readonly specificationVersion = 'v4';
+
+  static [WORKFLOW_SERIALIZE](model: OpenAITranslationModel) {
+    return serializeModelOptions({
+      modelId: model.modelId,
+      config: model.config,
+    });
+  }
+
+  static [WORKFLOW_DESERIALIZE](options: {
+    modelId: OpenAITranslationModelId;
+    config: OpenAITranslationModelConfig;
+  }) {
+    return new OpenAITranslationModel(options.modelId, options.config);
+  }
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  constructor(
+    readonly modelId: OpenAITranslationModelId,
+    private readonly config: OpenAITranslationModelConfig,
+  ) {}
+
+  async doStream(
+    options: SpeechTranslationModelV4StreamOptions,
+  ): Promise<Awaited<ReturnType<TranslationModelV4['doStream']>>> {
+    if (options.targetLanguage == null) {
+      throw new InvalidArgumentError({
+        argument: 'targetLanguage',
+        message: `targetLanguage is required for translation model '${this.modelId}'.`,
+      });
+    }
+
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    await parseProviderOptions({
+      provider: 'openai',
+      providerOptions: options.providerOptions,
+      schema: openAITranslationModelOptions,
+    });
+    const warnings: SharedV4Warning[] = [];
+
+    validateOpenAITranslationInputAudioFormat(options.inputAudioFormat);
+
+    if (options.sourceLanguage != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'sourceLanguage',
+        details:
+          'The OpenAI Realtime translation API auto-detects the source language and does not accept a source language.',
+      });
+    }
+
+    if (options.outputAudioFormat != null) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'outputAudioFormat',
+        details:
+          'The OpenAI Realtime translation API always outputs 24kHz 16-bit PCM audio and does not accept an output audio format.',
+      });
+    }
+
+    const headers = combineHeaders(this.config.headers?.(), options.headers);
+    const sessionUpdate = buildOpenAIRealtimeTranslationSession({
+      targetLanguage: options.targetLanguage,
+    });
+
+    return {
+      request: { body: sessionUpdate },
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+      },
+      stream: createOpenAIRealtimeTranslationStream({
+        webSocket: this.config.webSocket,
+        url: toWebSocketUrl(
+          this.config.url({
+            path: `/realtime/translations?model=${encodeURIComponent(this.modelId)}`,
+            modelId: this.modelId,
+          }),
+        ),
+        headers,
+        sessionUpdate,
+        warnings,
+        audio: options.audio,
+        abortSignal: options.abortSignal,
+        includeRawChunks: options.includeRawChunks,
+      }),
+    };
+  }
+}
+
+function createOpenAIRealtimeTranslationStream({
+  webSocket,
+  url,
+  headers,
+  sessionUpdate,
+  warnings,
+  audio,
+  abortSignal,
+  includeRawChunks,
+}: {
+  webSocket: OpenAIConfig['webSocket'];
+  url: URL;
+  headers: Record<string, string | undefined>;
+  sessionUpdate: unknown;
+  warnings: SharedV4Warning[];
+  audio: ReadableStream<Uint8Array | string>;
+  abortSignal: AbortSignal | undefined;
+  includeRawChunks: boolean | undefined;
+}) {
+  let finished = false;
+  let cleanup: (closeCode?: number) => void = () => {};
+
+  return new ReadableStream<SpeechTranslationModelV4StreamPart>({
+    start: controller => {
+      const realtimeConnection = getOpenAIRealtimeConnection(headers);
+      let audioReader:
+        | ReadableStreamDefaultReader<Uint8Array | string>
+        | undefined;
+      let connection: WebSocketConnection | undefined;
+
+      let sourceText = '';
+      let translationText = '';
+
+      cleanup = (closeCode?: number) => {
+        if (audioReader != null) {
+          void audioReader.cancel().catch(() => {});
+        } else {
+          // pre-open failure or abort: cancel the caller's audio stream so an
+          // upstream producer piping into it does not hang:
+          void audio.cancel().catch(() => {});
+        }
+        connection?.close(closeCode);
+      };
+
+      const finishWithError = (error: unknown) => {
+        if (finished) return;
+        finished = true;
+        cleanup();
+        controller.error(error);
+      };
+
+      const finish = () => {
+        if (finished) return;
+        finished = true;
+        if (sourceText !== '') {
+          controller.enqueue({
+            type: 'source-transcript-final',
+            text: sourceText,
+          });
+        }
+        if (translationText !== '') {
+          controller.enqueue({
+            type: 'output-text-final',
+            text: translationText,
+          });
+        }
+        controller.enqueue({
+          type: 'finish',
+          sourceText,
+          outputText: translationText,
+          usage: undefined,
+        });
+        controller.close();
+        cleanup(1000);
+      };
+
+      const sendAudio = async (socket: WebSocketLike) => {
+        audioReader = audio.getReader();
+        try {
+          while (true) {
+            const { done, value } = await audioReader.read();
+            if (done || finished) break;
+            socket.send(
+              JSON.stringify({
+                type: 'session.input_audio_buffer.append',
+                audio: convertToBase64(value),
+              }),
+            );
+            // backpressure: pause reads while the socket buffer is full
+            await waitForWebSocketBufferDrain(socket);
+          }
+        } finally {
+          audioReader.releaseLock();
+          // unlocked again: cleanup must cancel `audio`, not the reader
+          audioReader = undefined;
+        }
+        if (!finished) {
+          socket.send(JSON.stringify({ type: 'session.close' }));
+        }
+      };
+
+      connection = connectToWebSocket({
+        url,
+        protocols: realtimeConnection.protocols,
+        headers: realtimeConnection.headers,
+        webSocket,
+        abortSignal,
+        onAbort: finishWithError,
+        onProcessingError: finishWithError,
+        onOpen: socket => {
+          controller.enqueue({ type: 'stream-start', warnings });
+          socket.send(JSON.stringify(sessionUpdate));
+          void sendAudio(socket).catch(finishWithError);
+        },
+        onMessageText: async text => {
+          if (finished) return;
+          const parsed = await safeParseJSON({ text });
+          if (!parsed.success) return;
+          const raw = parsed.value as OpenAIRealtimeTranslationEvent;
+
+          if (includeRawChunks) {
+            controller.enqueue({ type: 'raw', rawValue: raw });
+          }
+
+          switch (raw.type) {
+            case 'session.output_audio.delta': {
+              // skip empty deltas: an empty `audio` part carries no data
+              if (raw.delta) {
+                controller.enqueue({
+                  type: 'audio',
+                  audio: raw.delta,
+                });
+              }
+              break;
+            }
+
+            case 'session.output_transcript.delta': {
+              translationText += raw.delta ?? '';
+              controller.enqueue({
+                type: 'output-text-delta',
+                delta: raw.delta ?? '',
+              });
+              break;
+            }
+
+            case 'session.input_transcript.delta': {
+              sourceText += raw.delta ?? '';
+              controller.enqueue({
+                type: 'source-transcript-delta',
+                delta: raw.delta ?? '',
+              });
+              break;
+            }
+
+            case 'session.closed': {
+              finish();
+              break;
+            }
+
+            case 'error': {
+              controller.enqueue({
+                type: 'error',
+                error: new Error(raw.error?.message ?? 'OpenAI realtime error'),
+              });
+              break;
+            }
+          }
+        },
+        onSocketError: () => {
+          finishWithError(new Error('OpenAI realtime translation error'));
+        },
+        onClose: ({ code, reason }) => {
+          if (finished) return;
+          // a close before the finish event is an abnormal termination:
+          // surface the close diagnostics instead of silently closing
+          finishWithError(
+            new Error(
+              `OpenAI realtime translation WebSocket closed unexpectedly before finishing` +
+                ` (code ${code ?? 'unknown'}${reason ? `, reason: ${reason}` : ''}).`,
+            ),
+          );
+        },
+      });
+    },
+
+    cancel: () => {
+      if (finished) return;
+      finished = true;
+      cleanup();
+    },
+  });
+}
+
+function buildOpenAIRealtimeTranslationSession({
+  targetLanguage,
+}: {
+  targetLanguage: string;
+}) {
+  return {
+    type: 'session.update',
+    session: {
+      audio: {
+        input: {
+          transcription: {
+            model: 'gpt-realtime-whisper',
+          },
+          noise_reduction: null,
+        },
+        output: {
+          language: targetLanguage,
+        },
+      },
+    },
+  };
+}
+
+function validateOpenAITranslationInputAudioFormat(
+  inputAudioFormat: SpeechTranslationModelV4StreamOptions['inputAudioFormat'],
+) {
+  if (
+    inputAudioFormat.type !== 'audio/pcm' ||
+    (inputAudioFormat.rate != null && inputAudioFormat.rate !== 24000)
+  ) {
+    throw new InvalidArgumentError({
+      argument: 'inputAudioFormat',
+      message:
+        'The OpenAI Realtime translation API only supports 24kHz 16-bit PCM input audio.',
+    });
+  }
+}
+
+// The bearer token rides the `openai-insecure-api-key` subprotocol (native
+// `WebSocket` cannot send headers) and the Authorization header is stripped:
+// OpenAI rejects handshakes that send both auth channels.
+function getOpenAIRealtimeConnection(
+  headers: Record<string, string | undefined>,
+): {
+  protocols: string[];
+  headers: Record<string, string | undefined>;
+} {
+  // last case-variant wins: combineHeaders keeps case-distinct keys and
+  // spreads per-call headers after configuration headers
+  let authorization: string | undefined;
+  for (const [key, value] of Object.entries(headers)) {
+    if (key.toLowerCase() === 'authorization' && value != null) {
+      authorization = value;
+    }
+  }
+  // the HTTP auth scheme is case-insensitive
+  const token = authorization?.match(/^bearer\s+(.+)$/i)?.[1];
+
+  if (token == null) {
+    return { protocols: ['realtime'], headers };
+  }
+
+  return {
+    protocols: ['realtime', `openai-insecure-api-key.${token}`],
+    headers: Object.fromEntries(
+      Object.entries(headers).filter(
+        ([key]) => key.toLowerCase() !== 'authorization',
+      ),
+    ),
+  };
+}