@ai-sdk/openai 3.0.14 → 3.0.15

package/src/responses/openai-responses-prepare-tools.ts

@@ -0,0 +1,264 @@
+import {
+  LanguageModelV3CallOptions,
+  SharedV3Warning,
+  UnsupportedFunctionalityError,
+} from '@ai-sdk/provider';
+import { validateTypes } from '@ai-sdk/provider-utils';
+import { codeInterpreterArgsSchema } from '../tool/code-interpreter';
+import { fileSearchArgsSchema } from '../tool/file-search';
+import { imageGenerationArgsSchema } from '../tool/image-generation';
+import { mcpArgsSchema } from '../tool/mcp';
+import { webSearchArgsSchema } from '../tool/web-search';
+import { webSearchPreviewArgsSchema } from '../tool/web-search-preview';
+import { OpenAIResponsesTool } from './openai-responses-api';
+
+export async function prepareResponsesTools({
+  tools,
+  toolChoice,
+}: {
+  tools: LanguageModelV3CallOptions['tools'];
+  toolChoice: LanguageModelV3CallOptions['toolChoice'] | undefined;
+}): Promise<{
+  tools?: Array<OpenAIResponsesTool>;
+  toolChoice?:
+    | 'auto'
+    | 'none'
+    | 'required'
+    | { type: 'file_search' }
+    | { type: 'web_search_preview' }
+    | { type: 'web_search' }
+    | { type: 'function'; name: string }
+    | { type: 'code_interpreter' }
+    | { type: 'mcp' }
+    | { type: 'image_generation' }
+    | { type: 'apply_patch' };
+  toolWarnings: SharedV3Warning[];
+}> {
+  // when the tools array is empty, change it to undefined to prevent errors:
+  tools = tools?.length ? tools : undefined;
+
+  const toolWarnings: SharedV3Warning[] = [];
+
+  if (tools == null) {
+    return { tools: undefined, toolChoice: undefined, toolWarnings };
+  }
+
+  const openaiTools: Array<OpenAIResponsesTool> = [];
+
+  for (const tool of tools) {
+    switch (tool.type) {
+      case 'function':
+        openaiTools.push({
+          type: 'function',
+          name: tool.name,
+          description: tool.description,
+          parameters: tool.inputSchema,
+          ...(tool.strict != null ? { strict: tool.strict } : {}),
+        });
+        break;
+      case 'provider': {
+        switch (tool.id) {
+          case 'openai.file_search': {
+            const args = await validateTypes({
+              value: tool.args,
+              schema: fileSearchArgsSchema,
+            });
+
+            openaiTools.push({
+              type: 'file_search',
+              vector_store_ids: args.vectorStoreIds,
+              max_num_results: args.maxNumResults,
+              ranking_options: args.ranking
+                ? {
+                    ranker: args.ranking.ranker,
+                    score_threshold: args.ranking.scoreThreshold,
+                  }
+                : undefined,
+              filters: args.filters,
+            });
+
+            break;
+          }
+          case 'openai.local_shell': {
+            openaiTools.push({
+              type: 'local_shell',
+            });
+            break;
+          }
+          case 'openai.shell': {
+            openaiTools.push({
+              type: 'shell',
+            });
+            break;
+          }
+          case 'openai.apply_patch': {
+            openaiTools.push({
+              type: 'apply_patch',
+            });
+            break;
+          }
+          case 'openai.web_search_preview': {
+            const args = await validateTypes({
+              value: tool.args,
+              schema: webSearchPreviewArgsSchema,
+            });
+            openaiTools.push({
+              type: 'web_search_preview',
+              search_context_size: args.searchContextSize,
+              user_location: args.userLocation,
+            });
+            break;
+          }
+          case 'openai.web_search': {
+            const args = await validateTypes({
+              value: tool.args,
+              schema: webSearchArgsSchema,
+            });
+            openaiTools.push({
+              type: 'web_search',
+              filters:
+                args.filters != null
+                  ? { allowed_domains: args.filters.allowedDomains }
+                  : undefined,
+              external_web_access: args.externalWebAccess,
+              search_context_size: args.searchContextSize,
+              user_location: args.userLocation,
+            });
+            break;
+          }
+          case 'openai.code_interpreter': {
+            const args = await validateTypes({
+              value: tool.args,
+              schema: codeInterpreterArgsSchema,
+            });
+
+            openaiTools.push({
+              type: 'code_interpreter',
+              container:
+                args.container == null
+                  ? { type: 'auto', file_ids: undefined }
+                  : typeof args.container === 'string'
+                    ? args.container
+                    : { type: 'auto', file_ids: args.container.fileIds },
+            });
+            break;
+          }
+          case 'openai.image_generation': {
+            const args = await validateTypes({
+              value: tool.args,
+              schema: imageGenerationArgsSchema,
+            });
+
+            openaiTools.push({
+              type: 'image_generation',
+              background: args.background,
+              input_fidelity: args.inputFidelity,
+              input_image_mask: args.inputImageMask
+                ? {
+                    file_id: args.inputImageMask.fileId,
+                    image_url: args.inputImageMask.imageUrl,
+                  }
+                : undefined,
+              model: args.model,
+              moderation: args.moderation,
+              partial_images: args.partialImages,
+              quality: args.quality,
+              output_compression: args.outputCompression,
+              output_format: args.outputFormat,
+              size: args.size,
+            });
+            break;
+          }
+          case 'openai.mcp': {
+            const args = await validateTypes({
+              value: tool.args,
+              schema: mcpArgsSchema,
+            });
+
+            const mapApprovalFilter = (filter: { toolNames?: string[] }) => ({
+              tool_names: filter.toolNames,
+            });
+
+            const requireApproval = args.requireApproval;
+            const requireApprovalParam:
+              | 'always'
+              | 'never'
+              | {
+                  never?: { tool_names?: string[] };
+                }
+              | undefined =
+              requireApproval == null
+                ? undefined
+                : typeof requireApproval === 'string'
+                  ? requireApproval
+                  : requireApproval.never != null
+                    ? { never: mapApprovalFilter(requireApproval.never) }
+                    : undefined;
+
+            openaiTools.push({
+              type: 'mcp',
+              server_label: args.serverLabel,
+              allowed_tools: Array.isArray(args.allowedTools)
+                ? args.allowedTools
+                : args.allowedTools
+                  ? {
+                      read_only: args.allowedTools.readOnly,
+                      tool_names: args.allowedTools.toolNames,
+                    }
+                  : undefined,
+              authorization: args.authorization,
+              connector_id: args.connectorId,
+              headers: args.headers,
+              require_approval: requireApprovalParam ?? 'never',
+              server_description: args.serverDescription,
+              server_url: args.serverUrl,
+            });
+
+            break;
+          }
+        }
+        break;
+      }
+      default:
+        toolWarnings.push({
+          type: 'unsupported',
+          feature: `function tool ${tool}`,
+        });
+        break;
+    }
+  }
+
+  if (toolChoice == null) {
+    return { tools: openaiTools, toolChoice: undefined, toolWarnings };
+  }
+
+  const type = toolChoice.type;
+
+  switch (type) {
+    case 'auto':
+    case 'none':
+    case 'required':
+      return { tools: openaiTools, toolChoice: type, toolWarnings };
+    case 'tool':
+      return {
+        tools: openaiTools,
+        toolChoice:
+          toolChoice.toolName === 'code_interpreter' ||
+          toolChoice.toolName === 'file_search' ||
+          toolChoice.toolName === 'image_generation' ||
+          toolChoice.toolName === 'web_search_preview' ||
+          toolChoice.toolName === 'web_search' ||
+          toolChoice.toolName === 'mcp' ||
+          toolChoice.toolName === 'apply_patch'
+            ? { type: toolChoice.toolName }
+            : { type: 'function', name: toolChoice.toolName },
+        toolWarnings,
+      };
+    default: {
+      const _exhaustiveCheck: never = type;
+      throw new UnsupportedFunctionalityError({
+        functionality: `tool choice type: ${_exhaustiveCheck}`,
+      });
+    }
+  }
+}

package/src/responses/openai-responses-provider-metadata.ts

@@ -0,0 +1,39 @@
+import { openaiResponsesChunkSchema } from './openai-responses-api';
+import { InferSchema } from '@ai-sdk/provider-utils';
+
+type OpenaiResponsesChunk = InferSchema<typeof openaiResponsesChunkSchema>;
+
+type ResponsesOutputTextAnnotationProviderMetadata = Extract<
+  OpenaiResponsesChunk,
+  { type: 'response.output_text.annotation.added' }
+>['annotation'];
+
+export type ResponsesTextProviderMetadata = {
+  itemId: string;
+  annotations?: Array<ResponsesOutputTextAnnotationProviderMetadata>;
+};
+
+export type OpenaiResponsesTextProviderMetadata = {
+  openai: ResponsesTextProviderMetadata;
+};
+
+export type ResponsesSourceDocumentProviderMetadata =
+  | {
+      type: 'file_citation';
+      fileId: string;
+      index: number;
+    }
+  | {
+      type: 'container_file_citation';
+      fileId: string;
+      containerId: string;
+    }
+  | {
+      type: 'file_path';
+      fileId: string;
+      index: number;
+    };
+
+export type OpenaiResponsesSourceDocumentProviderMetadata = {
+  openai: ResponsesSourceDocumentProviderMetadata;
+};

package/src/speech/openai-speech-api.ts

@@ -0,0 +1,38 @@
+export type OpenAISpeechAPITypes = {
+  /**
+   * The voice to use when generating the audio.
+   * Supported voices are alloy, ash, ballad, coral, echo, fable, onyx, nova, sage, shimmer, and verse.
+   * @default 'alloy'
+   */
+  voice?:
+    | 'alloy'
+    | 'ash'
+    | 'ballad'
+    | 'coral'
+    | 'echo'
+    | 'fable'
+    | 'onyx'
+    | 'nova'
+    | 'sage'
+    | 'shimmer'
+    | 'verse';
+
+  /**
+   * The speed of the generated audio.
+   * Select a value from 0.25 to 4.0.
+   * @default 1.0
+   */
+  speed?: number;
+
+  /**
+   * The format of the generated audio.
+   * @default 'mp3'
+   */
+  response_format?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';
+
+  /**
+   * Instructions for the speech generation e.g. "Speak in a slow and steady tone".
+   * Does not work with tts-1 or tts-1-hd.
+   */
+  instructions?: string;
+};

package/src/speech/openai-speech-model.test.ts

@@ -0,0 +1,202 @@
+import { createTestServer } from '@ai-sdk/test-server/with-vitest';
+import { createOpenAI } from '../openai-provider';
+import { OpenAISpeechModel } from './openai-speech-model';
+import { describe, it, expect, vi } from 'vitest';
+
+vi.mock('../version', () => ({
+  VERSION: '0.0.0-test',
+}));
+
+const provider = createOpenAI({ apiKey: 'test-api-key' });
+const model = provider.speech('tts-1');
+
+const server = createTestServer({
+  'https://api.openai.com/v1/audio/speech': {},
+});
+
+describe('doGenerate', () => {
+  function prepareAudioResponse({
+    headers,
+    format = 'mp3',
+  }: {
+    headers?: Record<string, string>;
+    format?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';
+  } = {}) {
+    const audioBuffer = new Uint8Array(100); // Mock audio data
+    server.urls['https://api.openai.com/v1/audio/speech'].response = {
+      type: 'binary',
+      headers: {
+        'content-type': `audio/${format}`,
+        ...headers,
+      },
+      body: Buffer.from(audioBuffer),
+    };
+    return audioBuffer;
+  }
+
+  it('should pass the model and text', async () => {
+    prepareAudioResponse();
+
+    await model.doGenerate({
+      text: 'Hello from the AI SDK!',
+    });
+
+    expect(await server.calls[0].requestBodyJson).toMatchObject({
+      model: 'tts-1',
+      input: 'Hello from the AI SDK!',
+    });
+  });
+
+  it('should pass headers', async () => {
+    prepareAudioResponse();
+
+    const provider = createOpenAI({
+      apiKey: 'test-api-key',
+      organization: 'test-organization',
+      project: 'test-project',
+      headers: {
+        'Custom-Provider-Header': 'provider-header-value',
+      },
+    });
+
+    await provider.speech('tts-1').doGenerate({
+      text: 'Hello from the AI SDK!',
+      headers: {
+        'Custom-Request-Header': 'request-header-value',
+      },
+    });
+
+    expect(server.calls[0].requestHeaders).toMatchObject({
+      authorization: 'Bearer test-api-key',
+      'content-type': 'application/json',
+      'custom-provider-header': 'provider-header-value',
+      'custom-request-header': 'request-header-value',
+      'openai-organization': 'test-organization',
+      'openai-project': 'test-project',
+    });
+
+    expect(server.calls[0].requestUserAgent).toContain(
+      `ai-sdk/openai/0.0.0-test`,
+    );
+  });
+
+  it('should pass options', async () => {
+    prepareAudioResponse();
+
+    await model.doGenerate({
+      text: 'Hello from the AI SDK!',
+      voice: 'nova',
+      outputFormat: 'opus',
+      speed: 1.5,
+    });
+
+    expect(await server.calls[0].requestBodyJson).toMatchObject({
+      model: 'tts-1',
+      input: 'Hello from the AI SDK!',
+      voice: 'nova',
+      speed: 1.5,
+      response_format: 'opus',
+    });
+  });
+
+  it('should return audio data with correct content type', async () => {
+    const audio = new Uint8Array(100); // Mock audio data
+    prepareAudioResponse({
+      format: 'opus',
+      headers: {
+        'x-request-id': 'test-request-id',
+        'x-ratelimit-remaining': '123',
+      },
+    });
+
+    const result = await model.doGenerate({
+      text: 'Hello from the AI SDK!',
+      outputFormat: 'opus',
+    });
+
+    expect(result.audio).toStrictEqual(audio);
+  });
+
+  it('should include response data with timestamp, modelId and headers', async () => {
+    prepareAudioResponse({
+      headers: {
+        'x-request-id': 'test-request-id',
+        'x-ratelimit-remaining': '123',
+      },
+    });
+
+    const testDate = new Date(0);
+    const customModel = new OpenAISpeechModel('tts-1', {
+      provider: 'test-provider',
+      url: () => 'https://api.openai.com/v1/audio/speech',
+      headers: () => ({}),
+      _internal: {
+        currentDate: () => testDate,
+      },
+    });
+
+    const result = await customModel.doGenerate({
+      text: 'Hello from the AI SDK!',
+    });
+
+    expect(result.response).toMatchObject({
+      timestamp: testDate,
+      modelId: 'tts-1',
+      headers: {
+        'content-type': 'audio/mp3',
+        'x-request-id': 'test-request-id',
+        'x-ratelimit-remaining': '123',
+      },
+    });
+  });
+
+  it('should use real date when no custom date provider is specified', async () => {
+    prepareAudioResponse();
+
+    const testDate = new Date(0);
+    const customModel = new OpenAISpeechModel('tts-1', {
+      provider: 'test-provider',
+      url: () => 'https://api.openai.com/v1/audio/speech',
+      headers: () => ({}),
+      _internal: {
+        currentDate: () => testDate,
+      },
+    });
+
+    const result = await customModel.doGenerate({
+      text: 'Hello from the AI SDK!',
+    });
+
+    expect(result.response.timestamp.getTime()).toEqual(testDate.getTime());
+    expect(result.response.modelId).toBe('tts-1');
+  });
+
+  it('should handle different audio formats', async () => {
+    const formats = ['mp3', 'opus', 'aac', 'flac', 'wav', 'pcm'] as const;
+
+    for (const format of formats) {
+      const audio = prepareAudioResponse({ format });
+
+      const result = await model.doGenerate({
+        text: 'Hello from the AI SDK!',
+        providerOptions: {
+          openai: {
+            response_format: format,
+          },
+        },
+      });
+
+      expect(result.audio).toStrictEqual(audio);
+    }
+  });
+
+  it('should include warnings if any are generated', async () => {
+    prepareAudioResponse();
+
+    const result = await model.doGenerate({
+      text: 'Hello from the AI SDK!',
+    });
+
+    expect(result.warnings).toEqual([]);
+  });
+});

package/src/speech/openai-speech-model.ts

@@ -0,0 +1,137 @@
+import { SpeechModelV3, SharedV3Warning } from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  createBinaryResponseHandler,
+  parseProviderOptions,
+  postJsonToApi,
+} from '@ai-sdk/provider-utils';
+import { OpenAIConfig } from '../openai-config';
+import { openaiFailedResponseHandler } from '../openai-error';
+import { OpenAISpeechAPITypes } from './openai-speech-api';
+import {
+  openaiSpeechProviderOptionsSchema,
+  OpenAISpeechModelId,
+} from './openai-speech-options';
+
+interface OpenAISpeechModelConfig extends OpenAIConfig {
+  _internal?: {
+    currentDate?: () => Date;
+  };
+}
+
+export class OpenAISpeechModel implements SpeechModelV3 {
+  readonly specificationVersion = 'v3';
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  constructor(
+    readonly modelId: OpenAISpeechModelId,
+    private readonly config: OpenAISpeechModelConfig,
+  ) {}
+
+  private async getArgs({
+    text,
+    voice = 'alloy',
+    outputFormat = 'mp3',
+    speed,
+    instructions,
+    language,
+    providerOptions,
+  }: Parameters<SpeechModelV3['doGenerate']>[0]) {
+    const warnings: SharedV3Warning[] = [];
+
+    // Parse provider options
+    const openAIOptions = await parseProviderOptions({
+      provider: 'openai',
+      providerOptions,
+      schema: openaiSpeechProviderOptionsSchema,
+    });
+
+    // Create request body
+    const requestBody: Record<string, unknown> = {
+      model: this.modelId,
+      input: text,
+      voice,
+      response_format: 'mp3',
+      speed,
+      instructions,
+    };
+
+    if (outputFormat) {
+      if (['mp3', 'opus', 'aac', 'flac', 'wav', 'pcm'].includes(outputFormat)) {
+        requestBody.response_format = outputFormat;
+      } else {
+        warnings.push({
+          type: 'unsupported',
+          feature: 'outputFormat',
+          details: `Unsupported output format: ${outputFormat}. Using mp3 instead.`,
+        });
+      }
+    }
+
+    // Add provider-specific options
+    if (openAIOptions) {
+      const speechModelOptions: OpenAISpeechAPITypes = {};
+
+      for (const key in speechModelOptions) {
+        const value = speechModelOptions[key as keyof OpenAISpeechAPITypes];
+        if (value !== undefined) {
+          requestBody[key] = value;
+        }
+      }
+    }
+
+    if (language) {
+      warnings.push({
+        type: 'unsupported',
+        feature: 'language',
+        details: `OpenAI speech models do not support language selection. Language parameter "${language}" was ignored.`,
+      });
+    }
+
+    return {
+      requestBody,
+      warnings,
+    };
+  }
+
+  async doGenerate(
+    options: Parameters<SpeechModelV3['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<SpeechModelV3['doGenerate']>>> {
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const { requestBody, warnings } = await this.getArgs(options);
+
+    const {
+      value: audio,
+      responseHeaders,
+      rawValue: rawResponse,
+    } = await postJsonToApi({
+      url: this.config.url({
+        path: '/audio/speech',
+        modelId: this.modelId,
+      }),
+      headers: combineHeaders(this.config.headers(), options.headers),
+      body: requestBody,
+      failedResponseHandler: openaiFailedResponseHandler,
+      successfulResponseHandler: createBinaryResponseHandler(),
+      abortSignal: options.abortSignal,
+      fetch: this.config.fetch,
+    });
+
+    return {
+      audio,
+      warnings,
+      request: {
+        body: JSON.stringify(requestBody),
+      },
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+        headers: responseHeaders,
+        body: rawResponse,
+      },
+    };
+  }
+}

package/src/speech/openai-speech-options.ts

@@ -0,0 +1,22 @@
+import { InferSchema, lazySchema, zodSchema } from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+
+export type OpenAISpeechModelId =
+  | 'tts-1'
+  | 'tts-1-hd'
+  | 'gpt-4o-mini-tts'
+  | (string & {});
+
+// https://platform.openai.com/docs/api-reference/audio/createSpeech
+export const openaiSpeechProviderOptionsSchema = lazySchema(() =>
+  zodSchema(
+    z.object({
+      instructions: z.string().nullish(),
+      speed: z.number().min(0.25).max(4.0).default(1.0).nullish(),
+    }),
+  ),
+);
+
+export type OpenAISpeechCallOptions = InferSchema<
+  typeof openaiSpeechProviderOptionsSchema
+>;