@ai-sdk/gateway 3.0.135 → 3.0.136

package/docs/00-ai-gateway.mdx

@@ -204,6 +204,40 @@ console.log(ranking);
 
 Reranking models are useful for improving search results in retrieval-augmented generation (RAG) pipelines by re-scoring candidate documents after an initial retrieval step.
 
+## Speech Models
+
+You can create speech models using the `speechModel` method on the provider instance:
+
+```ts
+import { experimental_generateSpeech as generateSpeech } from "ai";
+import { gateway } from "@ai-sdk/gateway";
+
+const { audio } = await generateSpeech({
+  model: gateway.speechModel("openai/tts-1"),
+  text: "Hello, world!",
+  voice: "alloy",
+});
+```
+
+Speech models are useful for generating audio from text using the same Gateway authentication, routing, and observability as other Gateway model types.
+
+## Transcription Models
+
+You can create transcription models using the `transcriptionModel` method on the provider instance:
+
+```ts
+import { experimental_transcribe as transcribe } from "ai";
+import { gateway } from "@ai-sdk/gateway";
+import { readFile } from "fs/promises";
+
+const { text } = await transcribe({
+  model: gateway.transcriptionModel("openai/gpt-4o-transcribe"),
+  audio: await readFile("audio.mp3"),
+});
+```
+
+Transcription models are useful for converting audio to text through the AI Gateway.
+
 ## Available Models
 
 The AI Gateway supports models from OpenAI, Anthropic, Google, Meta, xAI, Mistral, DeepSeek, Amazon Bedrock, Cohere, Perplexity, Alibaba, and other providers.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ai-sdk/gateway",
   "private": false,
-  "version": "3.0.135",
+  "version": "3.0.136",
   "license": "Apache-2.0",
   "sideEffects": false,
   "main": "./dist/index.js",

package/src/gateway-language-model-settings.ts

@@ -195,5 +195,6 @@ export type GatewayModelId =
   | 'zai/glm-5-turbo'
   | 'zai/glm-5.1'
   | 'zai/glm-5.2'
+  | 'zai/glm-5.2-fast'
   | 'zai/glm-5v-turbo'
   | (string & {});

package/src/gateway-model-entry.ts

@@ -5,6 +5,8 @@ export const KNOWN_MODEL_TYPES = [
   'image',
   'language',
   'reranking',
+  'speech',
+  'transcription',
   'video',
 ] as const;
 

package/src/gateway-provider.ts

@@ -29,9 +29,13 @@ import { GatewayEmbeddingModel } from './gateway-embedding-model';
 import { GatewayImageModel } from './gateway-image-model';
 import { GatewayVideoModel } from './gateway-video-model';
 import { GatewayRerankingModel } from './gateway-reranking-model';
+import { GatewaySpeechModel } from './gateway-speech-model';
+import { GatewayTranscriptionModel } from './gateway-transcription-model';
 import type { GatewayEmbeddingModelId } from './gateway-embedding-model-settings';
 import type { GatewayImageModelId } from './gateway-image-model-settings';
 import type { GatewayRerankingModelId } from './gateway-reranking-model-settings';
+import type { GatewaySpeechModelId } from './gateway-speech-model-settings';
+import type { GatewayTranscriptionModelId } from './gateway-transcription-model-settings';
 import type { GatewayVideoModelId } from './gateway-video-model-settings';
 import { gatewayTools } from './gateway-tools';
 import { getVercelOidcToken, getVercelRequestId } from './vercel-environment';
@@ -41,6 +45,8 @@ import type {
   EmbeddingModelV3,
   ImageModelV3,
   RerankingModelV3,
+  SpeechModelV3,
+  TranscriptionModelV3,
   Experimental_VideoModelV3,
   ProviderV3,
 } from '@ai-sdk/provider';
@@ -130,6 +136,28 @@ export interface GatewayProvider extends ProviderV3 {
    */
   rerankingModel(modelId: GatewayRerankingModelId): RerankingModelV3;
 
+  /**
+   * Creates a model for text-to-speech generation.
+   */
+  speech(modelId: GatewaySpeechModelId): SpeechModelV3;
+
+  /**
+   * Creates a model for text-to-speech generation.
+   */
+  speechModel(modelId: GatewaySpeechModelId): SpeechModelV3;
+
+  /**
+   * Creates a model for audio transcription.
+   */
+  transcription(modelId: GatewayTranscriptionModelId): TranscriptionModelV3;
+
+  /**
+   * Creates a model for audio transcription.
+   */
+  transcriptionModel(
+    modelId: GatewayTranscriptionModelId,
+  ): TranscriptionModelV3;
+
   /**
    * Gateway-specific tools executed server-side.
    */
@@ -378,6 +406,28 @@ export function createGatewayProvider(
   };
   provider.rerankingModel = createRerankingModel;
   provider.reranking = createRerankingModel;
+  const createSpeechModel = (modelId: GatewaySpeechModelId) => {
+    return new GatewaySpeechModel(modelId, {
+      provider: 'gateway',
+      baseURL,
+      headers: getHeaders,
+      fetch: options.fetch,
+      o11yHeaders: createO11yHeaders(),
+    });
+  };
+  provider.speechModel = createSpeechModel;
+  provider.speech = createSpeechModel;
+  const createTranscriptionModel = (modelId: GatewayTranscriptionModelId) => {
+    return new GatewayTranscriptionModel(modelId, {
+      provider: 'gateway',
+      baseURL,
+      headers: getHeaders,
+      fetch: options.fetch,
+      o11yHeaders: createO11yHeaders(),
+    });
+  };
+  provider.transcriptionModel = createTranscriptionModel;
+  provider.transcription = createTranscriptionModel;
   provider.chat = provider.languageModel;
   provider.embedding = provider.embeddingModel;
   provider.image = provider.imageModel;

package/src/gateway-speech-model-settings.ts

@@ -0,0 +1,5 @@
+export type GatewaySpeechModelId =
+  | 'openai/tts-1'
+  | 'openai/tts-1-hd'
+  | 'xai/grok-tts'
+  | (string & {});

package/src/gateway-speech-model.ts

@@ -0,0 +1,138 @@
+import type {
+  SharedV3ProviderMetadata,
+  SharedV3Warning,
+  SpeechModelV3,
+} from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  createJsonErrorResponseHandler,
+  createJsonResponseHandler,
+  postJsonToApi,
+  resolve,
+  type Resolvable,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import { asGatewayError } from './errors';
+import { parseAuthMethod } from './errors/parse-auth-method';
+import type { GatewayConfig } from './gateway-config';
+
+export class GatewaySpeechModel implements SpeechModelV3 {
+  readonly specificationVersion = 'v3' as const;
+
+  constructor(
+    readonly modelId: string,
+    private readonly config: GatewayConfig & {
+      provider: string;
+      o11yHeaders: Resolvable<Record<string, string>>;
+    },
+  ) {}
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  async doGenerate({
+    text,
+    voice,
+    outputFormat,
+    instructions,
+    speed,
+    language,
+    providerOptions,
+    headers,
+    abortSignal,
+  }: Parameters<SpeechModelV3['doGenerate']>[0]): Promise<
+    Awaited<ReturnType<SpeechModelV3['doGenerate']>>
+  > {
+    const resolvedHeaders = await resolve(this.config.headers());
+    try {
+      const {
+        responseHeaders,
+        value: responseBody,
+        rawValue,
+      } = await postJsonToApi({
+        url: this.getUrl(),
+        headers: combineHeaders(
+          resolvedHeaders,
+          headers ?? {},
+          this.getModelConfigHeaders(),
+          await resolve(this.config.o11yHeaders),
+        ),
+        body: {
+          text,
+          ...(voice && { voice }),
+          ...(outputFormat && { outputFormat }),
+          ...(instructions && { instructions }),
+          ...(speed != null && { speed }),
+          ...(language && { language }),
+          ...(providerOptions && { providerOptions }),
+        },
+        successfulResponseHandler: createJsonResponseHandler(
+          gatewaySpeechResponseSchema,
+        ),
+        failedResponseHandler: createJsonErrorResponseHandler({
+          errorSchema: z.any(),
+          errorToMessage: data => data,
+        }),
+        ...(abortSignal && { abortSignal }),
+        fetch: this.config.fetch,
+      });
+
+      return {
+        audio: responseBody.audio,
+        warnings: (responseBody.warnings ?? []) as Array<SharedV3Warning>,
+        providerMetadata:
+          responseBody.providerMetadata as SharedV3ProviderMetadata,
+        response: {
+          timestamp: new Date(),
+          modelId: this.modelId,
+          headers: responseHeaders,
+          body: rawValue,
+        },
+      };
+    } catch (error) {
+      throw await asGatewayError(
+        error,
+        await parseAuthMethod(resolvedHeaders ?? {}),
+      );
+    }
+  }
+
+  private getUrl() {
+    return `${this.config.baseURL}/speech-model`;
+  }
+
+  private getModelConfigHeaders() {
+    return {
+      'ai-speech-model-specification-version': '3',
+      'ai-model-id': this.modelId,
+    };
+  }
+}
+
+const providerMetadataEntrySchema = z.object({}).catchall(z.unknown());
+
+const gatewaySpeechWarningSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('unsupported'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('compatibility'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('other'),
+    message: z.string(),
+  }),
+]);
+
+const gatewaySpeechResponseSchema = z.object({
+  audio: z.string(),
+  warnings: z.array(gatewaySpeechWarningSchema).optional(),
+  providerMetadata: z
+    .record(z.string(), providerMetadataEntrySchema)
+    .optional(),
+});

package/src/gateway-transcription-model-settings.ts

@@ -0,0 +1,6 @@
+export type GatewayTranscriptionModelId =
+  | 'openai/gpt-4o-mini-transcribe'
+  | 'openai/gpt-4o-transcribe'
+  | 'openai/whisper-1'
+  | 'xai/grok-stt'
+  | (string & {});

package/src/gateway-transcription-model.ts

@@ -0,0 +1,148 @@
+import type {
+  SharedV3ProviderMetadata,
+  SharedV3Warning,
+  TranscriptionModelV3,
+} from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  convertUint8ArrayToBase64,
+  createJsonErrorResponseHandler,
+  createJsonResponseHandler,
+  postJsonToApi,
+  resolve,
+  type Resolvable,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import { asGatewayError } from './errors';
+import { parseAuthMethod } from './errors/parse-auth-method';
+import type { GatewayConfig } from './gateway-config';
+
+export class GatewayTranscriptionModel implements TranscriptionModelV3 {
+  readonly specificationVersion = 'v3' as const;
+
+  constructor(
+    readonly modelId: string,
+    private readonly config: GatewayConfig & {
+      provider: string;
+      o11yHeaders: Resolvable<Record<string, string>>;
+    },
+  ) {}
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  async doGenerate({
+    audio,
+    mediaType,
+    providerOptions,
+    headers,
+    abortSignal,
+  }: Parameters<TranscriptionModelV3['doGenerate']>[0]): Promise<
+    Awaited<ReturnType<TranscriptionModelV3['doGenerate']>>
+  > {
+    const resolvedHeaders = await resolve(this.config.headers());
+    try {
+      const {
+        responseHeaders,
+        value: responseBody,
+        rawValue,
+      } = await postJsonToApi({
+        url: this.getUrl(),
+        headers: combineHeaders(
+          resolvedHeaders,
+          headers ?? {},
+          this.getModelConfigHeaders(),
+          await resolve(this.config.o11yHeaders),
+        ),
+        body: {
+          audio:
+            audio instanceof Uint8Array
+              ? convertUint8ArrayToBase64(audio)
+              : audio,
+          mediaType,
+          ...(providerOptions && { providerOptions }),
+        },
+        successfulResponseHandler: createJsonResponseHandler(
+          gatewayTranscriptionResponseSchema,
+        ),
+        failedResponseHandler: createJsonErrorResponseHandler({
+          errorSchema: z.any(),
+          errorToMessage: data => data,
+        }),
+        ...(abortSignal && { abortSignal }),
+        fetch: this.config.fetch,
+      });
+
+      return {
+        text: responseBody.text,
+        segments: responseBody.segments ?? [],
+        language: responseBody.language ?? undefined,
+        durationInSeconds: responseBody.durationInSeconds ?? undefined,
+        warnings: (responseBody.warnings ?? []) as Array<SharedV3Warning>,
+        providerMetadata:
+          responseBody.providerMetadata as SharedV3ProviderMetadata,
+        response: {
+          timestamp: new Date(),
+          modelId: this.modelId,
+          headers: responseHeaders,
+          body: rawValue,
+        },
+      };
+    } catch (error) {
+      throw await asGatewayError(
+        error,
+        await parseAuthMethod(resolvedHeaders ?? {}),
+      );
+    }
+  }
+
+  private getUrl() {
+    return `${this.config.baseURL}/transcription-model`;
+  }
+
+  private getModelConfigHeaders() {
+    return {
+      'ai-transcription-model-specification-version': '3',
+      'ai-model-id': this.modelId,
+    };
+  }
+}
+
+const providerMetadataEntrySchema = z.object({}).catchall(z.unknown());
+
+const gatewayTranscriptionWarningSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('unsupported'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('compatibility'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('other'),
+    message: z.string(),
+  }),
+]);
+
+const gatewayTranscriptionResponseSchema = z.object({
+  text: z.string(),
+  segments: z
+    .array(
+      z.object({
+        text: z.string(),
+        startSecond: z.number(),
+        endSecond: z.number(),
+      }),
+    )
+    .optional(),
+  language: z.string().nullish(),
+  durationInSeconds: z.number().nullish(),
+  warnings: z.array(gatewayTranscriptionWarningSchema).optional(),
+  providerMetadata: z
+    .record(z.string(), providerMetadataEntrySchema)
+    .optional(),
+});

package/src/gateway-video-model-settings.ts

@@ -23,5 +23,6 @@ export type GatewayVideoModelId =
   | 'klingai/kling-v3.0-motion-control'
   | 'klingai/kling-v3.0-t2v'
   | 'xai/grok-imagine-video'
+  | 'xai/grok-imagine-video-1.5'
   | 'xai/grok-imagine-video-1.5-preview'
   | (string & {});

package/src/index.ts

@@ -1,5 +1,7 @@
 export type { GatewayModelId } from './gateway-language-model-settings';
 export type { GatewayRerankingModelId } from './gateway-reranking-model-settings';
+export type { GatewaySpeechModelId } from './gateway-speech-model-settings';
+export type { GatewayTranscriptionModelId } from './gateway-transcription-model-settings';
 export type { GatewayVideoModelId } from './gateway-video-model-settings';
 export type {
   GatewayLanguageModelEntry,