@ai-sdk/gateway 4.0.17 → 4.0.19

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ai-sdk/gateway",
   "private": false,
-  "version": "4.0.17",
+  "version": "4.0.19",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -32,7 +32,7 @@
   "dependencies": {
     "@vercel/oidc": "3.2.0",
     "@ai-sdk/provider": "4.0.3",
-    "@ai-sdk/provider-utils": "5.0.7"
+    "@ai-sdk/provider-utils": "5.0.9"
   },
   "devDependencies": {
     "@types/node": "22.19.19",

package/src/gateway-config.ts

@@ -1,7 +1,12 @@
-import type { FetchFunction, Resolvable } from '@ai-sdk/provider-utils';
+import type {
+  FetchFunction,
+  Resolvable,
+  WebSocketConstructor,
+} from '@ai-sdk/provider-utils';
 
 export type GatewayConfig = {
   baseURL: string;
   headers?: Resolvable<Record<string, string | undefined>>;
   fetch?: FetchFunction;
+  webSocket?: WebSocketConstructor;
 };

package/src/gateway-fetch-metadata.ts

@@ -34,6 +34,7 @@ export class GatewayFetchMetadata {
     try {
       const { value } = await getFromApi({
         url: `${this.config.baseURL}/config`,
+        validateUrl: false,
         headers: this.config.headers
           ? await resolve(this.config.headers)
           : undefined,
@@ -59,6 +60,7 @@ export class GatewayFetchMetadata {
 
       const { value } = await getFromApi({
         url: `${baseUrl.origin}/v1/credits`,
+        validateUrl: false,
         headers: this.config.headers
           ? await resolve(this.config.headers)
           : undefined,

package/src/gateway-generation-info.ts

@@ -65,6 +65,7 @@ export class GatewayGenerationInfoFetcher {
 
       const { value } = await getFromApi({
         url: `${baseUrl.origin}/v1/generation?id=${encodeURIComponent(params.id)}`,
+        validateUrl: false,
         headers: this.config.headers
           ? await resolve(this.config.headers)
           : undefined,

package/src/gateway-provider.ts

@@ -6,6 +6,7 @@ import {
   withoutTrailingSlash,
   withUserAgentSuffix,
   type FetchFunction,
+  type WebSocketConstructor,
 } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
 import { asGatewayError, GatewayAuthenticationError } from './errors';
@@ -208,6 +209,14 @@ export interface GatewayProviderSettings {
    */
   fetch?: FetchFunction;
 
+  /**
+   * Custom WebSocket implementation used for streaming transcription. This is
+   * useful for testing or for runtimes without a global WebSocket. A
+   * header-capable implementation is not required — Gateway WebSocket auth is
+   * carried in the subprotocols.
+   */
+  webSocket?: WebSocketConstructor;
+
   /**
    * How frequently to refresh the metadata cache in milliseconds.
    */
@@ -516,6 +525,7 @@ export function createGateway(
       headers: getHeaders,
       fetch: options.fetch,
       o11yHeaders: createO11yHeaders(),
+      webSocket: options.webSocket,
     });
   };
   provider.transcriptionModel = createTranscriptionModel;

package/src/gateway-realtime-auth.ts

@@ -1,5 +1,6 @@
 /**
- * Shared WebSocket subprotocol contract for AI Gateway realtime auth.
+ * Shared WebSocket subprotocol contract for AI Gateway realtime and streaming
+ * transcription auth.
  *
  * The browser `WebSocket` API cannot set request headers, so the Gateway auth
  * (bearer) token is carried through the `Sec-WebSocket-Protocol` handshake
@@ -8,8 +9,9 @@
  *
  * This module is the single source of truth for that contract so the client and
  * the Gateway server can't drift: the client encodes values with
- * `getGatewayRealtimeProtocols`, and the Gateway server decodes them with
- * `getGatewayRealtimeAuthToken` / `getGatewayRealtimeTeamIdOrSlug`.
+ * `getGatewayRealtimeProtocols` / `getGatewayTranscriptionProtocols`, and the
+ * Gateway server decodes them with `getGatewayRealtimeAuthToken` /
+ * `getGatewayRealtimeTeamIdOrSlug`.
  *
  * WebSocket subprotocol values must fit the RFC token grammar. The auth token is
  * sent as-is, so callers must use tokens that are valid subprotocol tokens; the
@@ -19,12 +21,18 @@
  */
 
 /**
- * Marker subprotocol offered on every handshake so the Gateway can echo a
- * negotiated subprotocol on the 101 response (some clients require the server to
- * select one of the offered subprotocols).
+ * Marker subprotocol offered on every realtime handshake so the Gateway can
+ * echo a negotiated subprotocol on the 101 response (some clients require the
+ * server to select one of the offered subprotocols).
  */
 export const GATEWAY_REALTIME_SUBPROTOCOL = 'ai-gateway-realtime.v1';
 
+/**
+ * Marker subprotocol for streaming transcription handshakes (same negotiation
+ * purpose as `GATEWAY_REALTIME_SUBPROTOCOL`).
+ */
+export const GATEWAY_TRANSCRIPTION_SUBPROTOCOL = 'ai-gateway-transcription.v1';
+
 /** Subprotocol prefix that carries the Gateway auth (bearer) token. */
 export const GATEWAY_AUTH_SUBPROTOCOL_PREFIX = 'ai-gateway-auth.';
 
@@ -39,10 +47,30 @@ export function getGatewayRealtimeProtocols(
   token: string,
   options?: { teamIdOrSlug?: string },
 ): string[] {
-  const protocols = [
-    GATEWAY_REALTIME_SUBPROTOCOL,
-    `${GATEWAY_AUTH_SUBPROTOCOL_PREFIX}${token}`,
-  ];
+  return buildGatewayProtocols(GATEWAY_REALTIME_SUBPROTOCOL, token, options);
+}
+
+/**
+ * Client-side: `getGatewayRealtimeProtocols`, but with the streaming
+ * transcription marker subprotocol.
+ */
+export function getGatewayTranscriptionProtocols(
+  token: string,
+  options?: { teamIdOrSlug?: string },
+): string[] {
+  return buildGatewayProtocols(
+    GATEWAY_TRANSCRIPTION_SUBPROTOCOL,
+    token,
+    options,
+  );
+}
+
+function buildGatewayProtocols(
+  marker: string,
+  token: string,
+  options?: { teamIdOrSlug?: string },
+): string[] {
+  const protocols = [marker, `${GATEWAY_AUTH_SUBPROTOCOL_PREFIX}${token}`];
 
   if (options?.teamIdOrSlug) {
     protocols.push(

package/src/gateway-spend-report.ts

@@ -106,6 +106,7 @@ export class GatewaySpendReport {
 
       const { value } = await getFromApi({
         url: `${baseUrl.origin}/v1/report?${searchParams.toString()}`,
+        validateUrl: false,
         headers: this.config.headers
           ? await resolve(this.config.headers)
           : undefined,

package/src/gateway-transcription-model-settings.ts

@@ -1,6 +1,7 @@
 export type GatewayTranscriptionModelId =
   | 'openai/gpt-4o-mini-transcribe'
   | 'openai/gpt-4o-transcribe'
+  | 'openai/gpt-realtime-whisper'
   | 'openai/whisper-1'
   | 'xai/grok-stt'
   | (string & {});

package/src/gateway-transcription-model.ts

@@ -1,21 +1,41 @@
-import type {
-  SharedV4ProviderMetadata,
-  SharedV4Warning,
-  TranscriptionModelV4,
+import {
+  getErrorMessage,
+  type Experimental_TranscriptionModelV4StreamOptions as TranscriptionModelV4StreamOptions,
+  type Experimental_TranscriptionModelV4StreamPart as TranscriptionModelV4StreamPart,
+  type Experimental_TranscriptionModelV4StreamResult as TranscriptionModelV4StreamResult,
+  type SharedV4ProviderMetadata,
+  type SharedV4Warning,
+  type TranscriptionModelV4,
 } from '@ai-sdk/provider';
 import {
   combineHeaders,
+  convertBase64ToUint8Array,
   convertUint8ArrayToBase64,
   createJsonErrorResponseHandler,
   createJsonResponseHandler,
+  connectToWebSocket,
+  normalizeHeaders,
+  experimental_parseTranscriptionStreamPart as parseTranscriptionStreamPart,
   postJsonToApi,
   resolve,
+  EXPERIMENTAL_TRANSCRIPTION_STREAM_AUDIO_DONE_FRAME_TYPE as TRANSCRIPTION_STREAM_AUDIO_DONE_FRAME_TYPE,
+  waitForWebSocketBufferDrain,
+  EXPERIMENTAL_TRANSCRIPTION_STREAM_START_FRAME_TYPE as TRANSCRIPTION_STREAM_START_FRAME_TYPE,
+  type Experimental_TranscriptionStreamStartFrame,
   type Resolvable,
+  type WebSocketConnection,
+  type WebSocketConstructor,
+  type WebSocketLike,
 } from '@ai-sdk/provider-utils';
 import { z } from 'zod/v4';
-import { asGatewayError } from './errors';
+import { asGatewayError, createGatewayErrorFromResponse } from './errors';
 import { parseAuthMethod } from './errors/parse-auth-method';
 import type { GatewayConfig } from './gateway-config';
+import { VERCEL_AI_GATEWAY_TEAM_HEADER } from './gateway-headers';
+import {
+  GATEWAY_TRANSCRIPTION_SUBPROTOCOL,
+  getGatewayTranscriptionProtocols,
+} from './gateway-realtime-auth';
 
 export class GatewayTranscriptionModel implements TranscriptionModelV4 {
   readonly specificationVersion = 'v4' as const;
@@ -25,6 +45,9 @@ export class GatewayTranscriptionModel implements TranscriptionModelV4 {
     private readonly config: GatewayConfig & {
       provider: string;
       o11yHeaders: Resolvable<Record<string, string>>;
+      _internal?: {
+        currentDate?: () => Date;
+      };
     },
   ) {}
 
@@ -99,6 +122,48 @@ export class GatewayTranscriptionModel implements TranscriptionModelV4 {
     }
   }
 
+  async doStream(
+    options: TranscriptionModelV4StreamOptions,
+  ): Promise<TranscriptionModelV4StreamResult> {
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+
+    const headers = combineHeaders(
+      await resolve(this.config.headers ?? {}),
+      options.headers ?? {},
+      this.getModelConfigHeaders(),
+      await resolve(this.config.o11yHeaders),
+    );
+    const authMethod = await parseAuthMethod(headers);
+
+    // First frame after the WebSocket opens, per the transcription-stream
+    // envelope. Optional keys are omitted so the frame stays minimal.
+    const startFrame: Experimental_TranscriptionStreamStartFrame = {
+      type: TRANSCRIPTION_STREAM_START_FRAME_TYPE,
+      inputAudioFormat: options.inputAudioFormat,
+      ...(options.providerOptions != null && {
+        providerOptions: options.providerOptions,
+      }),
+      ...(options.includeRawChunks != null && {
+        includeRawChunks: options.includeRawChunks,
+      }),
+    };
+
+    return {
+      stream: createGatewayTranscriptionStream({
+        webSocket: this.config.webSocket,
+        url: toGatewayTranscriptionUrl(this.config.baseURL, this.modelId),
+        protocols: getProtocolsFromHeaders(headers),
+        headers,
+        startFrame,
+        audio: options.audio,
+        abortSignal: options.abortSignal,
+        authMethod,
+      }),
+      request: { body: startFrame },
+      response: { timestamp: currentDate, modelId: this.modelId },
+    };
+  }
+
   private getUrl() {
     return `${this.config.baseURL}/transcription-model`;
   }
@@ -111,6 +176,222 @@ export class GatewayTranscriptionModel implements TranscriptionModelV4 {
   }
 }
 
+/**
+ * Gateway streaming transcription WebSocket URL: HTTP(S) base upgraded to
+ * WS(S), model id in `?ai-model-id=` (browser `WebSocket` cannot set headers;
+ * slash-safe for qualified ids like `openai/gpt-realtime-whisper`).
+ */
+function toGatewayTranscriptionUrl(baseURL: string, modelId: string): string {
+  const url = new URL(`${baseURL.replace(/^http/, 'ws')}/transcription-model`);
+  url.searchParams.set('ai-model-id', modelId);
+  return url.toString();
+}
+
+/**
+ * Auth-carrying subprotocols from the resolved request headers (bearer token
+ * + optional team scope). Native `WebSocket` cannot send headers, so auth
+ * rides the `Sec-WebSocket-Protocol` handshake. Header lookups are
+ * case-insensitive because `combineHeaders` does not normalize casing and
+ * callers can pass arbitrary casing via the `headers` option.
+ */
+function getProtocolsFromHeaders(
+  headers: Record<string, string | undefined>,
+): string[] {
+  const normalizedHeaders = normalizeHeaders(headers);
+  const authorization = normalizedHeaders.authorization;
+  const token = authorization?.startsWith('Bearer ')
+    ? authorization.slice('Bearer '.length)
+    : undefined;
+
+  return token == null
+    ? [GATEWAY_TRANSCRIPTION_SUBPROTOCOL]
+    : getGatewayTranscriptionProtocols(token, {
+        teamIdOrSlug: normalizedHeaders[VERCEL_AI_GATEWAY_TEAM_HEADER],
+      });
+}
+
+/** Audio frames stay under server frame-size limits (envelope rule 7). */
+const MAX_AUDIO_FRAME_BYTES = 64 * 1024;
+
+function createGatewayTranscriptionStream({
+  webSocket,
+  url,
+  protocols,
+  headers,
+  startFrame,
+  audio,
+  abortSignal,
+  authMethod,
+}: {
+  webSocket: WebSocketConstructor | undefined;
+  url: string;
+  protocols: string[];
+  headers: Record<string, string | undefined>;
+  startFrame: Experimental_TranscriptionStreamStartFrame;
+  audio: ReadableStream<Uint8Array | string>;
+  abortSignal: AbortSignal | undefined;
+  authMethod: 'api-key' | 'oidc' | undefined;
+}): ReadableStream<TranscriptionModelV4StreamPart> {
+  let finished = false;
+  let cleanup: (closeCode?: number) => void = () => {};
+
+  return new ReadableStream<TranscriptionModelV4StreamPart>({
+    start: controller => {
+      let audioReader:
+        | ReadableStreamDefaultReader<Uint8Array | string>
+        | undefined;
+      let hasServerErrorPart = false;
+      let lastServerError: unknown;
+      let audioStopped = false;
+      let connection: WebSocketConnection | undefined;
+
+      cleanup = (closeCode?: number) => {
+        if (audioReader != null) {
+          void audioReader.cancel().catch(() => {});
+        } else {
+          // Pre-open failure or abort: `sendAudio` never took a reader, so
+          // cancel the caller's audio stream directly — otherwise an upstream
+          // producer piping into it hangs.
+          void audio.cancel().catch(() => {});
+        }
+        connection?.close(closeCode);
+      };
+
+      const stopAudio = () => {
+        audioStopped = true;
+        if (audioReader != null) {
+          void audioReader.cancel().catch(() => {});
+          audioReader = undefined;
+        } else {
+          void audio.cancel().catch(() => {});
+        }
+      };
+
+      const finishWithError = (error: unknown) => {
+        if (finished) return;
+        finished = true;
+        cleanup();
+        void errorControllerWithGatewayError(controller, error, authMethod);
+      };
+
+      const sendAudio = async (socket: WebSocketLike) => {
+        const reader = audio.getReader();
+        audioReader = reader;
+        try {
+          while (true) {
+            const { done, value } = await reader.read();
+            if (done || finished) break;
+            // Binary frames; base64 string chunks are decoded first. Caller
+            // chunks are split to stay under server frame-size limits
+            // (envelope rule 7), with backpressure between frames.
+            const bytes =
+              typeof value === 'string'
+                ? convertBase64ToUint8Array(value)
+                : value;
+            for (
+              let offset = 0;
+              offset < bytes.length;
+              offset += MAX_AUDIO_FRAME_BYTES
+            ) {
+              if (finished) break;
+              socket.send(
+                bytes.subarray(offset, offset + MAX_AUDIO_FRAME_BYTES),
+              );
+              await waitForWebSocketBufferDrain(socket);
+            }
+          }
+        } finally {
+          reader.releaseLock();
+          // unlocked again: cleanup must cancel `audio`, not the reader
+          if (audioReader === reader) {
+            audioReader = undefined;
+          }
+        }
+        if (!finished && !audioStopped) {
+          socket.send(
+            JSON.stringify({
+              type: TRANSCRIPTION_STREAM_AUDIO_DONE_FRAME_TYPE,
+            }),
+          );
+        }
+      };
+
+      connection = connectToWebSocket({
+        url,
+        protocols,
+        headers,
+        webSocket,
+        abortSignal,
+        onAbort: reason => {
+          if (finished) return;
+          finished = true;
+          cleanup();
+          controller.error(reason);
+        },
+        onProcessingError: finishWithError,
+        onOpen: socket => {
+          socket.send(JSON.stringify(startFrame));
+          void sendAudio(socket).catch(finishWithError);
+        },
+        // Server frames are envelope-serialized stream parts; the codec
+        // handles parsing, unknown-part skipping, and timestamp revival.
+        onMessageText: text => {
+          if (finished) return;
+          const part = parseTranscriptionStreamPart(text);
+          if (part == null) return;
+          if (part.type === 'finish') {
+            finished = true;
+            controller.enqueue(part);
+            controller.close();
+            cleanup(1000);
+            return;
+          }
+
+          if (part.type === 'error') {
+            // Remembered so the terminal close error (the server closes
+            // non-1000 after an error part) surfaces the server's message
+            // to promise-based consumers.
+            hasServerErrorPart = true;
+            lastServerError = part.error;
+            // envelope rule 5: error parts are terminal — stop sending audio
+            // while the server holds the connection open (e.g. for its final
+            // billing flush)
+            stopAudio();
+          }
+
+          controller.enqueue(part);
+        },
+        onSocketError: () => {
+          finishWithError(
+            new Error('Connection error on AI Gateway transcription stream'),
+          );
+        },
+        onClose: () => {
+          if (hasServerErrorPart) {
+            if (finished) return;
+            void createErrorFromServerErrorPart(
+              lastServerError,
+              authMethod,
+            ).then(finishWithError);
+            return;
+          }
+          finishWithError(
+            new Error(
+              'AI Gateway transcription stream closed before a finish part was received',
+            ),
+          );
+        },
+      });
+    },
+
+    cancel: () => {
+      if (finished) return;
+      finished = true;
+      cleanup();
+    },
+  });
+}
+
 const providerMetadataEntrySchema = z.object({}).catchall(z.unknown());
 
 const gatewayTranscriptionWarningSchema = z.discriminatedUnion('type', [
@@ -153,3 +434,71 @@ const gatewayTranscriptionResponseSchema = z.object({
     .record(z.string(), providerMetadataEntrySchema)
     .optional(),
 });
+
+/**
+ * `asGatewayError` is async while the WebSocket event handlers are
+ * synchronous, hence this helper.
+ */
+async function errorControllerWithGatewayError(
+  controller: ReadableStreamDefaultController<TranscriptionModelV4StreamPart>,
+  error: unknown,
+  authMethod: 'api-key' | 'oidc' | undefined,
+): Promise<void> {
+  controller.error(await asGatewayError(error, authMethod));
+}
+
+/**
+ * Extract a human-readable message from an `error` stream part's payload for
+ * the terminal close error.
+ */
+function getServerErrorMessage(error: unknown): string {
+  if (
+    error != null &&
+    typeof error === 'object' &&
+    'message' in error &&
+    typeof error.message === 'string'
+  ) {
+    return error.message;
+  }
+  // JSON-stringifies object payloads (`String` would yield '[object Object]').
+  return getErrorMessage(error);
+}
+
+/** Canonical status codes for server error-part types (no HTTP status exists on the WebSocket). */
+const SERVER_ERROR_STATUS_CODES: Record<string, number> = {
+  authentication_error: 401,
+  failed_dependency: 424,
+  forbidden: 403,
+  internal_server_error: 500,
+  invalid_request_error: 400,
+  model_not_found: 404,
+  rate_limit_exceeded: 429,
+};
+
+/**
+ * Maps a server error-part payload (`{ message, type }`) to the public
+ * Gateway error class for its type; unknown shapes keep the generic message.
+ */
+async function createErrorFromServerErrorPart(
+  error: unknown,
+  authMethod: 'api-key' | 'oidc' | undefined,
+): Promise<unknown> {
+  if (
+    typeof error === 'object' &&
+    error != null &&
+    'message' in error &&
+    typeof error.message === 'string' &&
+    'type' in error &&
+    typeof error.type === 'string' &&
+    error.type in SERVER_ERROR_STATUS_CODES
+  ) {
+    return createGatewayErrorFromResponse({
+      response: { error: { message: error.message, type: error.type } },
+      statusCode: SERVER_ERROR_STATUS_CODES[error.type],
+      authMethod,
+    });
+  }
+  return new Error(
+    `AI Gateway transcription stream failed: ${getServerErrorMessage(error)}`,
+  );
+}

package/src/index.ts

@@ -3,9 +3,11 @@ export {
   GATEWAY_AUTH_SUBPROTOCOL_PREFIX,
   GATEWAY_REALTIME_SUBPROTOCOL,
   GATEWAY_TEAM_SUBPROTOCOL_PREFIX,
+  GATEWAY_TRANSCRIPTION_SUBPROTOCOL,
   getGatewayRealtimeAuthToken,
   getGatewayRealtimeProtocols,
   getGatewayRealtimeTeamIdOrSlug,
+  getGatewayTranscriptionProtocols,
 } from './gateway-realtime-auth';
 export type { GatewayRealtimeModelId } from './gateway-realtime-model-settings';
 export type { GatewayRerankingModelId } from './gateway-reranking-model-settings';