@ai-sdk/gateway 4.0.0-canary.98 → 4.0.0

package/src/gateway-provider.ts

@@ -1,9 +1,13 @@
 import {
+  createJsonErrorResponseHandler,
+  createJsonResponseHandler,
   loadOptionalSetting,
+  postJsonToApi,
   withoutTrailingSlash,
   withUserAgentSuffix,
   type FetchFunction,
 } from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
 import { asGatewayError, GatewayAuthenticationError } from './errors';
 import {
   GATEWAY_AUTH_METHOD_HEADER,
@@ -32,11 +36,13 @@ import { GatewayVideoModel } from './gateway-video-model';
 import { GatewayRerankingModel } from './gateway-reranking-model';
 import { GatewaySpeechModel } from './gateway-speech-model';
 import { GatewayTranscriptionModel } from './gateway-transcription-model';
+import { GatewayRealtimeModel } from './gateway-realtime-model';
 import type { GatewayEmbeddingModelId } from './gateway-embedding-model-settings';
 import type { GatewayImageModelId } from './gateway-image-model-settings';
 import type { GatewayRerankingModelId } from './gateway-reranking-model-settings';
 import type { GatewaySpeechModelId } from './gateway-speech-model-settings';
 import type { GatewayTranscriptionModelId } from './gateway-transcription-model-settings';
+import type { GatewayRealtimeModelId } from './gateway-realtime-model-settings';
 import type { GatewayVideoModelId } from './gateway-video-model-settings';
 import { gatewayTools } from './gateway-tools';
 import { getVercelOidcToken, getVercelRequestId } from './vercel-environment';
@@ -49,6 +55,8 @@ import type {
   SpeechModelV4,
   TranscriptionModelV4,
   Experimental_VideoModelV4,
+  Experimental_RealtimeFactoryV4 as RealtimeFactoryV4,
+  Experimental_RealtimeFactoryV4GetTokenOptions as RealtimeFactoryV4GetTokenOptions,
   ProviderV4,
 } from '@ai-sdk/provider';
 import { VERSION } from './version';
@@ -159,6 +167,12 @@ export interface GatewayProvider extends ProviderV4 {
     modelId: GatewayTranscriptionModelId,
   ): TranscriptionModelV4;
 
+  /**
+   * Creates an experimental realtime model for bidirectional audio/text
+   * communication over WebSocket, normalized through the AI Gateway.
+   */
+  experimental_realtime: RealtimeFactoryV4;
+
   /**
    * Gateway-specific tools executed server-side.
    */
@@ -209,6 +223,12 @@ export interface GatewayProviderSettings {
 
 const AI_GATEWAY_PROTOCOL_VERSION = '0.0.1';
 
+/** Response shape of `POST /v1/realtime/client-secrets`. `expiresAt` is epoch seconds. */
+const gatewayClientSecretResponseSchema = z.object({
+  token: z.string(),
+  expiresAt: z.number().nullish(),
+});
+
 /**
  * Create a remote provider instance.
  */
@@ -255,6 +275,62 @@ export function createGateway(
     }
   };
 
+  const getRealtimeAuthToken = async () => {
+    try {
+      return await getGatewayAuthToken(options);
+    } catch (error) {
+      throw GatewayAuthenticationError.createContextualError({
+        apiKeyProvided: false,
+        oidcTokenProvided: false,
+        statusCode: 401,
+        cause: error,
+      });
+    }
+  };
+
+  // Mints a short-lived realtime client secret (`vcst_`) via the Gateway's
+  // `/v1/realtime/client-secrets` route, authenticated with the long-lived
+  // Gateway credential. Server-side only (asserted) — the credential never
+  // belongs in a browser; the browser receives only the minted token. The
+  // mint route lives at the gateway origin's `/v1/realtime/client-secrets`,
+  // not under the realtime `baseURL` path (which targets `/v4/ai`), so the
+  // URL is resolved against the origin.
+  const mintRealtimeClientSecret = async (params: {
+    modelId: string;
+    expiresAfterSeconds?: number;
+  }): Promise<{ token: string; expiresAt?: number }> => {
+    assertGatewayRealtimeServerEnvironment();
+    const auth = await getRealtimeAuthToken();
+    const headers = createAuthHeaders(auth);
+    const url = new URL('/v1/realtime/client-secrets', baseURL).toString();
+    try {
+      const { value } = await postJsonToApi({
+        url,
+        headers,
+        body: {
+          model: params.modelId,
+          ...(params.expiresAfterSeconds != null && {
+            expiresIn: params.expiresAfterSeconds,
+          }),
+        },
+        successfulResponseHandler: createJsonResponseHandler(
+          gatewayClientSecretResponseSchema,
+        ),
+        failedResponseHandler: createJsonErrorResponseHandler({
+          errorSchema: z.any(),
+          errorToMessage: data => data,
+        }),
+        fetch: options.fetch,
+      });
+      return {
+        token: value.token,
+        ...(value.expiresAt != null && { expiresAt: value.expiresAt }),
+      };
+    } catch (error) {
+      throw await asGatewayError(error, await parseAuthMethod(headers));
+    }
+  };
+
   const createO11yHeaders = () => {
     const deploymentId = loadOptionalSetting({
       settingValue: undefined,
@@ -444,6 +520,35 @@ export function createGateway(
   };
   provider.transcriptionModel = createTranscriptionModel;
   provider.transcription = createTranscriptionModel;
+
+  // No server-environment guard here: building the realtime model is just the
+  // event codec + WebSocket-config helper, which the browser legitimately
+  // needs to drive the transport with a server-minted client secret. The
+  // server-only boundary is enforced on minting itself
+  // (`mintRealtimeClientSecret`), which requires the Gateway credential.
+  const createRealtimeModel = (modelId: GatewayRealtimeModelId) =>
+    new GatewayRealtimeModel(modelId, {
+      provider: 'gateway.realtime',
+      baseURL,
+      teamIdOrSlug: options.teamIdOrSlug,
+      createClientSecret: mintRealtimeClientSecret,
+    });
+  provider.experimental_realtime = Object.assign(
+    (modelId: GatewayRealtimeModelId) => createRealtimeModel(modelId),
+    {
+      getToken: async (tokenOptions: RealtimeFactoryV4GetTokenOptions) => {
+        const { model: modelId, ...secretOptions } = tokenOptions;
+        const model = createRealtimeModel(modelId);
+        const secret = await model.doCreateClientSecret(secretOptions);
+        return {
+          token: secret.token,
+          url: secret.url,
+          ...(secret.expiresAt != null && { expiresAt: secret.expiresAt }),
+        };
+      },
+    },
+  ) as RealtimeFactoryV4;
+
   provider.chat = provider.languageModel;
   provider.embedding = provider.embeddingModel;
   provider.image = provider.imageModel;
@@ -475,3 +580,11 @@ export async function getGatewayAuthToken(
     authMethod: 'oidc',
   };
 }
+
+function assertGatewayRealtimeServerEnvironment(): void {
+  if (typeof globalThis.window !== 'undefined') {
+    throw new Error(
+      'AI Gateway realtime client secrets must be minted server-side: minting needs your Gateway credential, which must never reach the browser. Call gateway.experimental_realtime.getToken() from your server and pass the returned token to the client.',
+    );
+  }
+}

package/src/gateway-realtime-auth.ts

@@ -0,0 +1,126 @@
+/**
+ * Shared WebSocket subprotocol contract for AI Gateway realtime auth.
+ *
+ * The browser `WebSocket` API cannot set request headers, so the Gateway auth
+ * (bearer) token is carried through the `Sec-WebSocket-Protocol` handshake
+ * instead of an `Authorization` header — the same workaround OpenAI uses for
+ * `openai-insecure-api-key.<token>`.
+ *
+ * This module is the single source of truth for that contract so the client and
+ * the Gateway server can't drift: the client encodes values with
+ * `getGatewayRealtimeProtocols`, and the Gateway server decodes them with
+ * `getGatewayRealtimeAuthToken` / `getGatewayRealtimeTeamIdOrSlug`.
+ *
+ * WebSocket subprotocol values must fit the RFC token grammar. The auth token is
+ * sent as-is, so callers must use tokens that are valid subprotocol tokens; the
+ * optional team scope is base64url-encoded by this module. Keep the complete
+ * `Sec-WebSocket-Protocol` header compact (target under an 8 KiB safe header
+ * budget) because intermediaries may reject large upgrade headers.
+ */
+
+/**
+ * Marker subprotocol offered on every handshake so the Gateway can echo a
+ * negotiated subprotocol on the 101 response (some clients require the server to
+ * select one of the offered subprotocols).
+ */
+export const GATEWAY_REALTIME_SUBPROTOCOL = 'ai-gateway-realtime.v1';
+
+/** Subprotocol prefix that carries the Gateway auth (bearer) token. */
+export const GATEWAY_AUTH_SUBPROTOCOL_PREFIX = 'ai-gateway-auth.';
+
+/** Subprotocol prefix that carries optional Vercel team scoping. */
+export const GATEWAY_TEAM_SUBPROTOCOL_PREFIX = 'ai-gateway-team.';
+
+/**
+ * Client-side: build the WebSocket subprotocols that carry `token` to the
+ * Gateway. Pass the result as the second argument to `new WebSocket(url, ...)`.
+ */
+export function getGatewayRealtimeProtocols(
+  token: string,
+  options?: { teamIdOrSlug?: string },
+): string[] {
+  const protocols = [
+    GATEWAY_REALTIME_SUBPROTOCOL,
+    `${GATEWAY_AUTH_SUBPROTOCOL_PREFIX}${token}`,
+  ];
+
+  if (options?.teamIdOrSlug) {
+    protocols.push(
+      `${GATEWAY_TEAM_SUBPROTOCOL_PREFIX}${encodeSubprotocolValue(options.teamIdOrSlug)}`,
+    );
+  }
+
+  return protocols;
+}
+
+/**
+ * Server-side: extract the Gateway auth (bearer) token from a
+ * `Sec-WebSocket-Protocol` header value, or `undefined` when it is absent or
+ * empty. The Gateway upgrade handler turns this into an
+ * `Authorization: Bearer <token>` before its normal auth path.
+ *
+ * Accepts the raw header value (subprotocols are comma-separated and may carry
+ * surrounding whitespace).
+ */
+export function getGatewayRealtimeAuthToken(
+  secWebSocketProtocol: string | null | undefined,
+): string | undefined {
+  // `findProtocol` trims the protocol, so the sliced token needs no further
+  // trimming; `|| undefined` collapses the empty-token case.
+  return (
+    findProtocol(secWebSocketProtocol, GATEWAY_AUTH_SUBPROTOCOL_PREFIX)?.slice(
+      GATEWAY_AUTH_SUBPROTOCOL_PREFIX.length,
+    ) || undefined
+  );
+}
+
+/**
+ * Server-side: extract the optional Vercel team ID or slug from the
+ * `Sec-WebSocket-Protocol` header value. Team scoping is base64url-encoded so
+ * arbitrary team slugs stay within the WebSocket subprotocol token grammar.
+ */
+export function getGatewayRealtimeTeamIdOrSlug(
+  secWebSocketProtocol: string | null | undefined,
+): string | undefined {
+  const encoded = findProtocol(
+    secWebSocketProtocol,
+    GATEWAY_TEAM_SUBPROTOCOL_PREFIX,
+  )?.slice(GATEWAY_TEAM_SUBPROTOCOL_PREFIX.length);
+  if (!encoded) return undefined;
+
+  try {
+    return decodeSubprotocolValue(encoded) || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function findProtocol(
+  secWebSocketProtocol: string | null | undefined,
+  prefix: string,
+): string | undefined {
+  return secWebSocketProtocol
+    ?.split(',')
+    .map(protocol => protocol.trim())
+    .find(protocol => protocol.startsWith(prefix));
+}
+
+function encodeSubprotocolValue(value: string): string {
+  const bytes = new TextEncoder().encode(value);
+  let binary = '';
+  for (const byte of bytes) {
+    binary += String.fromCharCode(byte);
+  }
+  return btoa(binary)
+    .replace(/\+/g, '-')
+    .replace(/\//g, '_')
+    .replace(/=+$/u, '');
+}
+
+function decodeSubprotocolValue(value: string): string {
+  const base64 = value.replace(/-/g, '+').replace(/_/g, '/');
+  const padding = '='.repeat((4 - (base64.length % 4)) % 4);
+  const binary = atob(`${base64}${padding}`);
+  const bytes = Uint8Array.from(binary, char => char.charCodeAt(0));
+  return new TextDecoder().decode(bytes);
+}

package/src/gateway-realtime-model-settings.ts

@@ -0,0 +1,6 @@
+export type GatewayRealtimeModelId =
+  | 'openai/gpt-realtime-1.5'
+  | 'openai/gpt-realtime-2'
+  | 'openai/gpt-realtime-mini'
+  | 'xai/grok-voice-think-fast-1.0'
+  | (string & {});

package/src/gateway-realtime-model.ts

@@ -0,0 +1,118 @@
+import type {
+  Experimental_RealtimeModelV4 as RealtimeModelV4,
+  Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ClientSecretOptions as RealtimeModelV4ClientSecretOptions,
+  Experimental_RealtimeModelV4ClientSecretResult as RealtimeModelV4ClientSecretResult,
+  Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
+  Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+} from '@ai-sdk/provider';
+import { getGatewayRealtimeProtocols } from './gateway-realtime-auth';
+
+export type GatewayRealtimeModelConfig = {
+  provider: string;
+  baseURL: string;
+  teamIdOrSlug?: string;
+  /**
+   * Mints a short-lived client secret (`vcst_`) for this model via the
+   * Gateway's `/v1/realtime/client-secrets` endpoint. Implemented by the
+   * provider because minting requires the long-lived Gateway credential
+   * (API key / OIDC) and must run server-side.
+   */
+  createClientSecret: (params: {
+    modelId: string;
+    expiresAfterSeconds?: number;
+  }) => PromiseLike<{ token: string; expiresAt?: number }>;
+};
+
+/**
+ * Realtime model backed by the AI Gateway.
+ *
+ * The Gateway normalizes realtime exactly like it normalizes every other
+ * modality: the client speaks the normalized AI SDK realtime protocol and the
+ * Gateway translates to and from the upstream provider server-side. This model
+ * is therefore a thin identity codec over that normalized protocol — only the
+ * connection and authentication are Gateway-specific.
+ */
+export class GatewayRealtimeModel implements RealtimeModelV4 {
+  readonly specificationVersion = 'v4' as const;
+  readonly provider: string;
+  readonly modelId: string;
+
+  private readonly config: GatewayRealtimeModelConfig;
+
+  constructor(modelId: string, config: GatewayRealtimeModelConfig) {
+    this.modelId = modelId;
+    this.provider = config.provider;
+    this.config = config;
+  }
+
+  /**
+   * Mints a single-use, short-lived client secret (`vcst_`) the browser uses to
+   * open the realtime WebSocket without ever holding the long-lived Gateway
+   * credential. The customer's server calls this (via
+   * `gateway.experimental_realtime.getToken`) and hands the returned token to
+   * the browser, which connects with it through the `ai-gateway-auth.<token>`
+   * subprotocol. `expiresAfterSeconds` is forwarded to the mint endpoint;
+   * `sessionConfig` is intentionally unused here — it is applied later via the
+   * normalized `session-update` event.
+   */
+  async doCreateClientSecret(
+    options?: RealtimeModelV4ClientSecretOptions,
+  ): Promise<RealtimeModelV4ClientSecretResult> {
+    const secret = await this.config.createClientSecret({
+      modelId: this.modelId,
+      ...(options?.expiresAfterSeconds != null && {
+        expiresAfterSeconds: options.expiresAfterSeconds,
+      }),
+    });
+    return {
+      token: secret.token,
+      url: toGatewayRealtimeUrl(this.config.baseURL, this.modelId),
+      ...(secret.expiresAt != null && { expiresAt: secret.expiresAt }),
+    };
+  }
+
+  getWebSocketConfig(options: { token: string; url: string }): {
+    url: string;
+    protocols?: string[];
+  } {
+    return {
+      url: options.url,
+      protocols: getGatewayRealtimeProtocols(options.token, {
+        teamIdOrSlug: this.config.teamIdOrSlug,
+      }),
+    };
+  }
+
+  parseServerEvent(raw: unknown): RealtimeModelV4ServerEvent {
+    // The Gateway emits normalized AI SDK realtime events, so no
+    // provider-specific mapping is needed on the client.
+    return raw as RealtimeModelV4ServerEvent;
+  }
+
+  serializeClientEvent(event: RealtimeModelV4ClientEvent): unknown {
+    // The Gateway accepts normalized AI SDK realtime events directly.
+    return event;
+  }
+
+  buildSessionConfig(config: RealtimeModelV4SessionConfig): unknown {
+    // The session config is already normalized; the Gateway maps it to the
+    // upstream provider's session payload server-side.
+    return config;
+  }
+}
+
+/**
+ * Build the Gateway realtime WebSocket URL. The HTTP(S) base URL is upgraded to
+ * WS(S) and the model id rides the `?ai-model-id=` query — the WS transport of
+ * the `ai-model-id` header the HTTP routes use, since a browser `WebSocket`
+ * cannot set headers. The model id is passed through verbatim; the Gateway owns
+ * resolution (including the bare → `openai/` qualification), exactly like the
+ * non-realtime routes. The query is slash-safe for qualified ids such as
+ * `openai/gpt-realtime-2`.
+ */
+function toGatewayRealtimeUrl(baseURL: string, modelId: string): string {
+  const url = new URL(`${baseURL.replace(/^http/, 'ws')}/realtime-model`);
+  url.searchParams.set('ai-model-id', modelId);
+  return url.toString();
+}

package/src/gateway-reranking-model.ts

@@ -80,7 +80,7 @@ export class GatewayRerankingModel implements RerankingModelV4 {
         providerMetadata:
           responseBody.providerMetadata as unknown as SharedV4ProviderMetadata,
         response: { headers: responseHeaders, body: rawValue },
-        warnings: [],
+        warnings: responseBody.warnings ?? [],
       };
     } catch (error) {
       throw await asGatewayError(
@@ -102,6 +102,28 @@ export class GatewayRerankingModel implements RerankingModelV4 {
   }
 }
 
+const gatewayRerankingWarningSchema = z.discriminatedUnion('type', [
+  z.object({
+    type: z.literal('unsupported'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('compatibility'),
+    feature: z.string(),
+    details: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal('deprecated'),
+    setting: z.string(),
+    message: z.string(),
+  }),
+  z.object({
+    type: z.literal('other'),
+    message: z.string(),
+  }),
+]);
+
 const gatewayRerankingResponseSchema = lazySchema(() =>
   zodSchema(
     z.object({
@@ -111,6 +133,7 @@ const gatewayRerankingResponseSchema = lazySchema(() =>
           relevanceScore: z.number(),
         }),
       ),
+      warnings: z.array(gatewayRerankingWarningSchema).optional(),
       providerMetadata: z
         .record(z.string(), z.record(z.string(), z.unknown()))
         .optional(),

package/src/gateway-speech-model-settings.ts

@@ -1 +1,5 @@
-export type GatewaySpeechModelId = string & {};
+export type GatewaySpeechModelId =
+  | 'openai/tts-1'
+  | 'openai/tts-1-hd'
+  | 'xai/grok-tts'
+  | (string & {});

package/src/gateway-speech-model.ts

@@ -125,6 +125,11 @@ const gatewaySpeechWarningSchema = z.discriminatedUnion('type', [
     feature: z.string(),
     details: z.string().optional(),
   }),
+  z.object({
+    type: z.literal('deprecated'),
+    setting: z.string(),
+    message: z.string(),
+  }),
   z.object({
     type: z.literal('other'),
     message: z.string(),

package/src/gateway-tools.ts

@@ -1,3 +1,4 @@
+import { exaSearch } from './tool/exa-search';
 import { parallelSearch } from './tool/parallel-search';
 import { perplexitySearch } from './tool/perplexity-search';
 
@@ -5,6 +6,15 @@ import { perplexitySearch } from './tool/perplexity-search';
  * Gateway-specific provider-defined tools.
  */
 export const gatewayTools = {
+  /**
+   * Search the web using Exa for current information and token-efficient
+   * excerpts optimized for agent workflows.
+   *
+   * Supports search type, category, domain, date, location, and content
+   * extraction controls.
+   */
+  exaSearch,
+
   /**
    * Search the web using Parallel AI's Search API for LLM-optimized excerpts.
    *

package/src/gateway-transcription-model-settings.ts

@@ -1 +1,6 @@
-export type GatewayTranscriptionModelId = string & {};
+export type GatewayTranscriptionModelId =
+  | 'openai/gpt-4o-mini-transcribe'
+  | 'openai/gpt-4o-transcribe'
+  | 'openai/whisper-1'
+  | 'xai/grok-stt'
+  | (string & {});

package/src/gateway-transcription-model.ts

@@ -124,6 +124,11 @@ const gatewayTranscriptionWarningSchema = z.discriminatedUnion('type', [
     feature: z.string(),
     details: z.string().optional(),
   }),
+  z.object({
+    type: z.literal('deprecated'),
+    setting: z.string(),
+    message: z.string(),
+  }),
   z.object({
     type: z.literal('other'),
     message: z.string(),

package/src/gateway-video-model-settings.ts

@@ -7,8 +7,6 @@ export type GatewayVideoModelId =
   | 'alibaba/wan-v2.6-t2v'
   | 'bytedance/seedance-2.0'
   | 'bytedance/seedance-2.0-fast'
-  | 'bytedance/seedance-v1.0-lite-i2v'
-  | 'bytedance/seedance-v1.0-lite-t2v'
   | 'bytedance/seedance-v1.0-pro'
   | 'bytedance/seedance-v1.0-pro-fast'
   | 'bytedance/seedance-v1.5-pro'

package/src/gateway-video-model.ts

@@ -46,6 +46,7 @@ export class GatewayVideoModel implements Experimental_VideoModelV4 {
     duration,
     fps,
     seed,
+    generateAudio,
     image,
     providerOptions,
     headers,
@@ -81,6 +82,7 @@ export class GatewayVideoModel implements Experimental_VideoModelV4 {
           ...(duration && { duration }),
           ...(fps && { fps }),
           ...(seed && { seed }),
+          ...(generateAudio !== undefined && { generateAudio }),
           ...(providerOptions && { providerOptions }),
           ...(image && { image: maybeEncodeVideoFile(image) }),
         },
@@ -239,6 +241,11 @@ const gatewayVideoWarningSchema = z.discriminatedUnion('type', [
     feature: z.string(),
     details: z.string().optional(),
   }),
+  z.object({
+    type: z.literal('deprecated'),
+    setting: z.string(),
+    message: z.string(),
+  }),
   z.object({
     type: z.literal('other'),
     message: z.string(),

package/src/index.ts

@@ -1,4 +1,13 @@
 export type { GatewayModelId } from './gateway-language-model-settings';
+export {
+  GATEWAY_AUTH_SUBPROTOCOL_PREFIX,
+  GATEWAY_REALTIME_SUBPROTOCOL,
+  GATEWAY_TEAM_SUBPROTOCOL_PREFIX,
+  getGatewayRealtimeAuthToken,
+  getGatewayRealtimeProtocols,
+  getGatewayRealtimeTeamIdOrSlug,
+} from './gateway-realtime-auth';
+export type { GatewayRealtimeModelId } from './gateway-realtime-model-settings';
 export type { GatewayRerankingModelId } from './gateway-reranking-model-settings';
 export type { GatewaySpeechModelId } from './gateway-speech-model-settings';
 export type { GatewayTranscriptionModelId } from './gateway-transcription-model-settings';
@@ -36,6 +45,8 @@ export type {
 export {
   GatewayError,
   GatewayAuthenticationError,
+  GatewayFailedDependencyError,
+  GatewayForbiddenError,
   GatewayInvalidRequestError,
   GatewayRateLimitError,
   GatewayModelNotFoundError,