@ai-sdk/gateway 4.0.94 → 4.0.96

package/docs/00-ai-gateway.mdx

@@ -1472,12 +1472,16 @@ The following gateway provider options are available:
 
   When `sort` is active, the response's `providerMetadata.gateway.routing.sort` object contains the sort option used, the resulting execution order, per-provider metric values, and any providers that were deprioritized.
 
-- **models** _string[]_
+- **models** _string[] | [GatewayModelFallback, ...string[]]_
 
-  Specifies fallback models to use when the primary model fails or is unavailable. The gateway will try the primary model first (specified in the `model` parameter), then try each model in this array in order until one succeeds.
+  Specifies fallback models in attempt order. String entries keep their existing behavior: the gateway tries them when the primary model fails or is unavailable. Evaluation requests can also start the list with one conditional `{ model, when }` entry to retry a successful but uncertain evaluation with another model. Only the first entry can be conditional, and only one conditional entry is allowed. If the primary model fails, the gateway also advances to the conditional entry under its existing error fallback behavior. Conditional entries are rejected on non-evaluation requests.
 
   Example: `models: ['openai/gpt-5.4-nano', 'google/gemini-3.8-flash']` will try the fallback models in order if the primary model fails.
 
+  A direct condition uses `{ question, confidenceBelow }` for Choice and Score questions, or `{ question, probabilityBetween: [minimum, maximum] }` for Boolean questions. Boolean probability is P(true), not confidence in the selected Boolean outcome. Bounds are inclusive, finite, ordered, and within `[0, 1]`.
+
+  Combine conditions with `{ any: [...] }`, `{ all: [...] }`, or `{ atLeast: { count, conditions: [...] } }`. Each condition list holds 1 to 20 conditions. `atLeast.count` must be an integer from `1` through the number of conditions. Conditions nest at most 5 levels deep. Question IDs are 1 to 256 characters, and the conditional `model` must not be empty. The SDK checks these bounds before sending the request, and the gateway also checks that each question exists and has a kind that matches its condition. String error fallbacks may follow the conditional entry.
+
 - **user** _string_
 
   Optional identifier for the end user on whose behalf the request is being made. This is used for spend tracking and attribution purposes, allowing you to track usage per end-user in your application.
@@ -1608,6 +1612,63 @@ const { text } = await generateText({
 // 4. Return the result from the first model that succeeds
 ```
 
+#### Conditional Evaluation Fallbacks Example
+
+Conditional model entries are available only for evaluation requests. Keep the
+configuration under `providerOptions.gateway.models`. `experimental_evaluate`
+does not have a top-level fallback option.
+
+```ts
+import type { GatewayProviderOptions } from '@ai-sdk/gateway';
+import { experimental_evaluate } from 'ai';
+
+const questions = {
+  department: {
+    type: 'choice',
+    instructions: 'Which team should handle this?',
+    criteria: { billing: 'Charges and refunds', support: 'Other requests' },
+  },
+  requestsRefund: {
+    type: 'boolean',
+    instructions: 'Is the customer requesting money back?',
+  },
+} as const;
+
+const result = await experimental_evaluate({
+  model: 'typesafe-ai/jev',
+  state: 'I was charged twice. Please refund the duplicate.',
+  questions,
+  providerOptions: {
+    gateway: {
+      models: [
+        {
+          model: 'openai/gpt-5.6-sol',
+          when: {
+            any: [
+              { question: 'department', confidenceBelow: 0.6 },
+              {
+                question: 'requestsRefund',
+                probabilityBetween: [0.4, 0.6],
+              },
+            ],
+          },
+        },
+      ],
+    } satisfies GatewayProviderOptions,
+  },
+});
+
+// The model that produced the answers, the fallback when the condition matched.
+console.log(result.response.modelId);
+console.log(JSON.stringify(result.providerMetadata?.gateway?.routing, null, 2));
+```
+
+When the condition matches, `result.response.modelId` names the fallback model,
+and `providerMetadata.gateway.routing.modelAttempts` lists the primary and
+fallback attempts. The successful attempt of each stage carries its own
+`generationId`, `usage`, and cost fields, and the first fallback attempt lists
+the questions that triggered it in `triggeredBy`.
+
 #### Zero Data Retention Example
 
 Set `zeroDataRetention` to true to route requests to providers with zero data retention agreements with Vercel for AI Gateway. BYOK credentials are skipped by default, since your provider agreements differ from Vercel's. When this filter is on, AI Gateway routes only to providers Vercel has ZDR agreements with for the model. If you have BYOK keys marked as ZDR, those keys are tried first, then AI Gateway falls back to its system credentials. You are responsible for the accuracy of that marking. Applies to both account-wide and request-level ZDR. The request fails if no ZDR-eligible credentials are available. When `zeroDataRetention` is `false` or not specified, there is no enforcement of restricting routing. Request-level ZDR is only available for Vercel Pro and Enterprise plans.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ai-sdk/gateway",
   "private": false,
-  "version": "4.0.94",
+  "version": "4.0.96",
   "type": "module",
   "license": "Apache-2.0",
   "sideEffects": false,

package/src/gateway-evaluation-model.ts

@@ -8,6 +8,7 @@ import {
   createJsonResponseHandler,
   getErrorMessage,
   lazySchema,
+  parseProviderOptions,
   postJsonToApi,
   resolve,
   zodSchema,
@@ -17,6 +18,7 @@ import { z } from './zod';
 import { asGatewayError } from './errors';
 import { parseAuthMethod } from './errors/parse-auth-method';
 import type { GatewayConfig } from './gateway-config';
+import { gatewayEvaluationProviderOptionsSchema } from './gateway-provider-options';
 
 export class GatewayEvaluationModel implements EvaluationModelV4 {
   readonly specificationVersion = 'v4';
@@ -44,6 +46,15 @@ export class GatewayEvaluationModel implements EvaluationModelV4 {
   }: Parameters<EvaluationModelV4['doEvaluate']>[0]): Promise<
     Awaited<ReturnType<EvaluationModelV4['doEvaluate']>>
   > {
+    const gatewayOptions = await parseProviderOptions({
+      provider: 'gateway',
+      providerOptions,
+      schema: gatewayEvaluationProviderOptionsSchema,
+    });
+    const validatedProviderOptions =
+      gatewayOptions == null
+        ? providerOptions
+        : { ...providerOptions, gateway: gatewayOptions };
     const resolvedHeaders = this.config.headers
       ? await resolve(this.config.headers)
       : undefined;
@@ -63,7 +74,9 @@ export class GatewayEvaluationModel implements EvaluationModelV4 {
         body: {
           state,
           questions,
-          ...(providerOptions ? { providerOptions } : {}),
+          ...(validatedProviderOptions
+            ? { providerOptions: validatedProviderOptions }
+            : {}),
         },
         successfulResponseHandler: createJsonResponseHandler(
           gatewayEvaluationResponseSchema,
@@ -84,7 +97,7 @@ export class GatewayEvaluationModel implements EvaluationModelV4 {
         providerMetadata:
           responseBody.providerMetadata as unknown as SharedV4ProviderMetadata,
         response: {
-          modelId: this.modelId,
+          modelId: responseBody.model ?? this.modelId,
           headers: responseHeaders,
           body: rawValue,
         },
@@ -152,6 +165,7 @@ const gatewayEvaluationResponseSchema = lazySchema(() =>
   zodSchema(
     z.object({
       answers: z.record(z.string(), gatewayEvaluationAnswerSchema),
+      model: z.string().optional(),
       rounding: z
         .object({
           probabilityDecimals: z.number().optional(),

package/src/gateway-provider-options.ts

@@ -1,5 +1,50 @@
+import { lazySchema, zodSchema } from '@ai-sdk/provider-utils';
+import type { ZodType } from 'zod/v4';
+import { z } from './zod';
+
 // https://vercel.com/docs/ai-gateway/provider-options
-export type GatewayProviderOptions = {
+export const EVALUATION_FALLBACK_MAX_CONDITION_DEPTH = 5;
+export const EVALUATION_FALLBACK_MAX_CONDITIONS_PER_LIST = 20;
+export const EVALUATION_FALLBACK_MAX_QUESTION_LENGTH = 256;
+
+export const gatewayEvaluationProviderOptionsSchema = lazySchema(() =>
+  zodSchema(
+    z
+      .object({
+        models: gatewayModelFallbacksSchema.optional(),
+      })
+      .catchall(z.unknown()),
+  ),
+);
+
+/**
+ * A condition on the primary model's answers. `QUESTION_ID` narrows
+ * `question` to your question IDs. Groups nest at most five levels deep,
+ * which the SDK checks at runtime.
+ */
+export type EvaluationFallbackCondition<QUESTION_ID extends string = string> =
+  | ExclusiveCondition<{ question: QUESTION_ID; confidenceBelow: number }>
+  | ExclusiveCondition<{
+      question: QUESTION_ID;
+      probabilityBetween: [number, number];
+    }>
+  | ExclusiveCondition<{ any: EvaluationFallbackConditionList<QUESTION_ID> }>
+  | ExclusiveCondition<{ all: EvaluationFallbackConditionList<QUESTION_ID> }>
+  | ExclusiveCondition<{
+      atLeast: {
+        count: number;
+        conditions: EvaluationFallbackConditionList<QUESTION_ID>;
+      };
+    }>;
+
+export type GatewayModelFallback<QUESTION_ID extends string = string> =
+  | string
+  | {
+      model: string;
+      when: EvaluationFallbackCondition<QUESTION_ID>;
+    };
+
+export type GatewayProviderOptions<QUESTION_ID extends string = string> = {
   /**
    * Service-owned options may be added by the Gateway without requiring an SDK
    * release. The Gateway service validates and applies the runtime schema.
@@ -43,8 +88,14 @@ export type GatewayProviderOptions = {
    */
   idempotencyKey?: string;
 
-  /** Array of model slugs specifying fallback models to use in order. */
-  models?: string[];
+  /**
+   * Fallback models to try in order. On evaluation requests, the first entry
+   * can be a conditional `{ model, when }` fallback that reruns the evaluation
+   * when `when` matches the primary answers. Other request types reject a
+   * conditional entry. Pass your question IDs as `QUESTION_ID` to check the
+   * `question` names in `when`.
+   */
+  models?: GatewayModelFallbackList<QUESTION_ID>;
 
   /** Array of provider slugs that are the only ones allowed to be used. */
   only?: string[];
@@ -75,3 +126,133 @@ export type GatewayProviderOptions = {
   /** Filter to providers with zero data retention agreements. */
   zeroDataRetention?: boolean;
 };
+
+type EvaluationFallbackConditionList<QUESTION_ID extends string> = [
+  EvaluationFallbackCondition<QUESTION_ID>,
+  ...EvaluationFallbackCondition<QUESTION_ID>[],
+];
+
+type ConditionKey =
+  | 'question'
+  | 'confidenceBelow'
+  | 'probabilityBetween'
+  | 'any'
+  | 'all'
+  | 'atLeast';
+
+// Rules out the other shapes' keys, so a condition can't mix two shapes.
+type ExclusiveCondition<CONDITION> = CONDITION & {
+  [KEY in Exclude<ConditionKey, keyof CONDITION>]?: never;
+};
+
+type ConditionalGatewayModelFallback<QUESTION_ID extends string> = Exclude<
+  GatewayModelFallback<QUESTION_ID>,
+  string
+>;
+
+type GatewayModelFallbackList<QUESTION_ID extends string> =
+  | string[]
+  | [ConditionalGatewayModelFallback<QUESTION_ID>, ...string[]];
+
+const probabilitySchema = z.number().finite().min(0).max(1);
+const questionSchema = z
+  .string()
+  .min(1)
+  .max(EVALUATION_FALLBACK_MAX_QUESTION_LENGTH);
+const directConditionSchema = z.union([
+  z
+    .object({
+      question: questionSchema,
+      confidenceBelow: probabilitySchema,
+    })
+    .strict(),
+  z
+    .object({
+      question: questionSchema,
+      probabilityBetween: z
+        .array(probabilitySchema)
+        .length(2)
+        .refine(([minimum, maximum]) => minimum <= maximum, {
+          message:
+            'probabilityBetween minimum must be less than or equal to maximum',
+        }),
+    })
+    .strict(),
+]) as ZodType<EvaluationFallbackCondition>;
+
+const groupBeyondMaxDepthSchema = z
+  .union([
+    z.object({ any: z.unknown() }),
+    z.object({ all: z.unknown() }),
+    z.object({ atLeast: z.unknown() }),
+  ])
+  .superRefine((_, context) => {
+    context.addIssue({
+      code: 'custom',
+      message: `conditions can be nested at most ${EVALUATION_FALLBACK_MAX_CONDITION_DEPTH} levels deep`,
+    });
+  });
+
+const conditionalModelFallbackSchema = z
+  .object({
+    model: z.string().min(1),
+    when: conditionSchema(1),
+  })
+  .strict();
+
+const gatewayModelFallbacksSchema = z
+  .array(z.union([z.string(), conditionalModelFallbackSchema]))
+  .superRefine((entries, context) => {
+    const conditionalIndexes = entries.flatMap((entry, index) =>
+      typeof entry === 'string' ? [] : [index],
+    );
+    if (conditionalIndexes.length > 1) {
+      context.addIssue({
+        code: 'custom',
+        message: 'models supports at most one conditional evaluation fallback',
+      });
+    }
+    if (conditionalIndexes[0] !== undefined && conditionalIndexes[0] !== 0) {
+      context.addIssue({
+        code: 'custom',
+        message:
+          'a conditional evaluation fallback must be the first models entry',
+        path: [conditionalIndexes[0]],
+      });
+    }
+  });
+
+function conditionSchema(depth: number): ZodType<EvaluationFallbackCondition> {
+  if (depth === EVALUATION_FALLBACK_MAX_CONDITION_DEPTH) {
+    return z.union([
+      directConditionSchema,
+      groupBeyondMaxDepthSchema,
+    ]) as ZodType<EvaluationFallbackCondition>;
+  }
+
+  const childConditionSchema = conditionSchema(depth + 1);
+  const conditionListSchema = z
+    .array(childConditionSchema)
+    .min(1)
+    .max(EVALUATION_FALLBACK_MAX_CONDITIONS_PER_LIST);
+
+  return z.union([
+    directConditionSchema,
+    z.object({ any: conditionListSchema }).strict(),
+    z.object({ all: conditionListSchema }).strict(),
+    z
+      .object({
+        atLeast: z
+          .object({
+            count: z.number().int().min(1),
+            conditions: conditionListSchema,
+          })
+          .strict()
+          .refine(({ count, conditions }) => count <= conditions.length, {
+            message: 'atLeast count cannot exceed the number of conditions',
+            path: ['count'],
+          }),
+      })
+      .strict(),
+  ]) as ZodType<EvaluationFallbackCondition>;
+}

package/src/index.ts

@@ -47,6 +47,8 @@ export type {
   GatewayProviderMetadata,
 } from './gateway-provider-metadata';
 export type {
+  EvaluationFallbackCondition,
+  GatewayModelFallback,
   GatewayProviderOptions,
   /** @deprecated Use `GatewayProviderOptions` instead. */
   GatewayProviderOptions as GatewayLanguageModelOptions,