ai 7.0.102 → 7.0.104

package/docs/07-reference/05-ai-sdk-errors/ai-evaluation-unsupported-question-type-error.mdx

@@ -0,0 +1,31 @@
+---
+title: AI_EvaluationUnsupportedQuestionTypeError
+description: Identify an evaluation question type that a model does not support.
+---
+
+# AI_EvaluationUnsupportedQuestionTypeError
+
+This experimental error identifies a question whose type is unsupported by an
+evaluation model. It is exported from `ai` and `@ai-sdk/provider` as
+`Experimental_EvaluationUnsupportedQuestionTypeError`.
+
+## Properties
+
+- `questionId`: The ID of the unsupported question.
+- `questionType`: The requested question type.
+- `provider`: The provider of the evaluation model.
+- `modelId`: The evaluation model ID.
+- `message`: A description of the unsupported question and model. Providers can
+  supply a custom message.
+
+## Checking for this Error
+
+Use the marker-based `isInstance` check, which works across package copies:
+
+```typescript
+import { Experimental_EvaluationUnsupportedQuestionTypeError as EvaluationUnsupportedQuestionTypeError } from 'ai';
+
+if (EvaluationUnsupportedQuestionTypeError.isInstance(error)) {
+  console.log(error.questionId, error.questionType, error.modelId);
+}
+```

package/docs/07-reference/05-ai-sdk-errors/ai-no-such-model-error.mdx

@@ -24,3 +24,7 @@ if (NoSuchModelError.isInstance(error)) {
   // Handle the error
 }
 ```
+
+Experimental evaluation resolution uses `modelType: 'evaluationModel'`. This
+includes unavailable evaluation capabilities and unknown evaluation model or
+provider IDs. See [Evaluation](/docs/ai-sdk-core/evaluation#default-provider-strings).

package/docs/07-reference/05-ai-sdk-errors/ai-no-such-provider-error.mdx

@@ -26,3 +26,7 @@ if (NoSuchProviderError.isInstance(error)) {
   // Handle the error
 }
 ```
+
+Experimental evaluation resolution uses `modelType: 'evaluationModel'`. This
+includes unavailable evaluation capabilities and unknown evaluation model or
+provider IDs. See [Evaluation](/docs/ai-sdk-core/evaluation#default-provider-strings).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "7.0.102",
+  "version": "7.0.104",
   "type": "module",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
@@ -42,20 +42,20 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/gateway": "4.0.82",
-    "@ai-sdk/provider": "4.0.15",
-    "@ai-sdk/provider-utils": "5.0.41"
+    "@ai-sdk/gateway": "4.0.84",
+    "@ai-sdk/provider": "4.0.17",
+    "@ai-sdk/provider-utils": "5.0.43"
   },
   "devDependencies": {
-    "@ai-sdk/amazon-bedrock": "5.0.83",
-    "@ai-sdk/deepseek": "3.0.45",
-    "@ai-sdk/google": "4.0.71",
-    "@ai-sdk/groq": "4.0.42",
-    "@ai-sdk/huggingface": "2.0.49",
-    "@ai-sdk/moonshotai": "3.0.50",
-    "@ai-sdk/openai": "4.0.67",
+    "@ai-sdk/amazon-bedrock": "5.0.86",
+    "@ai-sdk/deepseek": "3.0.47",
+    "@ai-sdk/google": "4.0.74",
+    "@ai-sdk/groq": "4.0.44",
+    "@ai-sdk/huggingface": "2.0.51",
+    "@ai-sdk/moonshotai": "3.0.52",
+    "@ai-sdk/openai": "4.0.69",
     "@ai-sdk/test-server": "2.0.1",
-    "@ai-sdk/xai": "4.0.59",
+    "@ai-sdk/xai": "5.0.2",
     "@edge-runtime/vm": "^5.0.0",
     "@smithy/eventstream-codec": "^4.3.3",
     "@smithy/util-utf8": "^4.3.3",

package/src/error/index.ts

@@ -2,6 +2,7 @@ export {
   AISDKError,
   APICallError,
   EmptyResponseBodyError,
+  Experimental_EvaluationUnsupportedQuestionTypeError,
   InvalidPromptError,
   InvalidResponseDataError,
   JSONParseError,

package/src/evaluate/evaluate.ts

@@ -0,0 +1,106 @@
+import {
+  Experimental_EvaluationUnsupportedQuestionTypeError as EvaluationUnsupportedQuestionTypeError,
+  type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
+} from '@ai-sdk/provider';
+import {
+  withUserAgentSuffix,
+  type ProviderOptions,
+} from '@ai-sdk/provider-utils';
+import { resolveEvaluationModel } from '../model/resolve-model';
+import { logWarnings } from '../logger/log-warnings';
+import { prepareRetries } from '../util/prepare-retries';
+import { VERSION } from '../version';
+import type {
+  EvaluationModel,
+  EvaluationQuestion,
+  EvaluationResult,
+} from './evaluation-result';
+import {
+  validateEvaluationInput,
+  validateEvaluationAnswers,
+} from './validate-evaluation';
+
+/** Evaluate typed questions against one shared state. Experimental. */
+export async function evaluate<
+  const QUESTIONS extends Record<string, EvaluationQuestion>,
+>({
+  model: modelArg,
+  state,
+  questions,
+  maxRetries,
+  abortSignal,
+  headers,
+  providerOptions = {},
+}: {
+  /** An evaluation model instance or an ID resolved by the configured default provider. */
+  model: EvaluationModel;
+  state: EvaluationModelV4CallOptions['state'];
+  questions: QUESTIONS;
+  /** Maximum retries for transient provider failures. Defaults to 2. */
+  maxRetries?: number;
+  abortSignal?: AbortSignal;
+  headers?: Record<string, string>;
+  providerOptions?: ProviderOptions;
+}): Promise<EvaluationResult<QUESTIONS>> {
+  const model = resolveEvaluationModel(modelArg);
+
+  validateEvaluationInput({ state, questions });
+
+  for (const [questionId, question] of Object.entries(questions)) {
+    if (!model.supportedQuestionTypes.includes(question.type)) {
+      throw new EvaluationUnsupportedQuestionTypeError({
+        questionId,
+        questionType: question.type,
+        provider: model.provider,
+        modelId: model.modelId,
+      });
+    }
+  }
+
+  const { retry } = prepareRetries({ maxRetries, abortSignal });
+  const result = await retry(() => {
+    abortSignal?.throwIfAborted();
+    return model.doEvaluate({
+      state,
+      questions,
+      abortSignal,
+      headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
+      providerOptions,
+    });
+  });
+
+  abortSignal?.throwIfAborted();
+  validateEvaluationAnswers({
+    questions,
+    answers: result.answers,
+    rounding: result.rounding,
+  });
+  logWarnings({
+    warnings: result.warnings,
+    provider: model.provider,
+    model: model.modelId,
+  });
+
+  const inputTokens = result.usage?.inputTokens;
+  const outputTokens = result.usage?.outputTokens;
+
+  return {
+    answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
+    usage: {
+      inputTokens,
+      outputTokens,
+      totalTokens:
+        inputTokens != null && outputTokens != null
+          ? inputTokens + outputTokens
+          : undefined,
+    },
+    warnings: result.warnings,
+    rounding: result.rounding,
+    providerMetadata: result.providerMetadata,
+    response: {
+      ...result.response,
+      timestamp: result.response?.timestamp ?? new Date(),
+      modelId: result.response?.modelId ?? model.modelId,
+    },
+  };
+}

package/src/evaluate/evaluation-provider.ts

@@ -0,0 +1,6 @@
+import type { Experimental_EvaluationModelV4 as EvaluationModelV4 } from '@ai-sdk/provider';
+
+/** Structural extension; evaluation is not part of the stable provider contract. */
+export type EvaluationProvider = {
+  evaluationModel?: (modelId: string) => EvaluationModelV4;
+};

package/src/evaluate/evaluation-result.ts

@@ -0,0 +1,39 @@
+import type {
+  Experimental_EvaluationModelV4 as EvaluationModelV4,
+  Experimental_EvaluationModelV4Question as EvaluationModelV4Question,
+  Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
+} from '@ai-sdk/provider';
+
+export type EvaluationModel = string | EvaluationModelV4;
+export type EvaluationQuestion = EvaluationModelV4Question;
+
+export type EvaluationAnswer<QUESTION extends EvaluationQuestion> =
+  QUESTION extends { type: 'choice'; criteria: infer CRITERIA }
+    ? {
+        type: 'choice';
+        choice: Extract<keyof CRITERIA, string>;
+        probabilities?: Record<Extract<keyof CRITERIA, string>, number>;
+      }
+    : QUESTION extends { type: 'score' }
+      ? { type: 'score'; score: number; probabilities?: Record<string, number> }
+      : { type: 'boolean'; probability: number };
+
+export type EvaluationResult<
+  QUESTIONS extends Record<string, EvaluationQuestion>,
+> = {
+  readonly answers: {
+    [ID in keyof QUESTIONS]: EvaluationAnswer<QUESTIONS[ID]>;
+  };
+  readonly usage: {
+    inputTokens: number | undefined;
+    outputTokens: number | undefined;
+    totalTokens: number | undefined;
+  };
+  readonly warnings: EvaluationModelV4Result['warnings'];
+  readonly rounding: EvaluationModelV4Result['rounding'];
+  readonly providerMetadata: EvaluationModelV4Result['providerMetadata'];
+  readonly response: NonNullable<EvaluationModelV4Result['response']> & {
+    timestamp: Date;
+    modelId: string;
+  };
+};

package/src/evaluate/index.ts

@@ -0,0 +1,7 @@
+export { evaluate as experimental_evaluate } from './evaluate';
+export type {
+  EvaluationModel as Experimental_EvaluationModel,
+  EvaluationQuestion as Experimental_EvaluationQuestion,
+  EvaluationAnswer as Experimental_EvaluationAnswer,
+  EvaluationResult as Experimental_EvaluationResult,
+} from './evaluation-result';

package/src/evaluate/validate-evaluation.ts

@@ -0,0 +1,298 @@
+import {
+  InvalidResponseDataError,
+  type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
+  type Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
+} from '@ai-sdk/provider';
+import { InvalidArgumentError } from '../error/invalid-argument-error';
+
+// Absolute tolerance for sums and means. Never renormalize provider output.
+const tolerance = 1e-6;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  if (value == null || typeof value !== 'object' || Array.isArray(value))
+    return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+function isJSON(value: unknown, ancestors = new Set<object>()): boolean {
+  if (value === null || typeof value === 'string' || typeof value === 'boolean')
+    return true;
+  if (typeof value === 'number') return Number.isFinite(value);
+  if (typeof value !== 'object' || (!Array.isArray(value) && !isRecord(value)))
+    return false;
+  if (ancestors.has(value)) return false;
+  ancestors.add(value);
+  const valid =
+    Object.getOwnPropertySymbols(value).length === 0 &&
+    (Array.isArray(value)
+      ? Array.from(value).every(item => isJSON(item, ancestors))
+      : Object.values(value).every(item => isJSON(item, ancestors)));
+  ancestors.delete(value);
+  return valid;
+}
+
+function isInput(value: unknown): boolean {
+  return (
+    (typeof value === 'string' || Array.isArray(value) || isRecord(value)) &&
+    isJSON(value)
+  );
+}
+
+function invalidInput(
+  parameter: string,
+  value: unknown,
+  message: string,
+): never {
+  throw new InvalidArgumentError({ parameter, value, message });
+}
+
+export function validateEvaluationInput({
+  state,
+  questions,
+}: EvaluationModelV4CallOptions) {
+  if (!isInput(state)) {
+    invalidInput(
+      'state',
+      state,
+      'must be a JSON-compatible string, object, or array',
+    );
+  }
+  if (!isRecord(questions) || Object.keys(questions).length === 0) {
+    invalidInput('questions', questions, 'must be a nonempty question map');
+  }
+
+  for (const [id, question] of Object.entries(questions)) {
+    const parameter = `questions.${id}`;
+    if (!isRecord(question) || !isInput(question.instructions)) {
+      invalidInput(
+        parameter,
+        question,
+        'instructions must be a JSON-compatible string, object, or array',
+      );
+    }
+    const criteria = question.criteria;
+    switch (question.type) {
+      case 'choice':
+        if (!isRecord(criteria) || Object.keys(criteria).length === 0) {
+          invalidInput(
+            parameter,
+            question,
+            'choice criteria must be a nonempty option map',
+          );
+        }
+        break;
+      case 'score':
+        if (!Array.isArray(criteria) || criteria.length < 2) {
+          invalidInput(
+            parameter,
+            question,
+            'score criteria must contain at least two ordered levels',
+          );
+        }
+        break;
+      case 'boolean':
+        if (criteria === undefined) continue;
+        if (
+          !isRecord(criteria) ||
+          Object.keys(criteria).some(key => key !== 'true' && key !== 'false')
+        ) {
+          invalidInput(
+            parameter,
+            question,
+            'boolean criteria may only describe true and false',
+          );
+        }
+        break;
+      default:
+        invalidInput(
+          parameter,
+          question,
+          'question type must be choice, score, or boolean',
+        );
+    }
+    if (
+      !isJSON(criteria) ||
+      Object.values(criteria).some(value => value !== null && !isInput(value))
+    ) {
+      invalidInput(
+        parameter,
+        question,
+        'criteria descriptions must be JSON-compatible strings, objects, arrays, or null',
+      );
+    }
+  }
+}
+
+function invalidAnswer(answers: unknown, message: string): never {
+  throw new InvalidResponseDataError({ data: answers, message });
+}
+
+function isProbability(value: unknown): value is number {
+  return (
+    typeof value === 'number' &&
+    Number.isFinite(value) &&
+    value >= 0 &&
+    value <= 1
+  );
+}
+
+function hasExactKeys(value: Record<string, unknown>, keys: string[]): boolean {
+  return (
+    Object.keys(value).length === keys.length &&
+    keys.every(key => Object.hasOwn(value, key))
+  );
+}
+
+function validateDistribution(
+  value: unknown,
+  keys: string[],
+  answers: unknown,
+  id: string,
+  roundingError: number,
+): asserts value is Record<string, number> {
+  if (
+    !isRecord(value) ||
+    !hasExactKeys(value, keys) ||
+    !Object.values(value).every(isProbability)
+  ) {
+    invalidAnswer(
+      answers,
+      `Question "${id}" must have a complete distribution of finite probabilities in [0, 1].`,
+    );
+  }
+  const sum = Object.values(value).reduce<number>(
+    (total, probability) => total + (probability as number),
+    0,
+  );
+  if (Math.abs(sum - 1) > tolerance + keys.length * roundingError) {
+    invalidAnswer(
+      answers,
+      `Question "${id}" probabilities must sum to 1 within the declared rounding precision.`,
+    );
+  }
+}
+
+export function validateEvaluationAnswers({
+  questions,
+  answers,
+  rounding,
+}: {
+  questions: EvaluationModelV4CallOptions['questions'];
+  answers: unknown;
+  rounding?: EvaluationModelV4Result['rounding'];
+}) {
+  function roundingError(decimals: number | undefined): number {
+    if (decimals === undefined) return 0;
+    if (!Number.isInteger(decimals) || decimals < 0 || decimals > 15) {
+      invalidAnswer(
+        answers,
+        'Evaluation rounding decimals must be integers between 0 and 15.',
+      );
+    }
+    return 0.5 * 10 ** -decimals;
+  }
+  const probabilityError = roundingError(rounding?.probabilityDecimals);
+  const scoreError = roundingError(rounding?.scoreDecimals);
+  if (!isRecord(answers) || !hasExactKeys(answers, Object.keys(questions))) {
+    invalidAnswer(
+      answers,
+      'Evaluation must return exactly one answer for every question.',
+    );
+  }
+
+  for (const [id, question] of Object.entries(questions)) {
+    const answer = answers[id];
+    if (!isRecord(answer) || answer.type !== question.type) {
+      invalidAnswer(
+        answers,
+        `Question "${id}" returned an answer with the wrong type.`,
+      );
+    }
+
+    switch (question.type) {
+      case 'choice': {
+        if (
+          typeof answer.choice !== 'string' ||
+          !Object.hasOwn(question.criteria, answer.choice)
+        ) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" selected an unknown option.`,
+          );
+        }
+        if (answer.probabilities !== undefined) {
+          validateDistribution(
+            answer.probabilities,
+            Object.keys(question.criteria),
+            answers,
+            id,
+            probabilityError,
+          );
+          const selected = answer.probabilities[answer.choice];
+          if (
+            Object.values(answer.probabilities).some(
+              probability => probability > selected + tolerance,
+            )
+          ) {
+            invalidAnswer(
+              answers,
+              `Question "${id}" did not select a highest-probability option.`,
+            );
+          }
+        }
+        break;
+      }
+      case 'score': {
+        if (
+          typeof answer.score !== 'number' ||
+          !Number.isFinite(answer.score) ||
+          answer.score < 0 ||
+          answer.score > question.criteria.length - 1
+        ) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" score must be in [0, ${question.criteria.length - 1}].`,
+          );
+        }
+        if (answer.probabilities !== undefined) {
+          const keys = question.criteria.map((_, index) => String(index));
+          validateDistribution(
+            answer.probabilities,
+            keys,
+            answers,
+            id,
+            probabilityError,
+          );
+          const mean = Object.entries(answer.probabilities).reduce(
+            (total, [index, probability]) =>
+              total + Number(index) * probability,
+            0,
+          );
+          const meanRoundingError = keys.reduce(
+            (total, index) => total + Number(index) * probabilityError,
+            0,
+          );
+          if (
+            Math.abs(mean - answer.score) >
+            tolerance + meanRoundingError + scoreError
+          ) {
+            invalidAnswer(
+              answers,
+              `Question "${id}" score must equal the probability-weighted mean within the declared rounding precision.`,
+            );
+          }
+        }
+        break;
+      }
+      case 'boolean':
+        if (!isProbability(answer.probability)) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" must return P(true) as a finite probability in [0, 1].`,
+          );
+        }
+        break;
+    }
+  }
+}

package/src/generate-text/generate-text.ts

@@ -63,6 +63,7 @@ import type { ActiveTools } from './active-tools';
 import { calculateTokensPerSecond } from './calculate-tokens-per-second';
 import { collectToolApprovals } from './collect-tool-approvals';
 import { convertLanguageModelContent } from './convert-language-model-content';
+import { createToolSearchState } from '../tool-search/prepare-tool-search';
 import { executeToolCall } from './execute-tool-call';
 import {
   filterActiveTools,
@@ -106,6 +107,7 @@ import type { ToolApprovalConfiguration } from './tool-approval-configuration';
 import type { ToolApprovalRequestOutput } from './tool-approval-request-output';
 import type { ToolApprovalResponseOutput } from './tool-approval-response-output';
 import {
+  appendToolCallerMessages,
   prepareToolsForToolCallers,
   resolveToolCallerConfiguration,
   type Experimental_ToolCallers,
@@ -574,6 +576,10 @@ export async function generateText<
     tools,
     toolCallers: experimental_toolCallers,
   });
+  const prepareToolSearch = createToolSearchState({
+    tools,
+    toolCallers: resolvedToolCallers,
+  });
   const stopConditions = asArray(stopWhen);
   const resolvedOnStart = onStart ?? experimental_onStart;
   const resolvedOnStepStart = onStepStart ?? experimental_onStepStart;
@@ -906,16 +912,6 @@ export async function generateText<
                 prepareStepResult?.system ??
                 instructionsForNextStep;
 
-              const promptMessages = await convertToLanguageModelPrompt({
-                prompt: {
-                  instructions: stepInstructions,
-                  messages: prepareStepResult?.messages ?? stepInputMessages,
-                },
-                supportedUrls: await stepModel.supportedUrls,
-                download,
-                provider: stepModel.provider.split('.')[0],
-              });
-
               runtimeContext =
                 prepareStepResult?.runtimeContext ?? runtimeContext;
               toolsContext = prepareStepResult?.toolsContext ?? toolsContext;
@@ -927,8 +923,12 @@ export async function generateText<
               const {
                 executionTools: stepExecutionTools,
                 modelTools: stepModelTools,
+                toolCallerMessages,
               } = prepareToolsForToolCallers({
-                tools: stepActiveTools,
+                tools: prepareToolSearch(stepActiveTools, {
+                  toolsContext,
+                  experimental_sandbox: stepSandbox,
+                }),
                 toolCallers: resolvedToolCallers,
               });
               const stepToolOrder = prepareStepResult?.toolOrder ?? toolOrder;
@@ -952,8 +952,20 @@ export async function generateText<
                 toolChoice: prepareStepResult?.toolChoice ?? toolChoice,
               });
 
-              const stepMessages =
-                prepareStepResult?.messages ?? stepInputMessages;
+              const stepMessages = appendToolCallerMessages({
+                messages: prepareStepResult?.messages ?? stepInputMessages,
+                toolCallerMessages,
+              });
+
+              const promptMessages = await convertToLanguageModelPrompt({
+                prompt: {
+                  instructions: stepInstructions,
+                  messages: stepMessages,
+                },
+                supportedUrls: await stepModel.supportedUrls,
+                download,
+                provider: stepModel.provider.split('.')[0],
+              });
 
               const stepProviderOptions = mergeObjects(
                 providerOptions,

package/src/generate-text/stream-text.ts

@@ -89,6 +89,7 @@ import {
   executeToolsFromStream,
   type ExecuteToolsStreamPart,
 } from './execute-tools-from-stream';
+import { createToolSearchState } from '../tool-search/prepare-tool-search';
 import { executeToolCall } from './execute-tool-call';
 import {
   filterActiveTools,
@@ -146,6 +147,7 @@ import type {
 import { toResponseMessages } from './to-response-messages';
 import type { ToolApprovalConfiguration } from './tool-approval-configuration';
 import {
+  appendToolCallerMessages,
   prepareToolsForToolCallers,
   resolveToolCallerConfiguration,
   type Experimental_ToolCallers,
@@ -1381,6 +1383,10 @@ class DefaultStreamTextResult<
       tools,
       toolCallers: experimental_toolCallers,
     });
+    const prepareToolSearch = createToolSearchState({
+      tools,
+      toolCallers: resolvedToolCallers,
+    });
 
     const telemetryDispatcher = createRestrictedTelemetryDispatcher<
       TOOLS,
@@ -2330,8 +2336,12 @@ class DefaultStreamTextResult<
           const {
             executionTools: stepExecutionTools,
             modelTools: stepModelTools,
+            toolCallerMessages,
           } = prepareToolsForToolCallers({
-            tools: stepActiveTools,
+            tools: prepareToolSearch(stepActiveTools, {
+              toolsContext,
+              experimental_sandbox: stepSandbox,
+            }),
             toolCallers: resolvedToolCallers,
           });
           const stepToolOrder = prepareStepResult?.toolOrder ?? toolOrder;
@@ -2355,7 +2365,10 @@ class DefaultStreamTextResult<
             toolChoice: prepareStepResult?.toolChoice ?? toolChoice,
           });
 
-          const stepMessages = prepareStepResult?.messages ?? stepInputMessages;
+          const stepMessages = appendToolCallerMessages({
+            messages: prepareStepResult?.messages ?? stepInputMessages,
+            toolCallerMessages,
+          });
           currentStepMessages = stepMessages;
           const stepInstructions =
             prepareStepResult?.instructions ??