ai 7.0.102 → 7.0.103

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "7.0.102",
+  "version": "7.0.103",
   "type": "module",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
@@ -42,20 +42,20 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/gateway": "4.0.82",
-    "@ai-sdk/provider": "4.0.15",
-    "@ai-sdk/provider-utils": "5.0.41"
+    "@ai-sdk/gateway": "4.0.83",
+    "@ai-sdk/provider": "4.0.16",
+    "@ai-sdk/provider-utils": "5.0.42"
   },
   "devDependencies": {
-    "@ai-sdk/amazon-bedrock": "5.0.83",
-    "@ai-sdk/deepseek": "3.0.45",
-    "@ai-sdk/google": "4.0.71",
-    "@ai-sdk/groq": "4.0.42",
-    "@ai-sdk/huggingface": "2.0.49",
-    "@ai-sdk/moonshotai": "3.0.50",
-    "@ai-sdk/openai": "4.0.67",
+    "@ai-sdk/amazon-bedrock": "5.0.85",
+    "@ai-sdk/deepseek": "3.0.46",
+    "@ai-sdk/google": "4.0.73",
+    "@ai-sdk/groq": "4.0.43",
+    "@ai-sdk/huggingface": "2.0.50",
+    "@ai-sdk/moonshotai": "3.0.51",
+    "@ai-sdk/openai": "4.0.68",
     "@ai-sdk/test-server": "2.0.1",
-    "@ai-sdk/xai": "4.0.59",
+    "@ai-sdk/xai": "5.0.1",
     "@edge-runtime/vm": "^5.0.0",
     "@smithy/eventstream-codec": "^4.3.3",
     "@smithy/util-utf8": "^4.3.3",

package/src/error/index.ts

@@ -2,6 +2,7 @@ export {
   AISDKError,
   APICallError,
   EmptyResponseBodyError,
+  Experimental_EvaluationUnsupportedQuestionTypeError,
   InvalidPromptError,
   InvalidResponseDataError,
   JSONParseError,

package/src/evaluate/evaluate.ts

@@ -0,0 +1,112 @@
+import {
+  Experimental_EvaluationUnsupportedQuestionTypeError as EvaluationUnsupportedQuestionTypeError,
+  type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
+} from '@ai-sdk/provider';
+import {
+  withUserAgentSuffix,
+  type ProviderOptions,
+} from '@ai-sdk/provider-utils';
+import { UnsupportedModelVersionError } from '../error/unsupported-model-version-error';
+import { logWarnings } from '../logger/log-warnings';
+import { prepareRetries } from '../util/prepare-retries';
+import { VERSION } from '../version';
+import type {
+  EvaluationModel,
+  EvaluationQuestion,
+  EvaluationResult,
+} from './evaluation-result';
+import {
+  validateEvaluationInput,
+  validateEvaluationAnswers,
+} from './validate-evaluation';
+
+/** Evaluate typed questions against one shared state. Experimental. */
+export async function evaluate<
+  const QUESTIONS extends Record<string, EvaluationQuestion>,
+>({
+  model,
+  state,
+  questions,
+  maxRetries,
+  abortSignal,
+  headers,
+  providerOptions = {},
+}: {
+  /** An evaluation model instance. String model resolution is not yet supported. */
+  model: EvaluationModel;
+  state: EvaluationModelV4CallOptions['state'];
+  questions: QUESTIONS;
+  /** Maximum retries for transient provider failures. Defaults to 2. */
+  maxRetries?: number;
+  abortSignal?: AbortSignal;
+  headers?: Record<string, string>;
+  providerOptions?: ProviderOptions;
+}): Promise<EvaluationResult<QUESTIONS>> {
+  if (model.specificationVersion !== 'v4') {
+    throw new UnsupportedModelVersionError({
+      version: model.specificationVersion,
+      provider: model.provider,
+      modelId: model.modelId,
+    });
+  }
+
+  validateEvaluationInput({ state, questions });
+
+  for (const [questionId, question] of Object.entries(questions)) {
+    if (!model.supportedQuestionTypes.includes(question.type)) {
+      throw new EvaluationUnsupportedQuestionTypeError({
+        questionId,
+        questionType: question.type,
+        provider: model.provider,
+        modelId: model.modelId,
+      });
+    }
+  }
+
+  const { retry } = prepareRetries({ maxRetries, abortSignal });
+  const result = await retry(() => {
+    abortSignal?.throwIfAborted();
+    return model.doEvaluate({
+      state,
+      questions,
+      abortSignal,
+      headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
+      providerOptions,
+    });
+  });
+
+  abortSignal?.throwIfAborted();
+  validateEvaluationAnswers({
+    questions,
+    answers: result.answers,
+    rounding: result.rounding,
+  });
+  logWarnings({
+    warnings: result.warnings,
+    provider: model.provider,
+    model: model.modelId,
+  });
+
+  const inputTokens = result.usage?.inputTokens;
+  const outputTokens = result.usage?.outputTokens;
+
+  return {
+    answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
+    usage: {
+      inputTokens,
+      outputTokens,
+      totalTokens:
+        inputTokens != null && outputTokens != null
+          ? inputTokens + outputTokens
+          : undefined,
+    },
+    warnings: result.warnings,
+    rounding: result.rounding,
+    providerMetadata: result.providerMetadata,
+    response: {
+      ...result.response,
+      timestamp: result.response?.timestamp ?? new Date(),
+      modelId: result.response?.modelId ?? model.modelId,
+    },
+  };
+}

package/src/evaluate/evaluation-result.ts

@@ -0,0 +1,39 @@
+import type {
+  Experimental_EvaluationModelV4 as EvaluationModelV4,
+  Experimental_EvaluationModelV4Question as EvaluationModelV4Question,
+  Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
+} from '@ai-sdk/provider';
+
+export type EvaluationModel = EvaluationModelV4;
+export type EvaluationQuestion = EvaluationModelV4Question;
+
+export type EvaluationAnswer<QUESTION extends EvaluationQuestion> =
+  QUESTION extends { type: 'choice'; criteria: infer CRITERIA }
+    ? {
+        type: 'choice';
+        choice: Extract<keyof CRITERIA, string>;
+        probabilities?: Record<Extract<keyof CRITERIA, string>, number>;
+      }
+    : QUESTION extends { type: 'score' }
+      ? { type: 'score'; score: number; probabilities?: Record<string, number> }
+      : { type: 'boolean'; probability: number };
+
+export type EvaluationResult<
+  QUESTIONS extends Record<string, EvaluationQuestion>,
+> = {
+  readonly answers: {
+    [ID in keyof QUESTIONS]: EvaluationAnswer<QUESTIONS[ID]>;
+  };
+  readonly usage: {
+    inputTokens: number | undefined;
+    outputTokens: number | undefined;
+    totalTokens: number | undefined;
+  };
+  readonly warnings: EvaluationModelV4Result['warnings'];
+  readonly rounding: EvaluationModelV4Result['rounding'];
+  readonly providerMetadata: EvaluationModelV4Result['providerMetadata'];
+  readonly response: NonNullable<EvaluationModelV4Result['response']> & {
+    timestamp: Date;
+    modelId: string;
+  };
+};

package/src/evaluate/index.ts

@@ -0,0 +1,7 @@
+export { evaluate as experimental_evaluate } from './evaluate';
+export type {
+  EvaluationModel as Experimental_EvaluationModel,
+  EvaluationQuestion as Experimental_EvaluationQuestion,
+  EvaluationAnswer as Experimental_EvaluationAnswer,
+  EvaluationResult as Experimental_EvaluationResult,
+} from './evaluation-result';

package/src/evaluate/validate-evaluation.ts

@@ -0,0 +1,298 @@
+import {
+  InvalidResponseDataError,
+  type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
+  type Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
+} from '@ai-sdk/provider';
+import { InvalidArgumentError } from '../error/invalid-argument-error';
+
+// Absolute tolerance for sums and means. Never renormalize provider output.
+const tolerance = 1e-6;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  if (value == null || typeof value !== 'object' || Array.isArray(value))
+    return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+function isJSON(value: unknown, ancestors = new Set<object>()): boolean {
+  if (value === null || typeof value === 'string' || typeof value === 'boolean')
+    return true;
+  if (typeof value === 'number') return Number.isFinite(value);
+  if (typeof value !== 'object' || (!Array.isArray(value) && !isRecord(value)))
+    return false;
+  if (ancestors.has(value)) return false;
+  ancestors.add(value);
+  const valid =
+    Object.getOwnPropertySymbols(value).length === 0 &&
+    (Array.isArray(value)
+      ? Array.from(value).every(item => isJSON(item, ancestors))
+      : Object.values(value).every(item => isJSON(item, ancestors)));
+  ancestors.delete(value);
+  return valid;
+}
+
+function isInput(value: unknown): boolean {
+  return (
+    (typeof value === 'string' || Array.isArray(value) || isRecord(value)) &&
+    isJSON(value)
+  );
+}
+
+function invalidInput(
+  parameter: string,
+  value: unknown,
+  message: string,
+): never {
+  throw new InvalidArgumentError({ parameter, value, message });
+}
+
+export function validateEvaluationInput({
+  state,
+  questions,
+}: EvaluationModelV4CallOptions) {
+  if (!isInput(state)) {
+    invalidInput(
+      'state',
+      state,
+      'must be a JSON-compatible string, object, or array',
+    );
+  }
+  if (!isRecord(questions) || Object.keys(questions).length === 0) {
+    invalidInput('questions', questions, 'must be a nonempty question map');
+  }
+
+  for (const [id, question] of Object.entries(questions)) {
+    const parameter = `questions.${id}`;
+    if (!isRecord(question) || !isInput(question.instructions)) {
+      invalidInput(
+        parameter,
+        question,
+        'instructions must be a JSON-compatible string, object, or array',
+      );
+    }
+    const criteria = question.criteria;
+    switch (question.type) {
+      case 'choice':
+        if (!isRecord(criteria) || Object.keys(criteria).length === 0) {
+          invalidInput(
+            parameter,
+            question,
+            'choice criteria must be a nonempty option map',
+          );
+        }
+        break;
+      case 'score':
+        if (!Array.isArray(criteria) || criteria.length < 2) {
+          invalidInput(
+            parameter,
+            question,
+            'score criteria must contain at least two ordered levels',
+          );
+        }
+        break;
+      case 'boolean':
+        if (criteria === undefined) continue;
+        if (
+          !isRecord(criteria) ||
+          Object.keys(criteria).some(key => key !== 'true' && key !== 'false')
+        ) {
+          invalidInput(
+            parameter,
+            question,
+            'boolean criteria may only describe true and false',
+          );
+        }
+        break;
+      default:
+        invalidInput(
+          parameter,
+          question,
+          'question type must be choice, score, or boolean',
+        );
+    }
+    if (
+      !isJSON(criteria) ||
+      Object.values(criteria).some(value => value !== null && !isInput(value))
+    ) {
+      invalidInput(
+        parameter,
+        question,
+        'criteria descriptions must be JSON-compatible strings, objects, arrays, or null',
+      );
+    }
+  }
+}
+
+function invalidAnswer(answers: unknown, message: string): never {
+  throw new InvalidResponseDataError({ data: answers, message });
+}
+
+function isProbability(value: unknown): value is number {
+  return (
+    typeof value === 'number' &&
+    Number.isFinite(value) &&
+    value >= 0 &&
+    value <= 1
+  );
+}
+
+function hasExactKeys(value: Record<string, unknown>, keys: string[]): boolean {
+  return (
+    Object.keys(value).length === keys.length &&
+    keys.every(key => Object.hasOwn(value, key))
+  );
+}
+
+function validateDistribution(
+  value: unknown,
+  keys: string[],
+  answers: unknown,
+  id: string,
+  roundingError: number,
+): asserts value is Record<string, number> {
+  if (
+    !isRecord(value) ||
+    !hasExactKeys(value, keys) ||
+    !Object.values(value).every(isProbability)
+  ) {
+    invalidAnswer(
+      answers,
+      `Question "${id}" must have a complete distribution of finite probabilities in [0, 1].`,
+    );
+  }
+  const sum = Object.values(value).reduce<number>(
+    (total, probability) => total + (probability as number),
+    0,
+  );
+  if (Math.abs(sum - 1) > tolerance + keys.length * roundingError) {
+    invalidAnswer(
+      answers,
+      `Question "${id}" probabilities must sum to 1 within the declared rounding precision.`,
+    );
+  }
+}
+
+export function validateEvaluationAnswers({
+  questions,
+  answers,
+  rounding,
+}: {
+  questions: EvaluationModelV4CallOptions['questions'];
+  answers: unknown;
+  rounding?: EvaluationModelV4Result['rounding'];
+}) {
+  function roundingError(decimals: number | undefined): number {
+    if (decimals === undefined) return 0;
+    if (!Number.isInteger(decimals) || decimals < 0 || decimals > 15) {
+      invalidAnswer(
+        answers,
+        'Evaluation rounding decimals must be integers between 0 and 15.',
+      );
+    }
+    return 0.5 * 10 ** -decimals;
+  }
+  const probabilityError = roundingError(rounding?.probabilityDecimals);
+  const scoreError = roundingError(rounding?.scoreDecimals);
+  if (!isRecord(answers) || !hasExactKeys(answers, Object.keys(questions))) {
+    invalidAnswer(
+      answers,
+      'Evaluation must return exactly one answer for every question.',
+    );
+  }
+
+  for (const [id, question] of Object.entries(questions)) {
+    const answer = answers[id];
+    if (!isRecord(answer) || answer.type !== question.type) {
+      invalidAnswer(
+        answers,
+        `Question "${id}" returned an answer with the wrong type.`,
+      );
+    }
+
+    switch (question.type) {
+      case 'choice': {
+        if (
+          typeof answer.choice !== 'string' ||
+          !Object.hasOwn(question.criteria, answer.choice)
+        ) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" selected an unknown option.`,
+          );
+        }
+        if (answer.probabilities !== undefined) {
+          validateDistribution(
+            answer.probabilities,
+            Object.keys(question.criteria),
+            answers,
+            id,
+            probabilityError,
+          );
+          const selected = answer.probabilities[answer.choice];
+          if (
+            Object.values(answer.probabilities).some(
+              probability => probability > selected + tolerance,
+            )
+          ) {
+            invalidAnswer(
+              answers,
+              `Question "${id}" did not select a highest-probability option.`,
+            );
+          }
+        }
+        break;
+      }
+      case 'score': {
+        if (
+          typeof answer.score !== 'number' ||
+          !Number.isFinite(answer.score) ||
+          answer.score < 0 ||
+          answer.score > question.criteria.length - 1
+        ) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" score must be in [0, ${question.criteria.length - 1}].`,
+          );
+        }
+        if (answer.probabilities !== undefined) {
+          const keys = question.criteria.map((_, index) => String(index));
+          validateDistribution(
+            answer.probabilities,
+            keys,
+            answers,
+            id,
+            probabilityError,
+          );
+          const mean = Object.entries(answer.probabilities).reduce(
+            (total, [index, probability]) =>
+              total + Number(index) * probability,
+            0,
+          );
+          const meanRoundingError = keys.reduce(
+            (total, index) => total + Number(index) * probabilityError,
+            0,
+          );
+          if (
+            Math.abs(mean - answer.score) >
+            tolerance + meanRoundingError + scoreError
+          ) {
+            invalidAnswer(
+              answers,
+              `Question "${id}" score must equal the probability-weighted mean within the declared rounding precision.`,
+            );
+          }
+        }
+        break;
+      }
+      case 'boolean':
+        if (!isProbability(answer.probability)) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" must return P(true) as a finite probability in [0, 1].`,
+          );
+        }
+        break;
+    }
+  }
+}

package/src/generate-text/generate-text.ts

@@ -106,6 +106,7 @@ import type { ToolApprovalConfiguration } from './tool-approval-configuration';
 import type { ToolApprovalRequestOutput } from './tool-approval-request-output';
 import type { ToolApprovalResponseOutput } from './tool-approval-response-output';
 import {
+  appendToolCallerMessages,
   prepareToolsForToolCallers,
   resolveToolCallerConfiguration,
   type Experimental_ToolCallers,
@@ -906,16 +907,6 @@ export async function generateText<
                 prepareStepResult?.system ??
                 instructionsForNextStep;
 
-              const promptMessages = await convertToLanguageModelPrompt({
-                prompt: {
-                  instructions: stepInstructions,
-                  messages: prepareStepResult?.messages ?? stepInputMessages,
-                },
-                supportedUrls: await stepModel.supportedUrls,
-                download,
-                provider: stepModel.provider.split('.')[0],
-              });
-
               runtimeContext =
                 prepareStepResult?.runtimeContext ?? runtimeContext;
               toolsContext = prepareStepResult?.toolsContext ?? toolsContext;
@@ -927,6 +918,7 @@ export async function generateText<
               const {
                 executionTools: stepExecutionTools,
                 modelTools: stepModelTools,
+                toolCallerMessages,
               } = prepareToolsForToolCallers({
                 tools: stepActiveTools,
                 toolCallers: resolvedToolCallers,
@@ -952,8 +944,20 @@ export async function generateText<
                 toolChoice: prepareStepResult?.toolChoice ?? toolChoice,
               });
 
-              const stepMessages =
-                prepareStepResult?.messages ?? stepInputMessages;
+              const stepMessages = appendToolCallerMessages({
+                messages: prepareStepResult?.messages ?? stepInputMessages,
+                toolCallerMessages,
+              });
+
+              const promptMessages = await convertToLanguageModelPrompt({
+                prompt: {
+                  instructions: stepInstructions,
+                  messages: stepMessages,
+                },
+                supportedUrls: await stepModel.supportedUrls,
+                download,
+                provider: stepModel.provider.split('.')[0],
+              });
 
               const stepProviderOptions = mergeObjects(
                 providerOptions,

package/src/generate-text/stream-text.ts

@@ -146,6 +146,7 @@ import type {
 import { toResponseMessages } from './to-response-messages';
 import type { ToolApprovalConfiguration } from './tool-approval-configuration';
 import {
+  appendToolCallerMessages,
   prepareToolsForToolCallers,
   resolveToolCallerConfiguration,
   type Experimental_ToolCallers,
@@ -2330,6 +2331,7 @@ class DefaultStreamTextResult<
           const {
             executionTools: stepExecutionTools,
             modelTools: stepModelTools,
+            toolCallerMessages,
           } = prepareToolsForToolCallers({
             tools: stepActiveTools,
             toolCallers: resolvedToolCallers,
@@ -2355,7 +2357,10 @@ class DefaultStreamTextResult<
             toolChoice: prepareStepResult?.toolChoice ?? toolChoice,
           });
 
-          const stepMessages = prepareStepResult?.messages ?? stepInputMessages;
+          const stepMessages = appendToolCallerMessages({
+            messages: prepareStepResult?.messages ?? stepInputMessages,
+            toolCallerMessages,
+          });
           currentStepMessages = stepMessages;
           const stepInstructions =
             prepareStepResult?.instructions ??

package/src/generate-text/tool-caller-configuration.ts

@@ -1,8 +1,10 @@
 import {
   experimental_getToolCaller,
   type Experimental_ToolCallerTool,
+  type ModelMessage,
   type Tool,
   type ToolSet,
+  type UserModelMessage,
 } from '@ai-sdk/provider-utils';
 import { InvalidArgumentError } from '../error/invalid-argument-error';
 
@@ -86,15 +88,28 @@ export function prepareToolsForToolCallers({
 }): {
   executionTools: ToolSet | undefined;
   modelTools: ToolSet | undefined;
+  toolCallerMessages: UserModelMessage[];
 } {
   if (tools == null || toolCallers == null) {
-    return { executionTools: tools, modelTools: tools };
+    return {
+      executionTools: tools,
+      modelTools: tools,
+      toolCallerMessages: [],
+    };
   }
 
+  // Tools available to the runtime for execution.
   const executionTools: ToolSet = { ...tools };
+
+  // Tools exposed directly to the model.
   const modelTools: ToolSet = { ...tools };
+
+  // Tools routed through each local caller.
   const localToolsByCaller = new Map<string, ToolSet>();
 
+  // User messages describing caller-accessible tools.
+  const toolCallerMessages: UserModelMessage[] = [];
+
   for (const [toolName, callerNames] of Object.entries(toolCallers)) {
     const tool = executionTools[toolName];
     if (tool == null) {
@@ -146,13 +161,52 @@ export function prepareToolsForToolCallers({
       continue;
     }
 
-    const boundCaller = caller.bind(localToolsByCaller.get(callerName) ?? {});
+    const callerTools = localToolsByCaller.get(callerName) ?? {};
+    const boundCaller = caller.bind(callerTools);
     executionTools[callerName] = boundCaller;
 
     if (Object.prototype.hasOwnProperty.call(modelTools, callerName)) {
-      modelTools[callerName] = boundCaller;
+      if (caller.prepareModelMessage == null) {
+        modelTools[callerName] = boundCaller;
+      } else {
+        const content = caller.prepareModelMessage(callerTools);
+        if (content != null) {
+          toolCallerMessages.push({ role: 'user', content });
+        }
+      }
     }
   }
 
-  return { executionTools, modelTools };
+  return { executionTools, modelTools, toolCallerMessages };
+}
+
+export function appendToolCallerMessages({
+  messages,
+  toolCallerMessages,
+}: {
+  messages: ModelMessage[];
+  toolCallerMessages: UserModelMessage[];
+}): ModelMessage[] {
+  if (toolCallerMessages.length === 0) {
+    return messages;
+  }
+
+  const latestUserText = messages.findLast(
+    message => message.role === 'user' && typeof message.content === 'string',
+  )?.content;
+  const existingUserText = new Set(
+    latestUserText == null ? [] : [latestUserText],
+  );
+  const additions = toolCallerMessages.filter(message => {
+    if (
+      typeof message.content !== 'string' ||
+      existingUserText.has(message.content)
+    ) {
+      return false;
+    }
+    existingUserText.add(message.content);
+    return true;
+  });
+
+  return additions.length === 0 ? messages : [...messages, ...additions];
 }

package/src/index.ts

@@ -41,6 +41,7 @@ export * from './agent';
 export * from './batch';
 export * from './embed';
 export * from './error';
+export * from './evaluate';
 export * from './generate-image';
 export * from './generate-object';
 export * from './generate-speech';