ai 7.0.102 → 7.0.104

package/src/generate-text/tool-caller-configuration.ts

@@ -1,12 +1,14 @@
 import {
   experimental_getToolCaller,
   type Experimental_ToolCallerTool,
+  type ModelMessage,
   type Tool,
   type ToolSet,
+  type UserModelMessage,
 } from '@ai-sdk/provider-utils';
 import { InvalidArgumentError } from '../error/invalid-argument-error';
 
-const DIRECT_TOOL_CALL = 'AI_SDK_DIRECT_TOOL_CALL';
+export const DIRECT_TOOL_CALL = 'AI_SDK_DIRECT_TOOL_CALL';
 
 type ToolCallerName<TOOLS extends ToolSet> = {
   [NAME in keyof TOOLS]: TOOLS[NAME] extends Experimental_ToolCallerTool
@@ -86,15 +88,28 @@ export function prepareToolsForToolCallers({
 }): {
   executionTools: ToolSet | undefined;
   modelTools: ToolSet | undefined;
+  toolCallerMessages: UserModelMessage[];
 } {
   if (tools == null || toolCallers == null) {
-    return { executionTools: tools, modelTools: tools };
+    return {
+      executionTools: tools,
+      modelTools: tools,
+      toolCallerMessages: [],
+    };
   }
 
+  // Tools available to the runtime for execution.
   const executionTools: ToolSet = { ...tools };
+
+  // Tools exposed directly to the model.
   const modelTools: ToolSet = { ...tools };
+
+  // Tools routed through each local caller.
   const localToolsByCaller = new Map<string, ToolSet>();
 
+  // User messages describing caller-accessible tools.
+  const toolCallerMessages: UserModelMessage[] = [];
+
   for (const [toolName, callerNames] of Object.entries(toolCallers)) {
     const tool = executionTools[toolName];
     if (tool == null) {
@@ -146,13 +161,52 @@ export function prepareToolsForToolCallers({
       continue;
     }
 
-    const boundCaller = caller.bind(localToolsByCaller.get(callerName) ?? {});
+    const callerTools = localToolsByCaller.get(callerName) ?? {};
+    const boundCaller = caller.bind(callerTools);
     executionTools[callerName] = boundCaller;
 
     if (Object.prototype.hasOwnProperty.call(modelTools, callerName)) {
-      modelTools[callerName] = boundCaller;
+      if (caller.prepareModelMessage == null) {
+        modelTools[callerName] = boundCaller;
+      } else {
+        const content = caller.prepareModelMessage(callerTools);
+        if (content != null) {
+          toolCallerMessages.push({ role: 'user', content });
+        }
+      }
     }
   }
 
-  return { executionTools, modelTools };
+  return { executionTools, modelTools, toolCallerMessages };
+}
+
+export function appendToolCallerMessages({
+  messages,
+  toolCallerMessages,
+}: {
+  messages: ModelMessage[];
+  toolCallerMessages: UserModelMessage[];
+}): ModelMessage[] {
+  if (toolCallerMessages.length === 0) {
+    return messages;
+  }
+
+  const latestUserText = messages.findLast(
+    message => message.role === 'user' && typeof message.content === 'string',
+  )?.content;
+  const existingUserText = new Set(
+    latestUserText == null ? [] : [latestUserText],
+  );
+  const additions = toolCallerMessages.filter(message => {
+    if (
+      typeof message.content !== 'string' ||
+      existingUserText.has(message.content)
+    ) {
+      return false;
+    }
+    existingUserText.add(message.content);
+    return true;
+  });
+
+  return additions.length === 0 ? messages : [...messages, ...additions];
 }

package/src/global.ts

@@ -9,6 +9,7 @@ declare global {
    * String model ids are resolved to the default provider and model id.
    *
    * If not set, the default provider is the Vercel AI gateway provider.
+   * Experimental evaluation requires an explicitly configured evaluation-capable provider.
    *
    * @see https://ai-sdk.dev/docs/ai-sdk-core/provider-management#global-provider-configuration
    */

package/src/index.ts

@@ -41,6 +41,7 @@ export * from './agent';
 export * from './batch';
 export * from './embed';
 export * from './error';
+export * from './evaluate';
 export * from './generate-image';
 export * from './generate-object';
 export * from './generate-speech';
@@ -54,6 +55,7 @@ export * from './registry';
 export * from './rerank';
 export * from './telemetry';
 export * from './text-stream';
+export { toolSearch } from './tool-search/tool-search';
 export * from './transcribe';
 export * from './translate';
 export * from './types';

package/src/model/resolve-model.ts

@@ -1,15 +1,19 @@
 import { gateway } from '@ai-sdk/gateway';
-import type {
-  EmbeddingModelV4,
-  Experimental_SpeechTranslationModelV4,
-  Experimental_VideoModelV4,
-  ImageModelV4,
-  LanguageModelV4,
-  ProviderV4,
-  RerankingModelV4,
-  SpeechModelV4,
-  TranscriptionModelV4,
+import {
+  NoSuchModelError,
+  type Experimental_EvaluationModelV4 as EvaluationModelV4,
+  type EmbeddingModelV4,
+  type Experimental_SpeechTranslationModelV4,
+  type Experimental_VideoModelV4,
+  type ImageModelV4,
+  type LanguageModelV4,
+  type ProviderV4,
+  type RerankingModelV4,
+  type SpeechModelV4,
+  type TranscriptionModelV4,
 } from '@ai-sdk/provider';
+import type { EvaluationModel } from '../evaluate/evaluation-result';
+import type { EvaluationProvider } from '../evaluate/evaluation-provider';
 import { UnsupportedModelVersionError } from '../error';
 import type { EmbeddingModel } from '../types/embedding-model';
 import type { LanguageModel } from '../types/language-model';
@@ -213,6 +217,47 @@ export function resolveRerankingModel(model: RerankingModel): RerankingModelV4 {
   return asRerankingModelV4(model);
 }
 
+export function resolveEvaluationModel(
+  model: EvaluationModel,
+): EvaluationModelV4 {
+  if (typeof model === 'string') {
+    // Use the original provider so experimental methods and their receiver survive.
+    // Evaluation does not default to Gateway.
+    const provider = globalThis.AI_SDK_DEFAULT_PROVIDER as
+      | EvaluationProvider
+      | undefined;
+
+    if (typeof provider?.evaluationModel !== 'function') {
+      throw new NoSuchModelError({
+        modelId: model,
+        modelType: 'evaluationModel',
+        message:
+          'The default provider does not support evaluation models. ' +
+          'Pass an evaluation model instance or configure AI_SDK_DEFAULT_PROVIDER with an evaluationModel method.',
+      });
+    }
+
+    const resolvedModel = provider.evaluationModel(model);
+    if (resolvedModel == null) {
+      throw new NoSuchModelError({
+        modelId: model,
+        modelType: 'evaluationModel',
+      });
+    }
+    model = resolvedModel;
+  }
+
+  if (model.specificationVersion !== 'v4') {
+    throw new UnsupportedModelVersionError({
+      version: model.specificationVersion,
+      provider: model.provider,
+      modelId: model.modelId,
+    });
+  }
+
+  return model;
+}
+
 function getGlobalProvider(): ProviderV4 {
   const provider = globalThis.AI_SDK_DEFAULT_PROVIDER ?? gateway;
   return asProviderV4(provider);

package/src/prompt/prepare-tools.ts

@@ -105,7 +105,7 @@ function orderToolEntries<TOOLS extends ToolSet>({
   return [...orderedTools, ...unorderedTools];
 }
 
-function resolveToolDescription<TOOLS extends ToolSet>({
+export function resolveToolDescription<TOOLS extends ToolSet>({
   tool,
   toolName,
   toolsContext,

package/src/realtime/browser-realtime-transport.ts

@@ -169,9 +169,20 @@ export class BrowserRealtimeTransport {
       if (this.ws === ws) {
         this.ws = null;
         const closeError = getCloseError(event) ?? connectionError;
+        const reportCloseErrorImmediately =
+          closeError != null && !codec.hasPendingIncoming;
         codec.stopWriting();
         this.notifyClosing();
         if (this.epoch !== epoch) return;
+        const drain = codec.finish();
+        if (reportCloseErrorImmediately) {
+          try {
+            this.onError(closeError);
+          } catch {
+            /* Application callbacks cannot interrupt teardown. */
+          }
+          if (this.epoch !== epoch) return;
+        }
         const complete = () => {
           if (this.epoch !== epoch) return;
           clearTimeout(this.drainTimer);
@@ -184,7 +195,7 @@ export class BrowserRealtimeTransport {
           }
         };
         this.drainTimer = setTimeout(complete, 1_000);
-        void this.awaitDrain(codec.finish(), complete);
+        void this.awaitDrain(drain, complete);
       }
     };
   }

package/src/realtime/realtime-event-channel.ts

@@ -34,6 +34,10 @@ export class RealtimeEventChannel {
       options.model.parseServerEvent.bind(options.model);
   }
 
+  get hasPendingIncoming(): boolean {
+    return this.incomingCount > 0;
+  }
+
   dispose(): void {
     this.active = false;
   }

package/src/realtime/realtime-session.ts

@@ -206,6 +206,7 @@ export abstract class AbstractRealtimeSession {
         if (current())
           this.fail(new Error('Realtime session startup timed out'));
       });
+      const reportedTransportErrors = new WeakSet<Error>();
       const callbacks = {
         model,
         onEvent: async (event: RealtimeServerEvent) => {
@@ -221,7 +222,10 @@ export abstract class AbstractRealtimeSession {
           }
         },
         onError: (error: Error) => {
-          if (current()) void this.reportError(error, attempt);
+          if (current()) {
+            reportedTransportErrors.add(error);
+            void this.reportError(error, attempt);
+          }
         },
         onFatalError: (error: Error, drain?: Promise<void>) => {
           if (current()) this.fail(error, drain);
@@ -244,7 +248,8 @@ export abstract class AbstractRealtimeSession {
           const finalizationConfirmed =
             model.capabilities?.finalization === 'session-close' &&
             this.state.session?.finalization === 'confirmed';
-          if (error != null && !finalizationConfirmed) this.fail(error);
+          if (error != null && !finalizationConfirmed)
+            this.fail(error, undefined, !reportedTransportErrors.has(error));
           else if (!attempt.ready && !finalizationConfirmed)
             this.fail(
               new Error('Realtime connection closed before becoming ready'),
@@ -348,7 +353,7 @@ export abstract class AbstractRealtimeSession {
     }
   }
 
-  private fail(error: unknown, drain?: Promise<void>): void {
+  private fail(error: unknown, drain?: Promise<void>, report = true): void {
     const attempt = this.attempt;
     if (attempt == null || !attempt.active || attempt.cause != null) return;
     attempt.cause = error instanceof Error ? error : new Error(String(error));
@@ -359,7 +364,7 @@ export abstract class AbstractRealtimeSession {
     if (this.attempt !== attempt || !attempt.active) return;
     if (drain == null) this.disconnect();
     else void this.drainAttempt(attempt, drain);
-    void this.reportError(attempt.cause, attempt);
+    if (report) void this.reportError(attempt.cause, attempt);
   }
 
   private async drainAttempt(

package/src/registry/custom-provider.ts

@@ -1,4 +1,5 @@
 import {
+  type Experimental_EvaluationModelV4 as EvaluationModelV4,
   type EmbeddingModelV4,
   type Experimental_VideoModelV4,
   type FilesV4,
@@ -13,6 +14,8 @@ import {
   type SpeechModelV4,
   type TranscriptionModelV4,
 } from '@ai-sdk/provider';
+import type { EvaluationModel } from '../evaluate/evaluation-result';
+import type { EvaluationProvider } from '../evaluate/evaluation-provider';
 import { asProviderV4 } from '../model/as-provider-v4';
 import {
   resolveEmbeddingModel,
@@ -22,6 +25,7 @@ import {
   resolveSpeechModel,
   resolveTranscriptionModel,
   resolveVideoModel,
+  resolveEvaluationModel,
 } from '../model/resolve-model';
 import type { EmbeddingModel } from '../types/embedding-model';
 import type { ImageModel } from '../types/image-model';
@@ -42,6 +46,7 @@ import type { VideoModel } from '../types/video-model';
  * @param {Record<string, SpeechModel>} [options.speechModels] - A record of speech models, where keys are model IDs and values are speech model instances.
  * @param {Record<string, RerankingModel>} [options.rerankingModels] - A record of reranking models, where keys are model IDs and values are reranking model instances.
  * @param {Record<string, VideoModel>} [options.videoModels] - A record of video models, where keys are model IDs and values are video model instances.
+ * @param {Record<string, EvaluationModel>} [options.evaluationModels] - Experimental evaluation models or default-provider model IDs, keyed by alias.
  * @param {FilesV4} [options.files] - A files interface for uploading files.
  * @param {SkillsV4} [options.skills] - A skills interface for uploading skills.
  * @param {ProviderV2 | ProviderV3 | ProviderV4} [options.fallbackProvider] - An optional fallback provider to use when a requested model is not found in the custom provider.
@@ -60,6 +65,10 @@ export function customProvider<
   FILES extends FilesV4 | undefined = undefined,
   SKILLS extends SkillsV4 | undefined = undefined,
   FALLBACK extends ProviderV2 | ProviderV3 | ProviderV4 | undefined = undefined,
+  EVALUATION_MODELS extends Record<string, EvaluationModel> = Record<
+    string,
+    EvaluationModel
+  >,
 >({
   languageModels,
   embeddingModels,
@@ -68,6 +77,7 @@ export function customProvider<
   speechModels,
   rerankingModels,
   videoModels,
+  evaluationModels,
   files,
   skills,
   fallbackProvider: fallbackProviderArg,
@@ -79,6 +89,7 @@ export function customProvider<
   speechModels?: SPEECH_MODELS;
   rerankingModels?: RERANKING_MODELS;
   videoModels?: VIDEO_MODELS;
+  evaluationModels?: EVALUATION_MODELS;
   files?: FILES;
   skills?: SKILLS;
   fallbackProvider?: FALLBACK;
@@ -92,6 +103,9 @@ export function customProvider<
   rerankingModel(modelId: ExtractModelId<RERANKING_MODELS>): RerankingModelV4;
   speechModel(modelId: ExtractModelId<SPEECH_MODELS>): SpeechModelV4;
   videoModel(modelId: ExtractModelId<VIDEO_MODELS>): Experimental_VideoModelV4;
+  evaluationModel(
+    modelId: ExtractModelId<EVALUATION_MODELS>,
+  ): EvaluationModelV4;
 } & (FILES extends FilesV4
     ? { files(): FilesV4 }
     : [FALLBACK] extends [{ files: () => FilesV4 }]
@@ -117,6 +131,9 @@ export function customProvider<
     videoModel(
       modelId: ExtractModelId<VIDEO_MODELS>,
     ): Experimental_VideoModelV4;
+    evaluationModel(
+      modelId: ExtractModelId<EVALUATION_MODELS>,
+    ): EvaluationModelV4;
   } = {
     specificationVersion: 'v4',
     languageModel(modelId: ExtractModelId<LANGUAGE_MODELS>): LanguageModelV4 {
@@ -203,6 +220,26 @@ export function customProvider<
 
       throw new NoSuchModelError({ modelId, modelType: 'rerankingModel' });
     },
+    evaluationModel(
+      modelId: ExtractModelId<EVALUATION_MODELS>,
+    ): EvaluationModelV4 {
+      if (
+        evaluationModels != null &&
+        Object.hasOwn(evaluationModels, modelId)
+      ) {
+        return resolveEvaluationModel(evaluationModels[modelId]);
+      }
+
+      const provider = fallbackProviderArg as EvaluationProvider | undefined;
+      if (typeof provider?.evaluationModel === 'function') {
+        const model = provider.evaluationModel(modelId);
+        if (model != null) {
+          return resolveEvaluationModel(model);
+        }
+      }
+
+      throw new NoSuchModelError({ modelId, modelType: 'evaluationModel' });
+    },
     videoModel(
       modelId: ExtractModelId<VIDEO_MODELS>,
     ): Experimental_VideoModelV4 {

package/src/registry/index.ts

@@ -5,3 +5,5 @@ export {
   experimental_createProviderRegistry,
 } from './provider-registry';
 export type { ProviderRegistryProvider } from './provider-registry';
+
+export type { EvaluationProviderRegistry as Experimental_EvaluationProviderRegistry } from './provider-registry';

package/src/registry/no-such-provider-error.ts

@@ -25,7 +25,8 @@ export class NoSuchProviderError extends NoSuchModelError {
       | 'transcriptionModel'
       | 'speechModel'
       | 'rerankingModel'
-      | 'videoModel';
+      | 'videoModel'
+      | 'evaluationModel';
     providerId: string;
     availableProviders: string[];
     message?: string;

package/src/registry/provider-registry.ts

@@ -1,4 +1,5 @@
 import {
+  type Experimental_EvaluationModelV4 as EvaluationModelV4,
   type EmbeddingModelV4,
   type Experimental_VideoModelV3,
   type Experimental_VideoModelV4,
@@ -13,6 +14,8 @@ import {
   type SpeechModelV4,
   type TranscriptionModelV4,
 } from '@ai-sdk/provider';
+import type { EvaluationProvider } from '../evaluate/evaluation-provider';
+import { resolveEvaluationModel } from '../model/resolve-model';
 import { wrapImageModel } from '../middleware/wrap-image-model';
 import { wrapLanguageModel } from '../middleware/wrap-language-model';
 import { asProviderV4 } from '../model/as-provider-v4';
@@ -34,7 +37,8 @@ type RegistryModelType =
   | 'transcriptionModel'
   | 'speechModel'
   | 'rerankingModel'
-  | 'videoModel';
+  | 'videoModel'
+  | 'evaluationModel';
 
 type ProviderVideoModelIdentifier<PROVIDER> = PROVIDER extends {
   videoModel: (...args: infer ARGS) => unknown;
@@ -42,6 +46,30 @@ type ProviderVideoModelIdentifier<PROVIDER> = PROVIDER extends {
   ? ExtractLiteralUnion<ARGS[0]>
   : never;
 
+type ProviderEvaluationModelIdentifier<PROVIDER> = PROVIDER extends {
+  evaluationModel: (...args: infer ARGS) => unknown;
+}
+  ? ExtractLiteralUnion<ARGS[0]>
+  : never;
+
+/** Registry with experimental evaluation access, separate from the stable interface. */
+export type EvaluationProviderRegistry<
+  PROVIDERS extends Record<string, ProviderV4 | ProviderV3> = Record<
+    string,
+    ProviderV4 | ProviderV3
+  >,
+  SEPARATOR extends string = ':',
+> = ProviderRegistryProvider<PROVIDERS, SEPARATOR> & {
+  evaluationModel<KEY extends keyof PROVIDERS>(
+    id: KEY extends string
+      ? `${KEY & string}${SEPARATOR}${ProviderEvaluationModelIdentifier<PROVIDERS[KEY]>}`
+      : never,
+  ): EvaluationModelV4;
+  evaluationModel<KEY extends keyof PROVIDERS>(
+    id: KEY extends string ? `${KEY & string}${SEPARATOR}${string}` : never,
+  ): EvaluationModelV4;
+};
+
 export interface ProviderRegistryProvider<
   PROVIDERS extends Record<string, ProviderV4 | ProviderV3> = Record<
     string,
@@ -150,7 +178,7 @@ export function createProviderRegistry<
       | LanguageModelMiddleware[];
     imageModelMiddleware?: ImageModelMiddleware | ImageModelMiddleware[];
   } = {},
-): ProviderRegistryProvider<PROVIDERS, SEPARATOR> {
+): EvaluationProviderRegistry<PROVIDERS, SEPARATOR> {
   const registry = new DefaultProviderRegistry<PROVIDERS, SEPARATOR>({
     separator,
     languageModelMiddleware,
@@ -175,9 +203,16 @@ export const experimental_createProviderRegistry = createProviderRegistry;
 class DefaultProviderRegistry<
   PROVIDERS extends Record<string, ProviderV4 | ProviderV3>,
   SEPARATOR extends string,
-> implements ProviderRegistryProvider<PROVIDERS, SEPARATOR> {
+>
+  implements
+    ProviderRegistryProvider<PROVIDERS, SEPARATOR>,
+    EvaluationProviderRegistry<PROVIDERS, SEPARATOR>
+{
   private providers: Partial<
-    Record<keyof PROVIDERS, ProviderV4 & ProviderWithOptionalVideoModel>
+    Record<
+      keyof PROVIDERS,
+      ProviderV4 & ProviderWithOptionalVideoModel & EvaluationProvider
+    >
   > = {};
   private separator: SEPARATOR;
   private languageModelMiddleware?:
@@ -213,7 +248,11 @@ class DefaultProviderRegistry<
       provider as ProviderWithOptionalVideoModel
     ).videoModel?.bind(provider);
 
-    this.providers[id] =
+    const evaluationModel = (
+      provider as EvaluationProvider
+    ).evaluationModel?.bind(provider);
+
+    const registeredProvider =
       videoModel == null
         ? providerV4
         : Object.assign(Object.create(Object.getPrototypeOf(providerV4)), {
@@ -221,12 +260,18 @@ class DefaultProviderRegistry<
             videoModel: (modelId: string) =>
               asVideoModelV4(videoModel(modelId)),
           });
+
+    // Keep v4 instances intact. Adapted providers need the original evaluation receiver.
+    if (registeredProvider !== provider && evaluationModel != null) {
+      Object.assign(registeredProvider, { evaluationModel });
+    }
+    this.providers[id] = registeredProvider;
   }
 
   private getProvider(
     id: string,
     modelType: RegistryModelType,
-  ): ProviderV4 & ProviderWithOptionalVideoModel {
+  ): ProviderV4 & ProviderWithOptionalVideoModel & EvaluationProvider {
     const provider = this.providers[id as keyof PROVIDERS];
 
     if (provider == null) {
@@ -382,6 +427,20 @@ class DefaultProviderRegistry<
     return asVideoModelV4(model);
   }
 
+  evaluationModel<KEY extends keyof PROVIDERS>(
+    id: `${KEY & string}${SEPARATOR}${string}`,
+  ): EvaluationModelV4 {
+    const [providerId, modelId] = this.splitId(id, 'evaluationModel');
+    const provider = this.getProvider(providerId, 'evaluationModel');
+    const model = provider.evaluationModel?.(modelId);
+
+    if (model == null) {
+      throw new NoSuchModelError({ modelId: id, modelType: 'evaluationModel' });
+    }
+
+    return resolveEvaluationModel(model);
+  }
+
   files<KEY extends keyof PROVIDERS>(id: KEY & string): FilesV4 {
     const provider = this.getProvider(id, 'languageModel');
     const files = provider.files?.();

package/src/test/evaluation-mock-model-v4.ts

@@ -0,0 +1,27 @@
+import type { Experimental_EvaluationModelV4 as EvaluationModelV4 } from '@ai-sdk/provider';
+import { notImplemented } from './not-implemented';
+
+export class EvaluationMockModelV4 implements EvaluationModelV4 {
+  readonly specificationVersion = 'v4';
+  readonly provider: string;
+  readonly modelId: string;
+  readonly supportedQuestionTypes: EvaluationModelV4['supportedQuestionTypes'];
+  doEvaluate: EvaluationModelV4['doEvaluate'];
+
+  constructor({
+    provider = 'mock-provider',
+    modelId = 'mock-model-id',
+    supportedQuestionTypes = ['choice', 'score', 'boolean'],
+    doEvaluate = notImplemented,
+  }: {
+    provider?: string;
+    modelId?: string;
+    supportedQuestionTypes?: EvaluationModelV4['supportedQuestionTypes'];
+    doEvaluate?: EvaluationModelV4['doEvaluate'];
+  } = {}) {
+    this.provider = provider;
+    this.modelId = modelId;
+    this.supportedQuestionTypes = supportedQuestionTypes;
+    this.doEvaluate = doEvaluate;
+  }
+}