ai 7.0.110 → 7.0.111

package/docs/03-ai-sdk-core/60-telemetry.mdx

@@ -126,7 +126,7 @@ const result = await generateText({
 });
 ```
 
-In this example, telemetry integrations receive `runtimeContext` as `{ requestId: 'req_abc' }`. Properties set to `false` or omitted are excluded. If `telemetry.includeRuntimeContext` is omitted, no runtime context properties are included. `telemetry.includeRuntimeContext` is supported by `generateText`, `streamText`, `ToolLoopAgent`, `embed`, `embedMany`, and `rerank`.
+In this example, telemetry integrations receive `runtimeContext` as `{ requestId: 'req_abc' }`. Properties set to `false` or omitted are excluded. If `telemetry.includeRuntimeContext` is omitted, no runtime context properties are included. `telemetry.includeRuntimeContext` is supported by `generateText`, `streamText`, `ToolLoopAgent`, `embed`, `embedMany`, `rerank`, and `experimental_evaluate`.
 
 <Note>
   `telemetry.includeRuntimeContext` only filters telemetry integrations,
@@ -383,6 +383,29 @@ export function myIntegration(): Telemetry {
       type: '(event: RerankingModelCallEndEvent) => void | PromiseLike<void>',
       description: 'Called when an individual reranking model call completes.',
     },
+    {
+      name: 'experimental_onEvaluateStart',
+      type: '(event: Experimental_EvaluateStartEvent) => void | PromiseLike<void>',
+      description: 'Called when an experimental evaluation operation begins.',
+    },
+    {
+      name: 'experimental_onEvaluationModelCallStart',
+      type: '(event: Experimental_EvaluationModelCallStartEvent) => void | PromiseLike<void>',
+      description:
+        'Called when the logical evaluation model call begins. The call includes any provider retries.',
+    },
+    {
+      name: 'experimental_onEvaluationModelCallEnd',
+      type: '(event: Experimental_EvaluationModelCallEndEvent) => void | PromiseLike<void>',
+      description:
+        'Called when the logical evaluation model call, including any retries, completes.',
+    },
+    {
+      name: 'experimental_onEvaluateEnd',
+      type: '(event: Experimental_EvaluateEndEvent) => void | PromiseLike<void>',
+      description:
+        'Called when an experimental evaluation operation completes.',
+    },
     {
       name: 'onEnd',
       type: '(event: GenerateTextEndEvent) => void | PromiseLike<void>',
@@ -398,6 +421,9 @@ export function myIntegration(): Telemetry {
   ]}
 />
 
+Experimental evaluation events use the four `experimental_*` callbacks above.
+They are not included in the stable `onStart` and `onEnd` event unions.
+
 The event types for each method are the same as the corresponding [lifecycle callbacks](/docs/ai-sdk-core/lifecycle-callbacks). See the lifecycle callbacks documentation for the full property reference of each event.
 
 ## Collected Data
@@ -524,6 +550,31 @@ For `rerank`, the integration records spans with `CLIENT` kind:
   - `gen_ai.provider.name`: the provider
   - `gen_ai.request.model`: the model ID
 
+#### experimental_evaluate
+
+For `experimental_evaluate`, the integration records two spans with `CLIENT`
+kind: an **`evaluate {modelId}`** root span for the whole operation and one
+**`evaluate {modelId}`** child span for evaluation model work, including
+retries. Both include the provider and requested model with
+`gen_ai.operation.name: "evaluate"`. Token
+usage is recorded on the child span. When evaluation attributes are enabled,
+answers are recorded on both spans so the operation span contains its output.
+
+`evaluate` is an AI SDK-defined custom operation name, not a well-known
+OpenTelemetry GenAI operation. OpenTelemetry permits custom operation names
+when no well-known value describes the operation.
+
+OpenTelemetry's `gen_ai.evaluation.result` event describes the result of
+evaluating a GenAI output for quality or other characteristics. In contrast,
+`experimental_evaluate` can evaluate arbitrary JSON state for classification,
+routing, scoring, and similar tasks. The integration therefore does not emit a
+`gen_ai.evaluation.result` event.
+
+Evaluation state, questions, and answers also do not have applicable GenAI
+semantic-convention attributes. Enable the `experimental_evaluation`
+supplemental option to record them under AI SDK-specific
+`ai.evaluation.*` attributes.
+
 #### GenAI span details
 
 ##### GenAI message format
@@ -579,10 +630,10 @@ registerTelemetry(
 
 The callback runs when each span is created and receives:
 
-- `spanType`: the type of span being created (`operation`, `step`, `languageModel`, `tool`, `embedding`, or `reranking`).
+- `spanType`: the type of span being created (`operation`, `step`, `languageModel`, `tool`, `embedding`, `reranking`, or `experimental_evaluation`).
 - `operationId`: the AI SDK operation ID for the current call, such as `ai.generateText` or `ai.streamText`.
 - `callId`: the unique ID for the current AI SDK call.
-- `runtimeContext`: the telemetry-filtered runtime context for text generation, embedding, and reranking spans. Text generation spans also reflect updates from `prepareStep`.
+- `runtimeContext`: the telemetry-filtered runtime context for text generation, embedding, reranking, and evaluation spans. Text generation spans also reflect updates from `prepareStep`.
 
 Custom attributes are merged with AI SDK attributes on the span. AI SDK-owned
 attributes take precedence when a custom attribute uses the same key.
@@ -642,6 +693,7 @@ registerTelemetry(
     providerMetadata: true,
     embedding: true,
     reranking: true,
+    experimental_evaluation: true,
     runtimeContext: true,
     headers: true,
     toolChoice: true,
@@ -656,6 +708,7 @@ The available options are:
 - `providerMetadata`: `ai.response.providerMetadata` on operation, step, and model-call spans.
 - `embedding`: embedding inputs and outputs.
 - `reranking`: rerank input documents and ranking output.
+- `experimental_evaluation`: experimental evaluation state, questions, and answers.
 - `runtimeContext`: `ai.settings.context.*`.
 - `headers`: `ai.request.headers.*`.
 - `toolChoice`: `ai.prompt.toolChoice`.

package/docs/03-ai-sdk-core/65-lifecycle-callbacks.mdx

@@ -1,12 +1,12 @@
 ---
 title: Lifecycle Callbacks
-description: Observe AI SDK lifecycle events in generateText, streamText, embed, embedMany, and rerank calls
+description: Observe AI SDK lifecycle events in generateText, streamText, embed, embedMany, rerank, and experimental_evaluate calls
 ---
 
 # Lifecycle Callbacks
 
 Event callbacks let you run your own code at important points in an AI SDK call.
-You can attach them directly to `generateText`, `streamText`, `embed`, `embedMany`, and `rerank` calls to observe what happened, record usage, debug multi-step generations, and monitor tool execution.
+You can attach them directly to `generateText`, `streamText`, `embed`, `embedMany`, `rerank`, and `experimental_evaluate` calls to observe what happened, record usage, debug multi-step generations, and monitor tool execution.
 
 They are especially useful when you want application-specific logic close to the call site:
 
@@ -449,6 +449,25 @@ const result = await generateText({
   ]}
 />
 
+### `experimental_evaluate`
+
+<PropertiesTable
+  content={[
+    {
+      name: 'onStart',
+      type: '(event: Experimental_EvaluateStartEvent) => void | Promise<void>',
+      description:
+        'Called when the evaluation operation begins, before the evaluation model is called.',
+    },
+    {
+      name: 'onEnd',
+      type: '(event: Experimental_EvaluateEndEvent) => void | Promise<void>',
+      description:
+        'Called when the evaluation operation completes successfully.',
+    },
+  ]}
+/>
+
 ## Event Data Reference
 
 The exact event data depends on the callback. The tables below summarize the fields you will most commonly use.
@@ -1151,3 +1170,106 @@ For `embed`, `value` is a single string. For `embedMany`, `value` is an array of
     },
   ]}
 />
+
+### Evaluation Events
+
+Evaluation callbacks are experimental. Their exported types are
+`Experimental_EvaluateStartEvent` and `Experimental_EvaluateEndEvent`.
+
+#### onStart
+
+<PropertiesTable
+  content={[
+    {
+      name: 'callId',
+      type: 'string',
+      description: 'Unique identifier for this evaluation call.',
+    },
+    {
+      name: 'operationId',
+      type: "'ai.evaluate'",
+      description: 'Evaluation operation identifier.',
+    },
+    {
+      name: 'runtimeContext',
+      type: 'RUNTIME_CONTEXT',
+      description: 'Full user-defined runtime context.',
+    },
+    {
+      name: 'provider',
+      type: 'string',
+      description: 'Provider identifier for the evaluation model.',
+    },
+    {
+      name: 'modelId',
+      type: 'string',
+      description: 'Evaluation model identifier.',
+    },
+    {
+      name: 'state',
+      type: 'string | object | array',
+      description: 'Shared state being evaluated.',
+    },
+    {
+      name: 'questions',
+      type: 'Readonly<Record<string, Experimental_EvaluationQuestion>>',
+      description: 'Questions being evaluated against the shared state.',
+    },
+    {
+      name: 'maxRetries',
+      type: 'number',
+      description: 'Maximum number of retries for the model call.',
+    },
+    {
+      name: 'headers',
+      type: 'Record<string, string> | undefined',
+      description: 'Additional HTTP headers sent with the request.',
+    },
+    {
+      name: 'providerOptions',
+      type: 'ProviderOptions',
+      description: 'Provider-specific options.',
+    },
+  ]}
+/>
+
+#### onEnd
+
+Includes every `onStart` field plus:
+
+<PropertiesTable
+  content={[
+    {
+      name: 'answers',
+      type: 'Record<string, Experimental_EvaluationAnswer>',
+      description: 'Exactly one typed answer per question ID.',
+    },
+    {
+      name: 'usage',
+      type: '{ inputTokens: number | undefined; outputTokens: number | undefined; totalTokens: number | undefined }',
+      description: 'Token usage for the evaluation operation.',
+    },
+    {
+      name: 'warnings',
+      type: 'Array<Warning>',
+      description: 'Warnings from the evaluation model.',
+    },
+    {
+      name: 'rounding',
+      type: '{ probabilityDecimals?: number; scoreDecimals?: number } | undefined',
+      description:
+        'Provider-declared decimal precision for probabilities and scores.',
+    },
+    {
+      name: 'providerMetadata',
+      type: 'ProviderMetadata | undefined',
+      description: 'Optional provider-specific metadata.',
+    },
+    {
+      name: 'response',
+      type: '{ id?: string; timestamp: Date; modelId: string; headers?: Record<string, string>; body?: unknown }',
+      description:
+        'Response metadata including the resolved model and timestamp.',
+    },
+  ]}
+/>

package/docs/07-reference/01-ai-sdk-core/14-evaluate.mdx

@@ -14,15 +14,22 @@ state. See [Evaluation](/docs/ai-sdk-core/evaluation) for examples and semantics
 
 ## Parameters
 
-| Parameter         | Type                                              | Description                                                                                                                                 |
-| ----------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
-| `model`           | `Experimental_EvaluationModel`                    | Required experimental v4 model instance or a string ID resolved by Gateway or an explicitly configured evaluation-capable default provider. |
-| `state`           | `string \| object \| array`                       | Required JSON-compatible shared state.                                                                                                      |
-| `questions`       | `Record<string, Experimental_EvaluationQuestion>` | Required nonempty question map.                                                                                                             |
-| `maxRetries`      | `number`                                          | Nonnegative integer; defaults to 2.                                                                                                         |
-| `abortSignal`     | `AbortSignal`                                     | Cancels evaluation.                                                                                                                         |
-| `headers`         | `Record<string, string>`                          | Additional HTTP headers.                                                                                                                    |
-| `providerOptions` | `ProviderOptions`                                 | Provider-specific options.                                                                                                                  |
+| Parameter         | Type                                               | Description                                                                                                                                 |
+| ----------------- | -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
+| `model`           | `Experimental_EvaluationModel`                     | Required experimental v4 model instance or a string ID resolved by Gateway or an explicitly configured evaluation-capable default provider. |
+| `state`           | `string \| object \| array`                        | Required JSON-compatible shared state.                                                                                                      |
+| `questions`       | `Record<string, Experimental_EvaluationQuestion>`  | Required nonempty question map.                                                                                                             |
+| `maxRetries`      | `number`                                           | Nonnegative integer; defaults to 2.                                                                                                         |
+| `abortSignal`     | `AbortSignal`                                      | Cancels evaluation.                                                                                                                         |
+| `headers`         | `Record<string, string>`                           | Additional HTTP headers.                                                                                                                    |
+| `providerOptions` | `ProviderOptions`                                  | Provider-specific options.                                                                                                                  |
+| `telemetry`       | `TelemetryOptions`                                 | Telemetry configuration, including per-call integrations, input/output recording, and a function ID.                                        |
+| `runtimeContext`  | `Record<string, unknown>`                          | Context available to lifecycle callbacks and selectively included in telemetry with `telemetry.includeRuntimeContext`.                      |
+| `onStart`         | `(event: Experimental_EvaluateStartEvent) => void` | Called when the evaluation operation begins.                                                                                                |
+| `onEnd`           | `(event: Experimental_EvaluateEndEvent) => void`   | Called when the evaluation operation completes successfully.                                                                                |
+
+See [Lifecycle Callbacks](/docs/ai-sdk-core/lifecycle-callbacks#experimental_evaluate)
+for the complete `onStart` and `onEnd` event fields.
 
 ## Result
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "7.0.110",
+  "version": "7.0.111",
   "type": "module",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",

package/src/batch/batch.ts

@@ -374,7 +374,13 @@ export function getBatchResults<TOOLS extends ToolSet>({
     cancel?: (reason?: unknown) => void;
   } = {
     async transform(item, controller) {
-      controller.enqueue(await convertBatchItemResult({ item, tools }));
+      controller.enqueue(
+        await convertBatchItemResult({
+          item,
+          tools,
+          abortSignal: operationAbortSignal,
+        }),
+      );
     },
 
     cancel(reason) {
@@ -507,9 +513,11 @@ function validateBatchReference({
 async function convertBatchItemResult<TOOLS extends ToolSet>({
   item,
   tools,
+  abortSignal,
 }: {
   item: BatchV4ItemResult;
   tools: TOOLS | undefined;
+  abortSignal: AbortSignal | undefined;
 }): Promise<BatchItemResult<TOOLS>> {
   switch (item.type) {
     case 'text':
@@ -519,7 +527,11 @@ async function convertBatchItemResult<TOOLS extends ToolSet>({
             type: item.type,
             id: item.id,
             status: item.status,
-            ...(await convertGenerateResult({ result: item.result, tools })),
+            ...(await convertGenerateResult({
+              result: item.result,
+              tools,
+              abortSignal,
+            })),
           };
         case 'failed':
           return {
@@ -600,9 +612,11 @@ function convertImageResult(
 async function convertGenerateResult<TOOLS extends ToolSet>({
   result,
   tools,
+  abortSignal,
 }: {
   result: LanguageModelV4GenerateResult;
   tools: TOOLS | undefined;
+  abortSignal: AbortSignal | undefined;
 }): Promise<TextBatchGenerationResult<TOOLS>> {
   const toolCalls = await Promise.all(
     result.content
@@ -620,13 +634,14 @@ async function convertGenerateResult<TOOLS extends ToolSet>({
         }),
       ),
   );
-  const content = convertLanguageModelContent<TOOLS>({
+  const content = await convertLanguageModelContent<TOOLS>({
     content: result.content,
     toolCalls,
     toolOutputs: [],
     toolApprovalRequests: [],
     toolApprovalResponses: [],
     tools,
+    abortSignal,
   });
 
   return {

package/src/evaluate/evaluate-events.ts

@@ -0,0 +1,131 @@
+import type {
+  Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
+  Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
+} from '@ai-sdk/provider';
+import type { Context, ProviderOptions } from '@ai-sdk/provider-utils';
+import type { EvaluationQuestion, EvaluationResult } from './evaluation-result';
+
+/**
+ * Event passed to the `onStart` callback for evaluation operations.
+ *
+ * Called when the operation begins, before the evaluation model is called.
+ */
+export type EvaluateStartEvent<RUNTIME_CONTEXT extends Context = Context> = {
+  /** User-defined runtime context. */
+  readonly runtimeContext: RUNTIME_CONTEXT;
+
+  /** Unique identifier for this evaluation call, used to correlate events. */
+  readonly callId: string;
+
+  /** Identifies the operation type (`ai.evaluate`). */
+  readonly operationId: 'ai.evaluate';
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The evaluation model identifier. */
+  readonly modelId: string;
+
+  /** The shared state being evaluated. */
+  readonly state: EvaluationModelV4CallOptions['state'];
+
+  /** The questions being evaluated against the shared state. */
+  readonly questions: Readonly<Record<string, EvaluationQuestion>>;
+
+  /** Maximum number of retries for the evaluation model call. */
+  readonly maxRetries: number;
+
+  /** Additional HTTP headers sent with the request. */
+  readonly headers: Record<string, string> | undefined;
+
+  /** Additional provider-specific options. */
+  readonly providerOptions: ProviderOptions;
+};
+
+/**
+ * Event passed to the `onEnd` callback for evaluation operations.
+ *
+ * Called when the operation completes successfully.
+ */
+export type EvaluateEndEvent<RUNTIME_CONTEXT extends Context = Context> =
+  EvaluateStartEvent<RUNTIME_CONTEXT> & {
+    /** Exactly one typed answer per question ID. */
+    readonly answers: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['answers'];
+
+    /** Token usage for the evaluation operation. */
+    readonly usage: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['usage'];
+
+    /** Warnings from the evaluation model. */
+    readonly warnings: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['warnings'];
+
+    /** Provider-declared decimal precision for probabilities and scores. */
+    readonly rounding: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['rounding'];
+
+    /** Optional provider-specific metadata. */
+    readonly providerMetadata: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['providerMetadata'];
+
+    /** Response metadata, including the resolved model ID and timestamp. */
+    readonly response: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['response'];
+  };
+
+/**
+ * Event fired when the evaluation model call begins.
+ *
+ * The logical model call includes any provider retries.
+ */
+export type EvaluationModelCallStartEvent = {
+  /** Unique identifier for the outer evaluation call. */
+  readonly callId: string;
+
+  /** Identifies the inner operation (`ai.evaluate.doEvaluate`). */
+  readonly operationId: 'ai.evaluate.doEvaluate';
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The evaluation model identifier. */
+  readonly modelId: string;
+
+  /** The shared state being evaluated. */
+  readonly state: EvaluationModelV4CallOptions['state'];
+
+  /** The questions being evaluated against the shared state. */
+  readonly questions: Readonly<Record<string, EvaluationQuestion>>;
+};
+
+/**
+ * Event fired after the evaluation model response has been validated.
+ *
+ * Contains the result of the logical model call, including any retries.
+ */
+export type EvaluationModelCallEndEvent = EvaluationModelCallStartEvent & {
+  /** Exactly one answer per question ID. */
+  readonly answers: EvaluationModelV4Result['answers'];
+
+  /** Token usage reported by the evaluation model. */
+  readonly usage?: EvaluationModelV4Result['usage'];
+
+  /** Warnings from the evaluation model. */
+  readonly warnings: EvaluationModelV4Result['warnings'];
+
+  /** Provider-declared decimal precision for probabilities and scores. */
+  readonly rounding?: EvaluationModelV4Result['rounding'];
+
+  /** Optional provider-specific metadata. */
+  readonly providerMetadata?: EvaluationModelV4Result['providerMetadata'];
+
+  /** Optional raw response metadata from the provider. */
+  readonly response?: EvaluationModelV4Result['response'];
+};