ai 7.0.110 → 7.0.111

package/src/evaluate/evaluate.ts

@@ -3,26 +3,40 @@ import {
   type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
 } from '@ai-sdk/provider';
 import {
+  createIdGenerator,
   withUserAgentSuffix,
+  type Context,
   type ProviderOptions,
 } from '@ai-sdk/provider-utils';
 import { resolveEvaluationModel } from '../model/resolve-model';
 import { logWarnings } from '../logger/log-warnings';
+import { filterIncludedContext } from '../telemetry/filter-included-context';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
+import type { Callback } from '../util/callback';
+import { notify } from '../util/notify';
 import { prepareRetries } from '../util/prepare-retries';
 import { VERSION } from '../version';
+import type { EvaluateEndEvent, EvaluateStartEvent } from './evaluate-events';
 import type {
   EvaluationModel,
   EvaluationQuestion,
   EvaluationResult,
 } from './evaluation-result';
+import { createRestrictedTelemetryDispatcher } from './restricted-telemetry-dispatcher';
 import {
   validateEvaluationInput,
   validateEvaluationAnswers,
 } from './validate-evaluation';
 
+const originalGenerateCallId = createIdGenerator({
+  prefix: 'call',
+  size: 24,
+});
+
 /** Evaluate typed questions against one shared state. Experimental. */
 export async function evaluate<
   const QUESTIONS extends Record<string, EvaluationQuestion>,
+  RUNTIME_CONTEXT extends Context = Context,
 >({
   model: modelArg,
   state,
@@ -31,6 +45,11 @@ export async function evaluate<
   abortSignal,
   headers,
   providerOptions = {},
+  telemetry,
+  runtimeContext = {} as RUNTIME_CONTEXT,
+  onStart,
+  onEnd,
+  _internal: { generateCallId = originalGenerateCallId } = {},
 }: {
   /** An evaluation model instance or an ID resolved by the configured default provider. */
   model: EvaluationModel;
@@ -41,6 +60,18 @@ export async function evaluate<
   abortSignal?: AbortSignal;
   headers?: Record<string, string>;
   providerOptions?: ProviderOptions;
+  /** Optional telemetry configuration. */
+  telemetry?: TelemetryOptions<RUNTIME_CONTEXT>;
+  /** User-defined runtime context. Treat runtime context as immutable. */
+  runtimeContext?: RUNTIME_CONTEXT;
+  /** Called when the evaluate operation begins. */
+  onStart?: Callback<EvaluateStartEvent<RUNTIME_CONTEXT>>;
+  /** Called when the evaluate operation completes. */
+  onEnd?: Callback<EvaluateEndEvent<RUNTIME_CONTEXT>>;
+  /** Internal. For test use only. May change without notice. */
+  _internal?: {
+    generateCallId?: () => string;
+  };
 }): Promise<EvaluationResult<QUESTIONS>> {
   const model = resolveEvaluationModel(modelArg);
 
@@ -57,50 +88,124 @@ export async function evaluate<
     }
   }
 
-  const { retry } = prepareRetries({ maxRetries, abortSignal });
-  const result = await retry(() => {
-    abortSignal?.throwIfAborted();
-    return model.doEvaluate({
-      state,
-      questions,
-      abortSignal,
-      headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
-      providerOptions,
-    });
+  const callId = generateCallId();
+  const { maxRetries: resolvedMaxRetries, retry } = prepareRetries({
+    maxRetries,
+    abortSignal,
   });
-
-  abortSignal?.throwIfAborted();
-  validateEvaluationAnswers({
-    questions,
-    answers: result.answers,
-    rounding: result.rounding,
+  const telemetryDispatcher = createRestrictedTelemetryDispatcher({
+    telemetry,
   });
-  logWarnings({
-    warnings: result.warnings,
+  const runInTracingChannelSpan =
+    telemetryDispatcher.runInTracingChannelSpan ??
+    (async <T>({ execute }: { execute: () => PromiseLike<T> }) =>
+      await execute());
+  const startEvent = {
+    callId,
+    operationId: 'ai.evaluate' as const,
+    runtimeContext,
     provider: model.provider,
-    model: model.modelId,
-  });
+    modelId: model.modelId,
+    state,
+    questions,
+    maxRetries: resolvedMaxRetries,
+    headers,
+    providerOptions,
+  };
+  const tracingStartEvent = {
+    ...startEvent,
+    runtimeContext: filterIncludedContext({
+      context: runtimeContext,
+      includeContext: telemetry?.includeRuntimeContext,
+    }),
+  };
 
-  const inputTokens = result.usage?.inputTokens;
-  const outputTokens = result.usage?.outputTokens;
+  return await runInTracingChannelSpan({
+    type: 'experimental_evaluate',
+    event: tracingStartEvent,
+    execute: async () => {
+      await notify({
+        event: startEvent,
+        callbacks: [onStart, telemetryDispatcher.onStart],
+      });
 
-  return {
-    answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
-    usage: {
-      inputTokens,
-      outputTokens,
-      totalTokens:
-        inputTokens != null && outputTokens != null
-          ? inputTokens + outputTokens
-          : undefined,
-    },
-    warnings: result.warnings,
-    rounding: result.rounding,
-    providerMetadata: result.providerMetadata,
-    response: {
-      ...result.response,
-      timestamp: result.response?.timestamp ?? new Date(),
-      modelId: result.response?.modelId ?? model.modelId,
+      try {
+        const modelCallEvent = {
+          callId,
+          operationId: 'ai.evaluate.doEvaluate' as const,
+          provider: model.provider,
+          modelId: model.modelId,
+          state,
+          questions,
+        };
+        await notify({
+          event: modelCallEvent,
+          callbacks: [
+            telemetryDispatcher.experimental_onEvaluationModelCallStart,
+          ],
+        });
+        const result = await retry(async () => {
+          abortSignal?.throwIfAborted();
+          return await model.doEvaluate({
+            state,
+            questions,
+            abortSignal,
+            headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
+            providerOptions,
+          });
+        });
+
+        abortSignal?.throwIfAborted();
+        validateEvaluationAnswers({
+          questions,
+          answers: result.answers,
+          rounding: result.rounding,
+        });
+        await notify({
+          event: { ...modelCallEvent, ...result },
+          callbacks: [
+            telemetryDispatcher.experimental_onEvaluationModelCallEnd,
+          ],
+        });
+
+        logWarnings({
+          warnings: result.warnings,
+          provider: model.provider,
+          model: model.modelId,
+        });
+
+        const inputTokens = result.usage?.inputTokens;
+        const outputTokens = result.usage?.outputTokens;
+        const evaluationResult: EvaluationResult<QUESTIONS> = {
+          answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
+          usage: {
+            inputTokens,
+            outputTokens,
+            totalTokens:
+              inputTokens != null && outputTokens != null
+                ? inputTokens + outputTokens
+                : undefined,
+          },
+          warnings: result.warnings,
+          rounding: result.rounding,
+          providerMetadata: result.providerMetadata,
+          response: {
+            ...result.response,
+            timestamp: result.response?.timestamp ?? new Date(),
+            modelId: result.response?.modelId ?? model.modelId,
+          },
+        };
+
+        await notify({
+          event: { ...startEvent, ...evaluationResult },
+          callbacks: [onEnd, telemetryDispatcher.onEnd],
+        });
+
+        return evaluationResult;
+      } catch (error) {
+        await telemetryDispatcher.onError?.({ callId, error });
+        throw error;
+      }
     },
-  };
+  });
 }

package/src/evaluate/index.ts

@@ -1,4 +1,10 @@
 export { evaluate as experimental_evaluate } from './evaluate';
+export type {
+  EvaluateStartEvent as Experimental_EvaluateStartEvent,
+  EvaluateEndEvent as Experimental_EvaluateEndEvent,
+  EvaluationModelCallStartEvent as Experimental_EvaluationModelCallStartEvent,
+  EvaluationModelCallEndEvent as Experimental_EvaluationModelCallEndEvent,
+} from './evaluate-events';
 export type {
   EvaluationModel as Experimental_EvaluationModel,
   EvaluationQuestion as Experimental_EvaluationQuestion,

package/src/evaluate/restricted-telemetry-dispatcher.ts

@@ -0,0 +1,46 @@
+import type { Context } from '@ai-sdk/provider-utils';
+import { createTelemetryDispatcher } from '../telemetry/create-telemetry-dispatcher';
+import { filterIncludedContext } from '../telemetry/filter-included-context';
+import type { TelemetryDispatcher } from '../telemetry/telemetry';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
+import type { Callback } from '../util/callback';
+import type { EvaluateEndEvent, EvaluateStartEvent } from './evaluate-events';
+
+export function createRestrictedTelemetryDispatcher<
+  RUNTIME_CONTEXT extends Context,
+>({
+  telemetry,
+}: {
+  telemetry?: TelemetryOptions<RUNTIME_CONTEXT>;
+}): Omit<
+  TelemetryDispatcher,
+  | 'onStart'
+  | 'onEnd'
+  | 'experimental_onEvaluateStart'
+  | 'experimental_onEvaluateEnd'
+> & {
+  onStart: Callback<EvaluateStartEvent<RUNTIME_CONTEXT>>;
+  onEnd: Callback<EvaluateEndEvent<RUNTIME_CONTEXT>>;
+} {
+  const dispatcher = createTelemetryDispatcher({ telemetry });
+
+  return {
+    ...dispatcher,
+    onStart: event =>
+      dispatcher.experimental_onEvaluateStart?.({
+        ...event,
+        runtimeContext: filterIncludedContext({
+          context: event.runtimeContext,
+          includeContext: telemetry?.includeRuntimeContext,
+        }),
+      }),
+    onEnd: event =>
+      dispatcher.experimental_onEvaluateEnd?.({
+        ...event,
+        runtimeContext: filterIncludedContext({
+          context: event.runtimeContext,
+          includeContext: telemetry?.includeRuntimeContext,
+        }),
+      }),
+  };
+}

package/src/generate-text/convert-language-model-content.ts

@@ -4,6 +4,10 @@ import { ToolCallNotFoundForApprovalError } from '../error/tool-call-not-found-f
 import { getOwn } from '../util/get-own';
 import type { ContentPart } from './content-part';
 import { DefaultGeneratedFile } from './generated-file';
+import {
+  resolveGeneratedFileData,
+  type GeneratedFileDataCache,
+} from './resolve-generated-file-data';
 import type { ToolApprovalRequestOutput } from './tool-approval-request-output';
 import type { ToolApprovalResponseOutput } from './tool-approval-response-output';
 import type { TypedToolCall } from './tool-call';
@@ -11,13 +15,15 @@ import type { TypedToolError } from './tool-error';
 import type { ToolOutput } from './tool-output';
 import type { TypedToolResult } from './tool-result';
 
-export function convertLanguageModelContent<TOOLS extends ToolSet>({
+export async function convertLanguageModelContent<TOOLS extends ToolSet>({
   content,
   toolCalls,
   toolOutputs,
   toolApprovalRequests,
   toolApprovalResponses,
   tools,
+  abortSignal,
+  generatedFileDataCache,
 }: {
   content: Array<LanguageModelV4Content>;
   toolCalls: Array<TypedToolCall<TOOLS>>;
@@ -25,7 +31,9 @@ export function convertLanguageModelContent<TOOLS extends ToolSet>({
   toolApprovalRequests: Array<ToolApprovalRequestOutput<TOOLS>>;
   toolApprovalResponses: Array<ToolApprovalResponseOutput<TOOLS>>;
   tools: TOOLS | undefined;
-}): Array<ContentPart<TOOLS>> {
+  abortSignal?: AbortSignal;
+  generatedFileDataCache?: GeneratedFileDataCache;
+}): Promise<Array<ContentPart<TOOLS>>> {
   const contentParts: Array<ContentPart<TOOLS>> = [];
   const toolOutputsWithApprovalResponses: Array<ToolOutput<TOOLS>> = [];
   const toolOutputsWithoutApprovalResponses: Array<ToolOutput<TOOLS>> = [];
@@ -49,10 +57,11 @@ export function convertLanguageModelContent<TOOLS extends ToolSet>({
         contentParts.push({
           type: part.type as 'file' | 'reasoning-file',
           file: new DefaultGeneratedFile({
-            data:
-              part.data.type === 'data'
-                ? part.data.data
-                : part.data.url.toString(),
+            data: await resolveGeneratedFileData({
+              data: part.data,
+              abortSignal,
+              cache: generatedFileDataCache,
+            }),
             mediaType: part.mediaType,
           }),
           ...(part.providerMetadata != null

package/src/generate-text/generate-text.ts

@@ -1099,13 +1099,16 @@ export async function generateText<
               > = {};
               const blockedToolCallIds = new Set<string>();
 
-              const modelCallContent = convertLanguageModelContent({
+              const generatedFileDataCache = new WeakMap();
+              const modelCallContent = await convertLanguageModelContent({
                 content: currentModelResponse.content,
                 toolCalls: stepToolCalls,
                 toolOutputs: [],
                 toolApprovalRequests: [],
                 toolApprovalResponses: [],
                 tools,
+                abortSignal: mergedAbortSignal,
+                generatedFileDataCache,
               });
 
               await notify({
@@ -1426,13 +1429,15 @@ export async function generateText<
               }
 
               // content:
-              const stepContent = convertLanguageModelContent({
+              const stepContent = await convertLanguageModelContent({
                 content: currentModelResponse.content,
                 toolCalls: stepToolCalls,
                 toolOutputs: clientToolOutputs,
                 toolApprovalRequests: Object.values(toolApprovalRequests),
                 toolApprovalResponses,
                 tools,
+                abortSignal: mergedAbortSignal,
+                generatedFileDataCache,
               });
 
               const stepResponseMessages = await toResponseMessages({

package/src/generate-text/resolve-generated-file-data.ts

@@ -0,0 +1,41 @@
+import type { LanguageModelV4Content } from '@ai-sdk/provider';
+import { download } from '../util/download/download';
+
+type GeneratedFileData = Extract<
+  LanguageModelV4Content,
+  { type: 'file' | 'reasoning-file' }
+>['data'];
+
+export type GeneratedFileDataCache = WeakMap<
+  GeneratedFileData,
+  string | Uint8Array
+>;
+
+export async function resolveGeneratedFileData({
+  data,
+  abortSignal,
+  cache,
+}: {
+  data: GeneratedFileData;
+  abortSignal?: AbortSignal;
+  cache?: GeneratedFileDataCache;
+}): Promise<string | Uint8Array> {
+  if (data.type === 'data') {
+    return data.data;
+  }
+
+  const cachedData = cache?.get(data);
+  if (cachedData != null) {
+    return cachedData;
+  }
+
+  const downloadedData = (
+    await download({
+      url: data.url,
+      abortSignal,
+    })
+  ).data;
+
+  cache?.set(data, downloadedData);
+  return downloadedData;
+}

package/src/generate-text/stream-language-model-call.ts

@@ -53,6 +53,7 @@ import type {
 } from './language-model-events';
 import type { Output } from './output';
 import { parseToolCall } from './parse-tool-call';
+import { resolveGeneratedFileData } from './resolve-generated-file-data';
 import type {
   TextStreamFilePart,
   TextStreamPart,
@@ -565,10 +566,10 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform<
         case 'file':
         case 'reasoning-file': {
           const file = new DefaultGeneratedFileWithType({
-            data:
-              chunk.data.type === 'data'
-                ? chunk.data.data
-                : chunk.data.url.toString(),
+            data: await resolveGeneratedFileData({
+              data: chunk.data,
+              abortSignal,
+            }),
             mediaType: chunk.mediaType,
           });
 

package/src/telemetry/create-telemetry-dispatcher.ts

@@ -163,6 +163,18 @@ export function createTelemetryDispatcher({
     onEmbedEnd: mergeTelemetryCallback('onEmbedEnd'),
     onRerankStart: mergeTelemetryCallback('onRerankStart'),
     onRerankEnd: mergeTelemetryCallback('onRerankEnd'),
+    experimental_onEvaluateStart: mergeTelemetryCallback(
+      'experimental_onEvaluateStart',
+    ),
+    experimental_onEvaluationModelCallStart: mergeTelemetryCallback(
+      'experimental_onEvaluationModelCallStart',
+    ),
+    experimental_onEvaluationModelCallEnd: mergeTelemetryCallback(
+      'experimental_onEvaluationModelCallEnd',
+    ),
+    experimental_onEvaluateEnd: mergeTelemetryCallback(
+      'experimental_onEvaluateEnd',
+    ),
     onEnd: mergeTelemetryCallback('onEnd'),
     onAbort: mergeTelemetryCallback('onAbort'),
     onError: mergeTelemetryCallback('onError'),

package/src/telemetry/telemetry.ts

@@ -5,6 +5,12 @@ import type {
   EmbeddingModelCallEndEvent,
   EmbeddingModelCallStartEvent,
 } from '../embed/embed-events';
+import type {
+  EvaluateEndEvent,
+  EvaluateStartEvent,
+  EvaluationModelCallEndEvent,
+  EvaluationModelCallStartEvent,
+} from '../evaluate/evaluate-events';
 import type {
   GenerateObjectEndEvent,
   GenerateObjectStartEvent,
@@ -92,6 +98,10 @@ export interface TelemetryDispatcher {
   onEmbedEnd?: Callback<EmbeddingModelCallEndEvent>;
   onRerankStart?: Callback<RerankingModelCallStartEvent>;
   onRerankEnd?: Callback<RerankingModelCallEndEvent>;
+  experimental_onEvaluateStart?: Callback<EvaluateStartEvent>;
+  experimental_onEvaluationModelCallStart?: Callback<EvaluationModelCallStartEvent>;
+  experimental_onEvaluationModelCallEnd?: Callback<EvaluationModelCallEndEvent>;
+  experimental_onEvaluateEnd?: Callback<EvaluateEndEvent>;
   onEnd?: Callback<OperationEndEvent>;
   onAbort?: Callback<GenerateTextAbortEvent<ToolSet>>;
   onError?: Callback<unknown>;
@@ -214,6 +224,30 @@ export interface Telemetry {
    */
   onRerankEnd?: Callback<InferTelemetryEvent<RerankingModelCallEndEvent>>;
 
+  /** Called when an experimental evaluation operation begins. */
+  experimental_onEvaluateStart?: Callback<
+    InferTelemetryEvent<EvaluateStartEvent>
+  >;
+
+  /**
+   * Called immediately before an experimental evaluation model call begins.
+   * The logical model call includes any provider retries.
+   */
+  experimental_onEvaluationModelCallStart?: Callback<
+    InferTelemetryEvent<EvaluationModelCallStartEvent>
+  >;
+
+  /**
+   * Called after an experimental evaluation model response has been validated.
+   * The logical model call includes any provider retries.
+   */
+  experimental_onEvaluationModelCallEnd?: Callback<
+    InferTelemetryEvent<EvaluationModelCallEndEvent>
+  >;
+
+  /** Called when an experimental evaluation operation completes. */
+  experimental_onEvaluateEnd?: Callback<InferTelemetryEvent<EvaluateEndEvent>>;
+
   /**
    * Called when an operation completes. Fired for text generation
    * (generateText/streamText), object generation (generateObject/streamObject),

package/src/telemetry/tracing-channel.ts

@@ -8,7 +8,8 @@ export type TelemetryTracingEventType =
   | 'executeTool'
   | 'embed'
   | 'embedMany'
-  | 'rerank';
+  | 'rerank'
+  | 'experimental_evaluate';
 
 export type TelemetryTracingChannelMessage<EVENT = unknown> = {
   readonly type: TelemetryTracingEventType;