ai 7.0.110 → 7.0.112

package/src/evaluate/evaluate-events.ts

@@ -0,0 +1,131 @@
+import type {
+  Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
+  Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
+} from '@ai-sdk/provider';
+import type { Context, ProviderOptions } from '@ai-sdk/provider-utils';
+import type { EvaluationQuestion, EvaluationResult } from './evaluation-result';
+
+/**
+ * Event passed to the `onStart` callback for evaluation operations.
+ *
+ * Called when the operation begins, before the evaluation model is called.
+ */
+export type EvaluateStartEvent<RUNTIME_CONTEXT extends Context = Context> = {
+  /** User-defined runtime context. */
+  readonly runtimeContext: RUNTIME_CONTEXT;
+
+  /** Unique identifier for this evaluation call, used to correlate events. */
+  readonly callId: string;
+
+  /** Identifies the operation type (`ai.evaluate`). */
+  readonly operationId: 'ai.evaluate';
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The evaluation model identifier. */
+  readonly modelId: string;
+
+  /** The shared state being evaluated. */
+  readonly state: EvaluationModelV4CallOptions['state'];
+
+  /** The questions being evaluated against the shared state. */
+  readonly questions: Readonly<Record<string, EvaluationQuestion>>;
+
+  /** Maximum number of retries for the evaluation model call. */
+  readonly maxRetries: number;
+
+  /** Additional HTTP headers sent with the request. */
+  readonly headers: Record<string, string> | undefined;
+
+  /** Additional provider-specific options. */
+  readonly providerOptions: ProviderOptions;
+};
+
+/**
+ * Event passed to the `onEnd` callback for evaluation operations.
+ *
+ * Called when the operation completes successfully.
+ */
+export type EvaluateEndEvent<RUNTIME_CONTEXT extends Context = Context> =
+  EvaluateStartEvent<RUNTIME_CONTEXT> & {
+    /** Exactly one typed answer per question ID. */
+    readonly answers: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['answers'];
+
+    /** Token usage for the evaluation operation. */
+    readonly usage: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['usage'];
+
+    /** Warnings from the evaluation model. */
+    readonly warnings: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['warnings'];
+
+    /** Provider-declared decimal precision for probabilities and scores. */
+    readonly rounding: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['rounding'];
+
+    /** Optional provider-specific metadata. */
+    readonly providerMetadata: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['providerMetadata'];
+
+    /** Response metadata, including the resolved model ID and timestamp. */
+    readonly response: EvaluationResult<
+      Record<string, EvaluationQuestion>
+    >['response'];
+  };
+
+/**
+ * Event fired when the evaluation model call begins.
+ *
+ * The logical model call includes any provider retries.
+ */
+export type EvaluationModelCallStartEvent = {
+  /** Unique identifier for the outer evaluation call. */
+  readonly callId: string;
+
+  /** Identifies the inner operation (`ai.evaluate.doEvaluate`). */
+  readonly operationId: 'ai.evaluate.doEvaluate';
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The evaluation model identifier. */
+  readonly modelId: string;
+
+  /** The shared state being evaluated. */
+  readonly state: EvaluationModelV4CallOptions['state'];
+
+  /** The questions being evaluated against the shared state. */
+  readonly questions: Readonly<Record<string, EvaluationQuestion>>;
+};
+
+/**
+ * Event fired after the evaluation model response has been validated.
+ *
+ * Contains the result of the logical model call, including any retries.
+ */
+export type EvaluationModelCallEndEvent = EvaluationModelCallStartEvent & {
+  /** Exactly one answer per question ID. */
+  readonly answers: EvaluationModelV4Result['answers'];
+
+  /** Token usage reported by the evaluation model. */
+  readonly usage?: EvaluationModelV4Result['usage'];
+
+  /** Warnings from the evaluation model. */
+  readonly warnings: EvaluationModelV4Result['warnings'];
+
+  /** Provider-declared decimal precision for probabilities and scores. */
+  readonly rounding?: EvaluationModelV4Result['rounding'];
+
+  /** Optional provider-specific metadata. */
+  readonly providerMetadata?: EvaluationModelV4Result['providerMetadata'];
+
+  /** Optional raw response metadata from the provider. */
+  readonly response?: EvaluationModelV4Result['response'];
+};

package/src/evaluate/evaluate.ts

@@ -3,26 +3,40 @@ import {
   type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
 } from '@ai-sdk/provider';
 import {
+  createIdGenerator,
   withUserAgentSuffix,
+  type Context,
   type ProviderOptions,
 } from '@ai-sdk/provider-utils';
 import { resolveEvaluationModel } from '../model/resolve-model';
 import { logWarnings } from '../logger/log-warnings';
+import { filterIncludedContext } from '../telemetry/filter-included-context';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
+import type { Callback } from '../util/callback';
+import { notify } from '../util/notify';
 import { prepareRetries } from '../util/prepare-retries';
 import { VERSION } from '../version';
+import type { EvaluateEndEvent, EvaluateStartEvent } from './evaluate-events';
 import type {
   EvaluationModel,
   EvaluationQuestion,
   EvaluationResult,
 } from './evaluation-result';
+import { createRestrictedTelemetryDispatcher } from './restricted-telemetry-dispatcher';
 import {
   validateEvaluationInput,
   validateEvaluationAnswers,
 } from './validate-evaluation';
 
+const originalGenerateCallId = createIdGenerator({
+  prefix: 'call',
+  size: 24,
+});
+
 /** Evaluate typed questions against one shared state. Experimental. */
 export async function evaluate<
   const QUESTIONS extends Record<string, EvaluationQuestion>,
+  RUNTIME_CONTEXT extends Context = Context,
 >({
   model: modelArg,
   state,
@@ -31,6 +45,11 @@ export async function evaluate<
   abortSignal,
   headers,
   providerOptions = {},
+  telemetry,
+  runtimeContext = {} as RUNTIME_CONTEXT,
+  onStart,
+  onEnd,
+  _internal: { generateCallId = originalGenerateCallId } = {},
 }: {
   /** An evaluation model instance or an ID resolved by the configured default provider. */
   model: EvaluationModel;
@@ -41,6 +60,18 @@ export async function evaluate<
   abortSignal?: AbortSignal;
   headers?: Record<string, string>;
   providerOptions?: ProviderOptions;
+  /** Optional telemetry configuration. */
+  telemetry?: TelemetryOptions<RUNTIME_CONTEXT>;
+  /** User-defined runtime context. Treat runtime context as immutable. */
+  runtimeContext?: RUNTIME_CONTEXT;
+  /** Called when the evaluate operation begins. */
+  onStart?: Callback<EvaluateStartEvent<RUNTIME_CONTEXT>>;
+  /** Called when the evaluate operation completes. */
+  onEnd?: Callback<EvaluateEndEvent<RUNTIME_CONTEXT>>;
+  /** Internal. For test use only. May change without notice. */
+  _internal?: {
+    generateCallId?: () => string;
+  };
 }): Promise<EvaluationResult<QUESTIONS>> {
   const model = resolveEvaluationModel(modelArg);
 
@@ -57,50 +88,124 @@ export async function evaluate<
     }
   }
 
-  const { retry } = prepareRetries({ maxRetries, abortSignal });
-  const result = await retry(() => {
-    abortSignal?.throwIfAborted();
-    return model.doEvaluate({
-      state,
-      questions,
-      abortSignal,
-      headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
-      providerOptions,
-    });
+  const callId = generateCallId();
+  const { maxRetries: resolvedMaxRetries, retry } = prepareRetries({
+    maxRetries,
+    abortSignal,
   });
-
-  abortSignal?.throwIfAborted();
-  validateEvaluationAnswers({
-    questions,
-    answers: result.answers,
-    rounding: result.rounding,
+  const telemetryDispatcher = createRestrictedTelemetryDispatcher({
+    telemetry,
   });
-  logWarnings({
-    warnings: result.warnings,
+  const runInTracingChannelSpan =
+    telemetryDispatcher.runInTracingChannelSpan ??
+    (async <T>({ execute }: { execute: () => PromiseLike<T> }) =>
+      await execute());
+  const startEvent = {
+    callId,
+    operationId: 'ai.evaluate' as const,
+    runtimeContext,
     provider: model.provider,
-    model: model.modelId,
-  });
+    modelId: model.modelId,
+    state,
+    questions,
+    maxRetries: resolvedMaxRetries,
+    headers,
+    providerOptions,
+  };
+  const tracingStartEvent = {
+    ...startEvent,
+    runtimeContext: filterIncludedContext({
+      context: runtimeContext,
+      includeContext: telemetry?.includeRuntimeContext,
+    }),
+  };
 
-  const inputTokens = result.usage?.inputTokens;
-  const outputTokens = result.usage?.outputTokens;
+  return await runInTracingChannelSpan({
+    type: 'experimental_evaluate',
+    event: tracingStartEvent,
+    execute: async () => {
+      await notify({
+        event: startEvent,
+        callbacks: [onStart, telemetryDispatcher.onStart],
+      });
 
-  return {
-    answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
-    usage: {
-      inputTokens,
-      outputTokens,
-      totalTokens:
-        inputTokens != null && outputTokens != null
-          ? inputTokens + outputTokens
-          : undefined,
-    },
-    warnings: result.warnings,
-    rounding: result.rounding,
-    providerMetadata: result.providerMetadata,
-    response: {
-      ...result.response,
-      timestamp: result.response?.timestamp ?? new Date(),
-      modelId: result.response?.modelId ?? model.modelId,
+      try {
+        const modelCallEvent = {
+          callId,
+          operationId: 'ai.evaluate.doEvaluate' as const,
+          provider: model.provider,
+          modelId: model.modelId,
+          state,
+          questions,
+        };
+        await notify({
+          event: modelCallEvent,
+          callbacks: [
+            telemetryDispatcher.experimental_onEvaluationModelCallStart,
+          ],
+        });
+        const result = await retry(async () => {
+          abortSignal?.throwIfAborted();
+          return await model.doEvaluate({
+            state,
+            questions,
+            abortSignal,
+            headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
+            providerOptions,
+          });
+        });
+
+        abortSignal?.throwIfAborted();
+        validateEvaluationAnswers({
+          questions,
+          answers: result.answers,
+          rounding: result.rounding,
+        });
+        await notify({
+          event: { ...modelCallEvent, ...result },
+          callbacks: [
+            telemetryDispatcher.experimental_onEvaluationModelCallEnd,
+          ],
+        });
+
+        logWarnings({
+          warnings: result.warnings,
+          provider: model.provider,
+          model: model.modelId,
+        });
+
+        const inputTokens = result.usage?.inputTokens;
+        const outputTokens = result.usage?.outputTokens;
+        const evaluationResult: EvaluationResult<QUESTIONS> = {
+          answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
+          usage: {
+            inputTokens,
+            outputTokens,
+            totalTokens:
+              inputTokens != null && outputTokens != null
+                ? inputTokens + outputTokens
+                : undefined,
+          },
+          warnings: result.warnings,
+          rounding: result.rounding,
+          providerMetadata: result.providerMetadata,
+          response: {
+            ...result.response,
+            timestamp: result.response?.timestamp ?? new Date(),
+            modelId: result.response?.modelId ?? model.modelId,
+          },
+        };
+
+        await notify({
+          event: { ...startEvent, ...evaluationResult },
+          callbacks: [onEnd, telemetryDispatcher.onEnd],
+        });
+
+        return evaluationResult;
+      } catch (error) {
+        await telemetryDispatcher.onError?.({ callId, error });
+        throw error;
+      }
     },
-  };
+  });
 }

package/src/evaluate/index.ts

@@ -1,4 +1,10 @@
 export { evaluate as experimental_evaluate } from './evaluate';
+export type {
+  EvaluateStartEvent as Experimental_EvaluateStartEvent,
+  EvaluateEndEvent as Experimental_EvaluateEndEvent,
+  EvaluationModelCallStartEvent as Experimental_EvaluationModelCallStartEvent,
+  EvaluationModelCallEndEvent as Experimental_EvaluationModelCallEndEvent,
+} from './evaluate-events';
 export type {
   EvaluationModel as Experimental_EvaluationModel,
   EvaluationQuestion as Experimental_EvaluationQuestion,

package/src/evaluate/restricted-telemetry-dispatcher.ts

@@ -0,0 +1,46 @@
+import type { Context } from '@ai-sdk/provider-utils';
+import { createTelemetryDispatcher } from '../telemetry/create-telemetry-dispatcher';
+import { filterIncludedContext } from '../telemetry/filter-included-context';
+import type { TelemetryDispatcher } from '../telemetry/telemetry';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
+import type { Callback } from '../util/callback';
+import type { EvaluateEndEvent, EvaluateStartEvent } from './evaluate-events';
+
+export function createRestrictedTelemetryDispatcher<
+  RUNTIME_CONTEXT extends Context,
+>({
+  telemetry,
+}: {
+  telemetry?: TelemetryOptions<RUNTIME_CONTEXT>;
+}): Omit<
+  TelemetryDispatcher,
+  | 'onStart'
+  | 'onEnd'
+  | 'experimental_onEvaluateStart'
+  | 'experimental_onEvaluateEnd'
+> & {
+  onStart: Callback<EvaluateStartEvent<RUNTIME_CONTEXT>>;
+  onEnd: Callback<EvaluateEndEvent<RUNTIME_CONTEXT>>;
+} {
+  const dispatcher = createTelemetryDispatcher({ telemetry });
+
+  return {
+    ...dispatcher,
+    onStart: event =>
+      dispatcher.experimental_onEvaluateStart?.({
+        ...event,
+        runtimeContext: filterIncludedContext({
+          context: event.runtimeContext,
+          includeContext: telemetry?.includeRuntimeContext,
+        }),
+      }),
+    onEnd: event =>
+      dispatcher.experimental_onEvaluateEnd?.({
+        ...event,
+        runtimeContext: filterIncludedContext({
+          context: event.runtimeContext,
+          includeContext: telemetry?.includeRuntimeContext,
+        }),
+      }),
+  };
+}

package/src/generate-text/convert-language-model-content.ts

@@ -4,6 +4,10 @@ import { ToolCallNotFoundForApprovalError } from '../error/tool-call-not-found-f
 import { getOwn } from '../util/get-own';
 import type { ContentPart } from './content-part';
 import { DefaultGeneratedFile } from './generated-file';
+import {
+  resolveGeneratedFileData,
+  type GeneratedFileDataCache,
+} from './resolve-generated-file-data';
 import type { ToolApprovalRequestOutput } from './tool-approval-request-output';
 import type { ToolApprovalResponseOutput } from './tool-approval-response-output';
 import type { TypedToolCall } from './tool-call';
@@ -11,13 +15,15 @@ import type { TypedToolError } from './tool-error';
 import type { ToolOutput } from './tool-output';
 import type { TypedToolResult } from './tool-result';
 
-export function convertLanguageModelContent<TOOLS extends ToolSet>({
+export async function convertLanguageModelContent<TOOLS extends ToolSet>({
   content,
   toolCalls,
   toolOutputs,
   toolApprovalRequests,
   toolApprovalResponses,
   tools,
+  abortSignal,
+  generatedFileDataCache,
 }: {
   content: Array<LanguageModelV4Content>;
   toolCalls: Array<TypedToolCall<TOOLS>>;
@@ -25,7 +31,9 @@ export function convertLanguageModelContent<TOOLS extends ToolSet>({
   toolApprovalRequests: Array<ToolApprovalRequestOutput<TOOLS>>;
   toolApprovalResponses: Array<ToolApprovalResponseOutput<TOOLS>>;
   tools: TOOLS | undefined;
-}): Array<ContentPart<TOOLS>> {
+  abortSignal?: AbortSignal;
+  generatedFileDataCache?: GeneratedFileDataCache;
+}): Promise<Array<ContentPart<TOOLS>>> {
   const contentParts: Array<ContentPart<TOOLS>> = [];
   const toolOutputsWithApprovalResponses: Array<ToolOutput<TOOLS>> = [];
   const toolOutputsWithoutApprovalResponses: Array<ToolOutput<TOOLS>> = [];
@@ -49,10 +57,11 @@ export function convertLanguageModelContent<TOOLS extends ToolSet>({
         contentParts.push({
           type: part.type as 'file' | 'reasoning-file',
           file: new DefaultGeneratedFile({
-            data:
-              part.data.type === 'data'
-                ? part.data.data
-                : part.data.url.toString(),
+            data: await resolveGeneratedFileData({
+              data: part.data,
+              abortSignal,
+              cache: generatedFileDataCache,
+            }),
             mediaType: part.mediaType,
           }),
           ...(part.providerMetadata != null

package/src/generate-text/generate-text.ts

@@ -1099,13 +1099,16 @@ export async function generateText<
               > = {};
               const blockedToolCallIds = new Set<string>();
 
-              const modelCallContent = convertLanguageModelContent({
+              const generatedFileDataCache = new WeakMap();
+              const modelCallContent = await convertLanguageModelContent({
                 content: currentModelResponse.content,
                 toolCalls: stepToolCalls,
                 toolOutputs: [],
                 toolApprovalRequests: [],
                 toolApprovalResponses: [],
                 tools,
+                abortSignal: mergedAbortSignal,
+                generatedFileDataCache,
               });
 
               await notify({
@@ -1426,13 +1429,15 @@ export async function generateText<
               }
 
               // content:
-              const stepContent = convertLanguageModelContent({
+              const stepContent = await convertLanguageModelContent({
                 content: currentModelResponse.content,
                 toolCalls: stepToolCalls,
                 toolOutputs: clientToolOutputs,
                 toolApprovalRequests: Object.values(toolApprovalRequests),
                 toolApprovalResponses,
                 tools,
+                abortSignal: mergedAbortSignal,
+                generatedFileDataCache,
               });
 
               const stepResponseMessages = await toResponseMessages({

package/src/generate-text/prune-messages.ts

@@ -47,7 +47,9 @@ export function pruneMessages({
 
       return {
         ...message,
-        content: message.content.filter(part => part.type !== 'reasoning'),
+        content: message.content.filter(
+          part => part.type !== 'reasoning' && part.type !== 'reasoning-file',
+        ),
       };
     });
   }

package/src/generate-text/resolve-generated-file-data.ts

@@ -0,0 +1,41 @@
+import type { LanguageModelV4Content } from '@ai-sdk/provider';
+import { download } from '../util/download/download';
+
+type GeneratedFileData = Extract<
+  LanguageModelV4Content,
+  { type: 'file' | 'reasoning-file' }
+>['data'];
+
+export type GeneratedFileDataCache = WeakMap<
+  GeneratedFileData,
+  string | Uint8Array
+>;
+
+export async function resolveGeneratedFileData({
+  data,
+  abortSignal,
+  cache,
+}: {
+  data: GeneratedFileData;
+  abortSignal?: AbortSignal;
+  cache?: GeneratedFileDataCache;
+}): Promise<string | Uint8Array> {
+  if (data.type === 'data') {
+    return data.data;
+  }
+
+  const cachedData = cache?.get(data);
+  if (cachedData != null) {
+    return cachedData;
+  }
+
+  const downloadedData = (
+    await download({
+      url: data.url,
+      abortSignal,
+    })
+  ).data;
+
+  cache?.set(data, downloadedData);
+  return downloadedData;
+}

package/src/generate-text/stream-language-model-call.ts

@@ -53,6 +53,7 @@ import type {
 } from './language-model-events';
 import type { Output } from './output';
 import { parseToolCall } from './parse-tool-call';
+import { resolveGeneratedFileData } from './resolve-generated-file-data';
 import type {
   TextStreamFilePart,
   TextStreamPart,
@@ -565,10 +566,10 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform<
         case 'file':
         case 'reasoning-file': {
           const file = new DefaultGeneratedFileWithType({
-            data:
-              chunk.data.type === 'data'
-                ? chunk.data.data
-                : chunk.data.url.toString(),
+            data: await resolveGeneratedFileData({
+              data: chunk.data,
+              abortSignal,
+            }),
             mediaType: chunk.mediaType,
           });
 

package/src/prompt/convert-to-language-model-prompt.ts

@@ -661,6 +661,7 @@ export function mapToolResultOutput({
         }
         case 'file-url': {
           const mediaType = item.mediaType ?? getMediaTypeFromUrl(item.url);
+          const url = new URL(item.url);
           let message = `The "file-url" type for tool result content is deprecated. Use the "file" type with mediaType and { type: 'url', url } instead.`;
           if (!item.mediaType) {
             const inferenceSuffix =
@@ -676,7 +677,11 @@ export function mapToolResultOutput({
           });
           return {
             type: 'file' as const,
-            data: { type: 'url' as const, url: new URL(item.url) },
+            data: {
+              type: 'url' as const,
+              url,
+              ...(url.toString() !== item.url ? { originalUrl: item.url } : {}),
+            },
             mediaType,
             providerOptions: item.providerOptions,
           };
@@ -732,6 +737,7 @@ export function mapToolResultOutput({
           };
         }
         case 'image-url': {
+          const url = new URL(item.url);
           warnings.push({
             type: 'deprecated',
             setting: '"tool-result" content of type "image-url"',
@@ -739,7 +745,11 @@ export function mapToolResultOutput({
           });
           return {
             type: 'file' as const,
-            data: { type: 'url' as const, url: new URL(item.url) },
+            data: {
+              type: 'url' as const,
+              url,
+              ...(url.toString() !== item.url ? { originalUrl: item.url } : {}),
+            },
             mediaType: 'image',
             providerOptions: item.providerOptions,
           };

package/src/prompt/file-part-data.ts

@@ -41,6 +41,16 @@ function convertUrlToFilePartData(url: URL): ConvertResult {
   return { data: { type: 'url', url }, mediaType: undefined };
 }
 
+function convertUrlStringToFilePartData(content: string): ConvertResult {
+  const result = convertUrlToFilePartData(new URL(content));
+
+  if (result.data.type === 'url' && result.data.url.toString() !== content) {
+    result.data.originalUrl = content;
+  }
+
+  return result;
+}
+
 function convertInlineDataToFilePartData(content: DataContent): ConvertResult {
   if (content instanceof Uint8Array) {
     return { data: { type: 'data', data: content }, mediaType: undefined };
@@ -108,7 +118,7 @@ export function convertToLanguageModelV4FilePart(
 
   if (typeof content === 'string') {
     try {
-      return convertUrlToFilePartData(new URL(content));
+      return convertUrlStringToFilePartData(content);
     } catch {
       return convertInlineDataToFilePartData(content);
     }

package/src/telemetry/create-telemetry-dispatcher.ts

@@ -163,6 +163,18 @@ export function createTelemetryDispatcher({
     onEmbedEnd: mergeTelemetryCallback('onEmbedEnd'),
     onRerankStart: mergeTelemetryCallback('onRerankStart'),
     onRerankEnd: mergeTelemetryCallback('onRerankEnd'),
+    experimental_onEvaluateStart: mergeTelemetryCallback(
+      'experimental_onEvaluateStart',
+    ),
+    experimental_onEvaluationModelCallStart: mergeTelemetryCallback(
+      'experimental_onEvaluationModelCallStart',
+    ),
+    experimental_onEvaluationModelCallEnd: mergeTelemetryCallback(
+      'experimental_onEvaluationModelCallEnd',
+    ),
+    experimental_onEvaluateEnd: mergeTelemetryCallback(
+      'experimental_onEvaluateEnd',
+    ),
     onEnd: mergeTelemetryCallback('onEnd'),
     onAbort: mergeTelemetryCallback('onAbort'),
     onError: mergeTelemetryCallback('onError'),