ai 7.0.110 → 7.0.111

package/dist/internal/index.d.ts

@@ -1,5 +1,5 @@
 import * as _ai_sdk_provider from '@ai-sdk/provider';
-import { EmbeddingModelV4Embedding, LanguageModelV4Source, SharedV4Warning, LanguageModelV4, LanguageModelV3, LanguageModelV2, SharedV4ProviderMetadata, JSONObject, LanguageModelV4Usage, LanguageModelV4CallOptions, LanguageModelV4Prompt, AISDKError, LanguageModelV4ToolCall, JSONSchema7, ProviderV4, ProviderV3, ProviderV2, LanguageModelV4ToolResultOutput, LanguageModelV4ToolChoice, LanguageModelV4FunctionTool, LanguageModelV4ProviderTool } from '@ai-sdk/provider';
+import { EmbeddingModelV4Embedding, LanguageModelV4Source, SharedV4Warning, LanguageModelV4, LanguageModelV3, LanguageModelV2, SharedV4ProviderMetadata, JSONObject, LanguageModelV4Usage, Experimental_EvaluationModelV4Question, Experimental_EvaluationModelV4Result, Experimental_EvaluationModelV4CallOptions, LanguageModelV4CallOptions, LanguageModelV4Prompt, AISDKError, LanguageModelV4ToolCall, JSONSchema7, ProviderV4, ProviderV3, ProviderV2, LanguageModelV4ToolResultOutput, LanguageModelV4ToolChoice, LanguageModelV4FunctionTool, LanguageModelV4ProviderTool } from '@ai-sdk/provider';
 import { GatewayModelId } from '@ai-sdk/gateway';
 import { ModelMessage, AssistantModelMessage, ToolModelMessage, Context, ProviderOptions, ToolSet, SystemModelMessage, InferToolSetContext, Arrayable, InferToolInput, InferToolOutput, ReasoningPart, ReasoningFilePart, MaybePromiseLike, ToolExecutionOptions, InferToolContext, HasRequiredKey, ToolResultOutput, Tool, Experimental_SandboxSession, ShouldRetryFunction, RetryFunction, FlexibleSchema, ToolApprovalRequest, ToolApprovalResponse, ToolResultPart } from '@ai-sdk/provider-utils';
 export { convertAsyncIteratorToReadableStream } from '@ai-sdk/provider-utils';
@@ -351,6 +351,127 @@ type EmbeddingModelCallEndEvent = {
     readonly usage: EmbeddingModelUsage;
 };
 
+type EvaluationQuestion = Experimental_EvaluationModelV4Question;
+type EvaluationAnswer<QUESTION extends EvaluationQuestion> = QUESTION extends {
+    type: 'choice';
+    criteria: infer CRITERIA;
+} ? {
+    type: 'choice';
+    choice: Extract<keyof CRITERIA, string>;
+    probabilities?: Record<Extract<keyof CRITERIA, string>, number>;
+} : QUESTION extends {
+    type: 'score';
+} ? {
+    type: 'score';
+    score: number;
+    probabilities?: Record<string, number>;
+} : {
+    type: 'boolean';
+    probability: number;
+};
+type EvaluationResult<QUESTIONS extends Record<string, EvaluationQuestion>> = {
+    readonly answers: {
+        [ID in keyof QUESTIONS]: EvaluationAnswer<QUESTIONS[ID]>;
+    };
+    readonly usage: {
+        inputTokens: number | undefined;
+        outputTokens: number | undefined;
+        totalTokens: number | undefined;
+    };
+    readonly warnings: Experimental_EvaluationModelV4Result['warnings'];
+    readonly rounding: Experimental_EvaluationModelV4Result['rounding'];
+    readonly providerMetadata: Experimental_EvaluationModelV4Result['providerMetadata'];
+    readonly response: NonNullable<Experimental_EvaluationModelV4Result['response']> & {
+        timestamp: Date;
+        modelId: string;
+    };
+};
+
+/**
+ * Event passed to the `onStart` callback for evaluation operations.
+ *
+ * Called when the operation begins, before the evaluation model is called.
+ */
+type EvaluateStartEvent<RUNTIME_CONTEXT extends Context = Context> = {
+    /** User-defined runtime context. */
+    readonly runtimeContext: RUNTIME_CONTEXT;
+    /** Unique identifier for this evaluation call, used to correlate events. */
+    readonly callId: string;
+    /** Identifies the operation type (`ai.evaluate`). */
+    readonly operationId: 'ai.evaluate';
+    /** The provider identifier. */
+    readonly provider: string;
+    /** The evaluation model identifier. */
+    readonly modelId: string;
+    /** The shared state being evaluated. */
+    readonly state: Experimental_EvaluationModelV4CallOptions['state'];
+    /** The questions being evaluated against the shared state. */
+    readonly questions: Readonly<Record<string, EvaluationQuestion>>;
+    /** Maximum number of retries for the evaluation model call. */
+    readonly maxRetries: number;
+    /** Additional HTTP headers sent with the request. */
+    readonly headers: Record<string, string> | undefined;
+    /** Additional provider-specific options. */
+    readonly providerOptions: ProviderOptions;
+};
+/**
+ * Event passed to the `onEnd` callback for evaluation operations.
+ *
+ * Called when the operation completes successfully.
+ */
+type EvaluateEndEvent<RUNTIME_CONTEXT extends Context = Context> = EvaluateStartEvent<RUNTIME_CONTEXT> & {
+    /** Exactly one typed answer per question ID. */
+    readonly answers: EvaluationResult<Record<string, EvaluationQuestion>>['answers'];
+    /** Token usage for the evaluation operation. */
+    readonly usage: EvaluationResult<Record<string, EvaluationQuestion>>['usage'];
+    /** Warnings from the evaluation model. */
+    readonly warnings: EvaluationResult<Record<string, EvaluationQuestion>>['warnings'];
+    /** Provider-declared decimal precision for probabilities and scores. */
+    readonly rounding: EvaluationResult<Record<string, EvaluationQuestion>>['rounding'];
+    /** Optional provider-specific metadata. */
+    readonly providerMetadata: EvaluationResult<Record<string, EvaluationQuestion>>['providerMetadata'];
+    /** Response metadata, including the resolved model ID and timestamp. */
+    readonly response: EvaluationResult<Record<string, EvaluationQuestion>>['response'];
+};
+/**
+ * Event fired when the evaluation model call begins.
+ *
+ * The logical model call includes any provider retries.
+ */
+type EvaluationModelCallStartEvent = {
+    /** Unique identifier for the outer evaluation call. */
+    readonly callId: string;
+    /** Identifies the inner operation (`ai.evaluate.doEvaluate`). */
+    readonly operationId: 'ai.evaluate.doEvaluate';
+    /** The provider identifier. */
+    readonly provider: string;
+    /** The evaluation model identifier. */
+    readonly modelId: string;
+    /** The shared state being evaluated. */
+    readonly state: Experimental_EvaluationModelV4CallOptions['state'];
+    /** The questions being evaluated against the shared state. */
+    readonly questions: Readonly<Record<string, EvaluationQuestion>>;
+};
+/**
+ * Event fired after the evaluation model response has been validated.
+ *
+ * Contains the result of the logical model call, including any retries.
+ */
+type EvaluationModelCallEndEvent = EvaluationModelCallStartEvent & {
+    /** Exactly one answer per question ID. */
+    readonly answers: Experimental_EvaluationModelV4Result['answers'];
+    /** Token usage reported by the evaluation model. */
+    readonly usage?: Experimental_EvaluationModelV4Result['usage'];
+    /** Warnings from the evaluation model. */
+    readonly warnings: Experimental_EvaluationModelV4Result['warnings'];
+    /** Provider-declared decimal precision for probabilities and scores. */
+    readonly rounding?: Experimental_EvaluationModelV4Result['rounding'];
+    /** Optional provider-specific metadata. */
+    readonly providerMetadata?: Experimental_EvaluationModelV4Result['providerMetadata'];
+    /** Optional raw response metadata from the provider. */
+    readonly response?: Experimental_EvaluationModelV4Result['response'];
+};
+
 /**
  * Model-facing generation controls. These settings influence how the model
  * generates its response (token limits, sampling, penalties, stop sequences,
@@ -2345,7 +2466,7 @@ type RerankingModelCallEndEvent = {
     }>;
 };
 
-type TelemetryTracingEventType = 'generateText' | 'streamText' | 'step' | 'languageModelCall' | 'executeTool' | 'embed' | 'embedMany' | 'rerank';
+type TelemetryTracingEventType = 'generateText' | 'streamText' | 'step' | 'languageModelCall' | 'executeTool' | 'embed' | 'embedMany' | 'rerank' | 'experimental_evaluate';
 
 type TracingChannelContext = {
     run<T>(execute: () => T): T;
@@ -2388,6 +2509,10 @@ interface TelemetryDispatcher {
     onEmbedEnd?: Callback<EmbeddingModelCallEndEvent>;
     onRerankStart?: Callback<RerankingModelCallStartEvent>;
     onRerankEnd?: Callback<RerankingModelCallEndEvent>;
+    experimental_onEvaluateStart?: Callback<EvaluateStartEvent>;
+    experimental_onEvaluationModelCallStart?: Callback<EvaluationModelCallStartEvent>;
+    experimental_onEvaluationModelCallEnd?: Callback<EvaluationModelCallEndEvent>;
+    experimental_onEvaluateEnd?: Callback<EvaluateEndEvent>;
     onEnd?: Callback<OperationEndEvent>;
     onAbort?: Callback<GenerateTextAbortEvent<ToolSet>>;
     onError?: Callback<unknown>;
@@ -2489,6 +2614,20 @@ interface Telemetry {
      * Contains the ranking results from the model response.
      */
     onRerankEnd?: Callback<InferTelemetryEvent<RerankingModelCallEndEvent>>;
+    /** Called when an experimental evaluation operation begins. */
+    experimental_onEvaluateStart?: Callback<InferTelemetryEvent<EvaluateStartEvent>>;
+    /**
+     * Called immediately before an experimental evaluation model call begins.
+     * The logical model call includes any provider retries.
+     */
+    experimental_onEvaluationModelCallStart?: Callback<InferTelemetryEvent<EvaluationModelCallStartEvent>>;
+    /**
+     * Called after an experimental evaluation model response has been validated.
+     * The logical model call includes any provider retries.
+     */
+    experimental_onEvaluationModelCallEnd?: Callback<InferTelemetryEvent<EvaluationModelCallEndEvent>>;
+    /** Called when an experimental evaluation operation completes. */
+    experimental_onEvaluateEnd?: Callback<InferTelemetryEvent<EvaluateEndEvent>>;
     /**
      * Called when an operation completes. Fired for text generation
      * (generateText/streamText), object generation (generateObject/streamObject),

package/dist/internal/index.js

@@ -91,7 +91,7 @@ import {
 } from "@ai-sdk/provider-utils";
 
 // src/version.ts
-var VERSION = true ? "7.0.110" : "0.0.0-test";
+var VERSION = true ? "7.0.111" : "0.0.0-test";
 
 // src/util/download/download.ts
 var download = async ({
@@ -2723,6 +2723,18 @@ function createTelemetryDispatcher({
     onEmbedEnd: mergeTelemetryCallback("onEmbedEnd"),
     onRerankStart: mergeTelemetryCallback("onRerankStart"),
     onRerankEnd: mergeTelemetryCallback("onRerankEnd"),
+    experimental_onEvaluateStart: mergeTelemetryCallback(
+      "experimental_onEvaluateStart"
+    ),
+    experimental_onEvaluationModelCallStart: mergeTelemetryCallback(
+      "experimental_onEvaluationModelCallStart"
+    ),
+    experimental_onEvaluationModelCallEnd: mergeTelemetryCallback(
+      "experimental_onEvaluationModelCallEnd"
+    ),
+    experimental_onEvaluateEnd: mergeTelemetryCallback(
+      "experimental_onEvaluateEnd"
+    ),
     onEnd: mergeTelemetryCallback("onEnd"),
     onAbort: mergeTelemetryCallback("onAbort"),
     onError: mergeTelemetryCallback("onError"),