ai 7.0.101 → 7.0.103

package/src/evaluate/validate-evaluation.ts

@@ -0,0 +1,298 @@
+import {
+  InvalidResponseDataError,
+  type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
+  type Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
+} from '@ai-sdk/provider';
+import { InvalidArgumentError } from '../error/invalid-argument-error';
+
+// Absolute tolerance for sums and means. Never renormalize provider output.
+const tolerance = 1e-6;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  if (value == null || typeof value !== 'object' || Array.isArray(value))
+    return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+function isJSON(value: unknown, ancestors = new Set<object>()): boolean {
+  if (value === null || typeof value === 'string' || typeof value === 'boolean')
+    return true;
+  if (typeof value === 'number') return Number.isFinite(value);
+  if (typeof value !== 'object' || (!Array.isArray(value) && !isRecord(value)))
+    return false;
+  if (ancestors.has(value)) return false;
+  ancestors.add(value);
+  const valid =
+    Object.getOwnPropertySymbols(value).length === 0 &&
+    (Array.isArray(value)
+      ? Array.from(value).every(item => isJSON(item, ancestors))
+      : Object.values(value).every(item => isJSON(item, ancestors)));
+  ancestors.delete(value);
+  return valid;
+}
+
+function isInput(value: unknown): boolean {
+  return (
+    (typeof value === 'string' || Array.isArray(value) || isRecord(value)) &&
+    isJSON(value)
+  );
+}
+
+function invalidInput(
+  parameter: string,
+  value: unknown,
+  message: string,
+): never {
+  throw new InvalidArgumentError({ parameter, value, message });
+}
+
+export function validateEvaluationInput({
+  state,
+  questions,
+}: EvaluationModelV4CallOptions) {
+  if (!isInput(state)) {
+    invalidInput(
+      'state',
+      state,
+      'must be a JSON-compatible string, object, or array',
+    );
+  }
+  if (!isRecord(questions) || Object.keys(questions).length === 0) {
+    invalidInput('questions', questions, 'must be a nonempty question map');
+  }
+
+  for (const [id, question] of Object.entries(questions)) {
+    const parameter = `questions.${id}`;
+    if (!isRecord(question) || !isInput(question.instructions)) {
+      invalidInput(
+        parameter,
+        question,
+        'instructions must be a JSON-compatible string, object, or array',
+      );
+    }
+    const criteria = question.criteria;
+    switch (question.type) {
+      case 'choice':
+        if (!isRecord(criteria) || Object.keys(criteria).length === 0) {
+          invalidInput(
+            parameter,
+            question,
+            'choice criteria must be a nonempty option map',
+          );
+        }
+        break;
+      case 'score':
+        if (!Array.isArray(criteria) || criteria.length < 2) {
+          invalidInput(
+            parameter,
+            question,
+            'score criteria must contain at least two ordered levels',
+          );
+        }
+        break;
+      case 'boolean':
+        if (criteria === undefined) continue;
+        if (
+          !isRecord(criteria) ||
+          Object.keys(criteria).some(key => key !== 'true' && key !== 'false')
+        ) {
+          invalidInput(
+            parameter,
+            question,
+            'boolean criteria may only describe true and false',
+          );
+        }
+        break;
+      default:
+        invalidInput(
+          parameter,
+          question,
+          'question type must be choice, score, or boolean',
+        );
+    }
+    if (
+      !isJSON(criteria) ||
+      Object.values(criteria).some(value => value !== null && !isInput(value))
+    ) {
+      invalidInput(
+        parameter,
+        question,
+        'criteria descriptions must be JSON-compatible strings, objects, arrays, or null',
+      );
+    }
+  }
+}
+
+function invalidAnswer(answers: unknown, message: string): never {
+  throw new InvalidResponseDataError({ data: answers, message });
+}
+
+function isProbability(value: unknown): value is number {
+  return (
+    typeof value === 'number' &&
+    Number.isFinite(value) &&
+    value >= 0 &&
+    value <= 1
+  );
+}
+
+function hasExactKeys(value: Record<string, unknown>, keys: string[]): boolean {
+  return (
+    Object.keys(value).length === keys.length &&
+    keys.every(key => Object.hasOwn(value, key))
+  );
+}
+
+function validateDistribution(
+  value: unknown,
+  keys: string[],
+  answers: unknown,
+  id: string,
+  roundingError: number,
+): asserts value is Record<string, number> {
+  if (
+    !isRecord(value) ||
+    !hasExactKeys(value, keys) ||
+    !Object.values(value).every(isProbability)
+  ) {
+    invalidAnswer(
+      answers,
+      `Question "${id}" must have a complete distribution of finite probabilities in [0, 1].`,
+    );
+  }
+  const sum = Object.values(value).reduce<number>(
+    (total, probability) => total + (probability as number),
+    0,
+  );
+  if (Math.abs(sum - 1) > tolerance + keys.length * roundingError) {
+    invalidAnswer(
+      answers,
+      `Question "${id}" probabilities must sum to 1 within the declared rounding precision.`,
+    );
+  }
+}
+
+export function validateEvaluationAnswers({
+  questions,
+  answers,
+  rounding,
+}: {
+  questions: EvaluationModelV4CallOptions['questions'];
+  answers: unknown;
+  rounding?: EvaluationModelV4Result['rounding'];
+}) {
+  function roundingError(decimals: number | undefined): number {
+    if (decimals === undefined) return 0;
+    if (!Number.isInteger(decimals) || decimals < 0 || decimals > 15) {
+      invalidAnswer(
+        answers,
+        'Evaluation rounding decimals must be integers between 0 and 15.',
+      );
+    }
+    return 0.5 * 10 ** -decimals;
+  }
+  const probabilityError = roundingError(rounding?.probabilityDecimals);
+  const scoreError = roundingError(rounding?.scoreDecimals);
+  if (!isRecord(answers) || !hasExactKeys(answers, Object.keys(questions))) {
+    invalidAnswer(
+      answers,
+      'Evaluation must return exactly one answer for every question.',
+    );
+  }
+
+  for (const [id, question] of Object.entries(questions)) {
+    const answer = answers[id];
+    if (!isRecord(answer) || answer.type !== question.type) {
+      invalidAnswer(
+        answers,
+        `Question "${id}" returned an answer with the wrong type.`,
+      );
+    }
+
+    switch (question.type) {
+      case 'choice': {
+        if (
+          typeof answer.choice !== 'string' ||
+          !Object.hasOwn(question.criteria, answer.choice)
+        ) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" selected an unknown option.`,
+          );
+        }
+        if (answer.probabilities !== undefined) {
+          validateDistribution(
+            answer.probabilities,
+            Object.keys(question.criteria),
+            answers,
+            id,
+            probabilityError,
+          );
+          const selected = answer.probabilities[answer.choice];
+          if (
+            Object.values(answer.probabilities).some(
+              probability => probability > selected + tolerance,
+            )
+          ) {
+            invalidAnswer(
+              answers,
+              `Question "${id}" did not select a highest-probability option.`,
+            );
+          }
+        }
+        break;
+      }
+      case 'score': {
+        if (
+          typeof answer.score !== 'number' ||
+          !Number.isFinite(answer.score) ||
+          answer.score < 0 ||
+          answer.score > question.criteria.length - 1
+        ) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" score must be in [0, ${question.criteria.length - 1}].`,
+          );
+        }
+        if (answer.probabilities !== undefined) {
+          const keys = question.criteria.map((_, index) => String(index));
+          validateDistribution(
+            answer.probabilities,
+            keys,
+            answers,
+            id,
+            probabilityError,
+          );
+          const mean = Object.entries(answer.probabilities).reduce(
+            (total, [index, probability]) =>
+              total + Number(index) * probability,
+            0,
+          );
+          const meanRoundingError = keys.reduce(
+            (total, index) => total + Number(index) * probabilityError,
+            0,
+          );
+          if (
+            Math.abs(mean - answer.score) >
+            tolerance + meanRoundingError + scoreError
+          ) {
+            invalidAnswer(
+              answers,
+              `Question "${id}" score must equal the probability-weighted mean within the declared rounding precision.`,
+            );
+          }
+        }
+        break;
+      }
+      case 'boolean':
+        if (!isProbability(answer.probability)) {
+          invalidAnswer(
+            answers,
+            `Question "${id}" must return P(true) as a finite probability in [0, 1].`,
+          );
+        }
+        break;
+    }
+  }
+}

package/src/generate-text/execute-tools-from-stream.ts

@@ -25,6 +25,7 @@ import type {
   OnToolExecutionEndCallback,
   OnToolExecutionStartCallback,
 } from './tool-execution-events';
+import type { StaticToolOutputDenied } from './tool-output-denied';
 
 export type ToolExecutionEndStreamPart = {
   type: 'tool-execution-end';
@@ -34,6 +35,7 @@ export type ToolExecutionEndStreamPart = {
 
 export type ExecuteToolsStreamPart<TOOLS extends ToolSet = ToolSet> =
   | LanguageModelStreamPart<TOOLS>
+  | StaticToolOutputDenied<TOOLS>
   | ToolExecutionEndStreamPart
   | StreamRetryAttemptBoundaryPart;
 
@@ -180,6 +182,11 @@ export function executeToolsFromStream<
                   reason: toolApprovalStatus.reason,
                   providerExecuted: chunk.providerExecuted,
                 });
+                controller.enqueue({
+                  type: 'tool-output-denied',
+                  toolCallId: chunk.toolCallId,
+                  toolName: chunk.toolName,
+                } as StaticToolOutputDenied<TOOLS>);
 
                 return; // don't execute tool
               }

package/src/generate-text/generate-text.ts

@@ -106,6 +106,7 @@ import type { ToolApprovalConfiguration } from './tool-approval-configuration';
 import type { ToolApprovalRequestOutput } from './tool-approval-request-output';
 import type { ToolApprovalResponseOutput } from './tool-approval-response-output';
 import {
+  appendToolCallerMessages,
   prepareToolsForToolCallers,
   resolveToolCallerConfiguration,
   type Experimental_ToolCallers,
@@ -906,16 +907,6 @@ export async function generateText<
                 prepareStepResult?.system ??
                 instructionsForNextStep;
 
-              const promptMessages = await convertToLanguageModelPrompt({
-                prompt: {
-                  instructions: stepInstructions,
-                  messages: prepareStepResult?.messages ?? stepInputMessages,
-                },
-                supportedUrls: await stepModel.supportedUrls,
-                download,
-                provider: stepModel.provider.split('.')[0],
-              });
-
               runtimeContext =
                 prepareStepResult?.runtimeContext ?? runtimeContext;
               toolsContext = prepareStepResult?.toolsContext ?? toolsContext;
@@ -927,6 +918,7 @@ export async function generateText<
               const {
                 executionTools: stepExecutionTools,
                 modelTools: stepModelTools,
+                toolCallerMessages,
               } = prepareToolsForToolCallers({
                 tools: stepActiveTools,
                 toolCallers: resolvedToolCallers,
@@ -952,8 +944,20 @@ export async function generateText<
                 toolChoice: prepareStepResult?.toolChoice ?? toolChoice,
               });
 
-              const stepMessages =
-                prepareStepResult?.messages ?? stepInputMessages;
+              const stepMessages = appendToolCallerMessages({
+                messages: prepareStepResult?.messages ?? stepInputMessages,
+                toolCallerMessages,
+              });
+
+              const promptMessages = await convertToLanguageModelPrompt({
+                prompt: {
+                  instructions: stepInstructions,
+                  messages: stepMessages,
+                },
+                supportedUrls: await stepModel.supportedUrls,
+                download,
+                provider: stepModel.provider.split('.')[0],
+              });
 
               const stepProviderOptions = mergeObjects(
                 providerOptions,

package/src/generate-text/stream-text.ts

@@ -146,6 +146,7 @@ import type {
 import { toResponseMessages } from './to-response-messages';
 import type { ToolApprovalConfiguration } from './tool-approval-configuration';
 import {
+  appendToolCallerMessages,
   prepareToolsForToolCallers,
   resolveToolCallerConfiguration,
   type Experimental_ToolCallers,
@@ -196,6 +197,7 @@ const isOutputChunkType = {
   'tool-call': true,
   'tool-result': false,
   'tool-error': false,
+  'tool-output-denied': false,
   'tool-execution-end': false,
   'model-call-start': false,
   'model-call-response-metadata': false,
@@ -2329,6 +2331,7 @@ class DefaultStreamTextResult<
           const {
             executionTools: stepExecutionTools,
             modelTools: stepModelTools,
+            toolCallerMessages,
           } = prepareToolsForToolCallers({
             tools: stepActiveTools,
             toolCallers: resolvedToolCallers,
@@ -2354,7 +2357,10 @@ class DefaultStreamTextResult<
             toolChoice: prepareStepResult?.toolChoice ?? toolChoice,
           });
 
-          const stepMessages = prepareStepResult?.messages ?? stepInputMessages;
+          const stepMessages = appendToolCallerMessages({
+            messages: prepareStepResult?.messages ?? stepInputMessages,
+            toolCallerMessages,
+          });
           currentStepMessages = stepMessages;
           const stepInstructions =
             prepareStepResult?.instructions ??
@@ -2833,7 +2839,8 @@ class DefaultStreamTextResult<
                     case 'tool-input-start':
                     case 'tool-input-end':
                     case 'tool-input-delta':
-                    case 'tool-approval-request': {
+                    case 'tool-approval-request':
+                    case 'tool-output-denied': {
                       enqueueStepPart(controller, chunk);
                       break;
                     }

package/src/generate-text/tool-caller-configuration.ts

@@ -1,8 +1,10 @@
 import {
   experimental_getToolCaller,
   type Experimental_ToolCallerTool,
+  type ModelMessage,
   type Tool,
   type ToolSet,
+  type UserModelMessage,
 } from '@ai-sdk/provider-utils';
 import { InvalidArgumentError } from '../error/invalid-argument-error';
 
@@ -86,15 +88,28 @@ export function prepareToolsForToolCallers({
 }): {
   executionTools: ToolSet | undefined;
   modelTools: ToolSet | undefined;
+  toolCallerMessages: UserModelMessage[];
 } {
   if (tools == null || toolCallers == null) {
-    return { executionTools: tools, modelTools: tools };
+    return {
+      executionTools: tools,
+      modelTools: tools,
+      toolCallerMessages: [],
+    };
   }
 
+  // Tools available to the runtime for execution.
   const executionTools: ToolSet = { ...tools };
+
+  // Tools exposed directly to the model.
   const modelTools: ToolSet = { ...tools };
+
+  // Tools routed through each local caller.
   const localToolsByCaller = new Map<string, ToolSet>();
 
+  // User messages describing caller-accessible tools.
+  const toolCallerMessages: UserModelMessage[] = [];
+
   for (const [toolName, callerNames] of Object.entries(toolCallers)) {
     const tool = executionTools[toolName];
     if (tool == null) {
@@ -146,13 +161,52 @@ export function prepareToolsForToolCallers({
       continue;
     }
 
-    const boundCaller = caller.bind(localToolsByCaller.get(callerName) ?? {});
+    const callerTools = localToolsByCaller.get(callerName) ?? {};
+    const boundCaller = caller.bind(callerTools);
     executionTools[callerName] = boundCaller;
 
     if (Object.prototype.hasOwnProperty.call(modelTools, callerName)) {
-      modelTools[callerName] = boundCaller;
+      if (caller.prepareModelMessage == null) {
+        modelTools[callerName] = boundCaller;
+      } else {
+        const content = caller.prepareModelMessage(callerTools);
+        if (content != null) {
+          toolCallerMessages.push({ role: 'user', content });
+        }
+      }
     }
   }
 
-  return { executionTools, modelTools };
+  return { executionTools, modelTools, toolCallerMessages };
+}
+
+export function appendToolCallerMessages({
+  messages,
+  toolCallerMessages,
+}: {
+  messages: ModelMessage[];
+  toolCallerMessages: UserModelMessage[];
+}): ModelMessage[] {
+  if (toolCallerMessages.length === 0) {
+    return messages;
+  }
+
+  const latestUserText = messages.findLast(
+    message => message.role === 'user' && typeof message.content === 'string',
+  )?.content;
+  const existingUserText = new Set(
+    latestUserText == null ? [] : [latestUserText],
+  );
+  const additions = toolCallerMessages.filter(message => {
+    if (
+      typeof message.content !== 'string' ||
+      existingUserText.has(message.content)
+    ) {
+      return false;
+    }
+    existingUserText.add(message.content);
+    return true;
+  });
+
+  return additions.length === 0 ? messages : [...messages, ...additions];
 }

package/src/index.ts

@@ -41,6 +41,7 @@ export * from './agent';
 export * from './batch';
 export * from './embed';
 export * from './error';
+export * from './evaluate';
 export * from './generate-image';
 export * from './generate-object';
 export * from './generate-speech';

package/src/realtime/__fixtures__/fake-live-websocket.ts

@@ -0,0 +1,71 @@
+import { vi, type Mock } from 'vitest';
+import type { RealtimeServerEvent } from '../../types/realtime-model';
+import { fakeStream } from './fake-realtime';
+
+export class FakeWebSocket {
+  static OPEN = 1;
+  static instances: FakeWebSocket[] = [];
+  readyState = 0;
+  bufferedAmount = 0;
+  onopen: (() => void) | null = null;
+  onclose: ((event: CloseEvent) => void) | null = null;
+  onerror: (() => void) | null = null;
+  onmessage: ((event: { data: unknown }) => void) | null = null;
+  sent: Array<{ type: string; [key: string]: unknown }> = [];
+  constructor(public url: string, public protocols?: string[]) { FakeWebSocket.instances.push(this); }
+  send = vi.fn((data: string) => { this.sent.push(JSON.parse(data)); });
+  close = vi.fn(() => { this.closeFromServer(); });
+  open() { this.readyState = 1; this.onopen?.(); }
+  emit(event: RealtimeServerEvent) { this.onmessage?.({ data: JSON.stringify(event) }); }
+  closeFromServer({ code = 1000, reason = '', wasClean = true }: Partial<Pick<CloseEvent, 'code' | 'reason' | 'wasClean'>> = {}) {
+    this.readyState = 3;
+    this.onclose?.({ code, reason, wasClean } as CloseEvent);
+  }
+}
+
+class FakeAudioNode {
+  connect: Mock = vi.fn();
+  disconnect: Mock = vi.fn();
+  onaudioprocess: ((event: { inputBuffer: { getChannelData: (channel: number) => Float32Array } }) => void) | null = null;
+  onended: (() => void) | null = null;
+  buffer: unknown;
+  start: Mock = vi.fn();
+  stop: Mock = vi.fn();
+}
+
+export class FakeAudioContext {
+  static instances: FakeAudioContext[] = [];
+  state = 'suspended';
+  currentTime = 0;
+  sampleRate: number;
+  destination = {};
+  processors: FakeAudioNode[] = [];
+  sources: FakeAudioNode[] = [];
+  mediaSources: FakeAudioNode[] = [];
+  constructor(options: { sampleRate: number }) { this.sampleRate = options.sampleRate; FakeAudioContext.instances.push(this); }
+  resume = vi.fn(async () => { this.state = 'running'; });
+  close = vi.fn(async () => { this.state = 'closed'; });
+  createScriptProcessor = vi.fn(() => {
+    const node = new FakeAudioNode(); this.processors.push(node); return node;
+  });
+  createMediaStreamSource = vi.fn(() => {
+    const node = new FakeAudioNode(); this.mediaSources.push(node); return node;
+  });
+  createBufferSource = vi.fn(() => {
+    const node = new FakeAudioNode(); this.sources.push(node); return node;
+  });
+  createBuffer = vi.fn((_channels: number, size: number, rate: number) => ({ duration: size / rate, getChannelData: () => new Float32Array(size) }));
+}
+
+export function installLiveWebSocket() {
+  const media = fakeStream();
+  const getUserMedia = vi.fn(async () => media.stream);
+  const fetch = vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<Response>>();
+  vi.stubGlobal('navigator', { mediaDevices: { getUserMedia } });
+  vi.stubGlobal('fetch', fetch);
+  FakeWebSocket.instances = [];
+  FakeAudioContext.instances = [];
+  vi.stubGlobal('WebSocket', FakeWebSocket);
+  vi.stubGlobal('AudioContext', FakeAudioContext);
+  return { ...media, getUserMedia, fetch };
+}

package/src/realtime/__fixtures__/fake-realtime.ts

@@ -0,0 +1,36 @@
+import { vi } from 'vitest';
+import type { RealtimeModel, RealtimeServerEvent } from '../../types/realtime-model';
+
+export class FakeTrack extends EventTarget {
+  enabled = true;
+  muted = false;
+  readyState = 'live';
+  stop = vi.fn(() => { this.readyState = 'ended'; });
+}
+
+export function fakeStream() {
+  const track = new FakeTrack();
+  return { track, stream: { getTracks: () => [track], getAudioTracks: () => [track] } as unknown as MediaStream };
+}
+
+export function liveModel(): RealtimeModel {
+  return {
+    specificationVersion: 'v4', provider: 'test', modelId: 'live',
+    capabilities: { conversation: 'continuous', transports: ['websocket'], connections: ['server-websocket'], startup: 'session-start', finalization: 'session-close' },
+    getWebSocketConfig: vi.fn(),
+    doCreateClientSecret: vi.fn(),
+    buildSessionConfig: vi.fn(),
+    parseServerEvent: event => event as RealtimeServerEvent,
+    serializeClientEvent: async event => event,
+  };
+}
+
+export async function flushEvents() {
+  for (let i = 0; i < 100; i++) await Promise.resolve();
+}
+
+export function deferred<T>() {
+  let resolve: (value: T) => void = () => {};
+  const promise = new Promise<T>(r => { resolve = r; });
+  return { promise, resolve };
+}