@vitest-evals/harness-ai-sdk 0.13.1 → 0.15.0

package/README.md

@@ -108,8 +108,9 @@ describeEval("refund agent", {
 
 The adapter infers:
 
-- normalized session and tool-call traces from AI SDK `steps`
-- usage diagnostics from `totalUsage` / `usage`
+- normalized session transcripts and tool calls from AI SDK `steps`
+- usage diagnostics from `totalUsage`, aggregated `steps[].usage`, or
+  top-level `usage` for non-step results
 - typed `run.output` from explicit `run()` results that return `output`, from
   common AI SDK provider fields such as `object` and `text`, or from a typed
   `output` selector when the app deliberately returns a raw provider result
@@ -118,5 +119,10 @@ The adapter infers:
   `output` selector
 - replay/cassette metadata for local tools configured with `toolReplay`
 
+Successful custom `run` entrypoints that do not return AI SDK `steps` use
+normalized transcript events for local tool executions. Output-only custom runs
+still get synthesized input/output messages; return a normalized `session` when
+evals need exact transcript control beyond that.
+
 See the workspace demo app in `apps/demo-ai-sdk` and the RFC notes in
 `docs/harness-first-rfc.md`.

package/dist/index.js

@@ -112,14 +112,14 @@ function createFailedAiSdkRun(input, context, error, harnessName, startedAt) {
 async function runAiSdkHarness(options, agent, input, context) {
   const trace = createTraceRecorder(options.name ?? "ai-sdk");
   const replayMetadataByToolCallId = /* @__PURE__ */ new Map();
-  const runtimeToolCalls = [];
+  const runtimeEvents = [];
   const tools = createToolset({
     input,
     context,
     tools: options.tools,
     toolReplay: options.toolReplay,
     replayMetadataByToolCallId,
-    runtimeToolCalls
+    runtimeEvents
   });
   const runtime = {
     tools,
@@ -149,13 +149,20 @@ async function runAiSdkHarness(options, agent, input, context) {
       result
     };
     const output = options.output ? await options.output(resultArgs) : resolveOutput(result);
-    const usage = resolveUsage(result, runtimeToolCalls.length);
+    const explicitSession = getResultSession(result);
+    const useRuntimeEvents = shouldUseRuntimeEvents(result);
+    const usage = resolveUsage(
+      result,
+      useRuntimeEvents ? runtimeEvents : [],
+      explicitSession
+    );
     const session = resolveSession(
       input,
       result,
       output,
       replayMetadataByToolCallId,
-      runtimeToolCalls
+      runtimeEvents,
+      explicitSession
     );
     const errors = (0, import_harness.resolveHarnessRunErrors)(result);
     const finishedAt = /* @__PURE__ */ new Date();
@@ -178,14 +185,11 @@ async function runAiSdkHarness(options, agent, input, context) {
   } catch (error) {
     const finishedAt = /* @__PURE__ */ new Date();
     const serializedError = (0, import_harness.serializeError)(error);
-    const usage = runtimeToolCalls.length > 0 ? { toolCalls: runtimeToolCalls.length } : {};
-    const session = resolveSession(
-      input,
-      void 0,
-      void 0,
-      replayMetadataByToolCallId,
-      runtimeToolCalls
-    );
+    const runtimeToolCallCount = runtimeEvents.filter(
+      (event) => event.type === "tool_call"
+    ).length;
+    const usage = runtimeToolCallCount > 0 ? { toolCalls: runtimeToolCallCount } : {};
+    const session = resolveFailureSession(input, runtimeEvents);
     const run = {
       session,
       output: void 0,
@@ -283,11 +287,6 @@ function finishAiSdkTrace(trace, options) {
     options.result,
     options.usage
   );
-  const toolSpans = (0, import_harness.createToolCallSpans)((0, import_harness.toolCalls)(options.session), {
-    traceId: trace.id,
-    parentId: trace.rootSpanId,
-    spanIdPrefix: `${trace.id}:tool`
-  });
   const finishedAt = options.finishedAt;
   const durationMs = finishedAt.getTime() - trace.startedAt.getTime();
   const rootError = options.errors?.[0] ? (0, import_harness.normalizeSpanError)(options.errors[0]) : void 0;
@@ -307,7 +306,7 @@ function finishAiSdkTrace(trace, options) {
       ...(0, import_harness.createGenAiUsageAttributes)(options.usage)
     })
   };
-  const spans = [rootSpan, ...modelSpans, ...toolSpans];
+  const spans = [rootSpan, ...modelSpans];
   return {
     id: trace.id,
     name: trace.name,
@@ -321,7 +320,7 @@ function finishAiSdkTrace(trace, options) {
   };
 }
 function createAiSdkModelSpans(trace, result, usage) {
-  const steps = resolveSteps(result);
+  const steps = readAiSdkSteps(result);
   if (steps.length === 0) {
     const fallback = createUsageModelSpan(trace, usage);
     return fallback ? [fallback] : [];
@@ -376,7 +375,7 @@ function createToolset({
   tools,
   toolReplay,
   replayMetadataByToolCallId,
-  runtimeToolCalls
+  runtimeEvents
 }) {
   return Object.fromEntries(
     Object.entries(tools ?? {}).map(([toolName, tool]) => {
@@ -395,6 +394,14 @@ function createToolset({
         execute: async (toolInput, execution) => {
           const startedAt = /* @__PURE__ */ new Date();
           const normalizedArgs = normalizeArguments(toolInput);
+          const call = {
+            type: "tool_call",
+            id: execution.toolCallId,
+            name: toolName,
+            ...normalizedArgs ? { arguments: normalizedArgs } : {},
+            startedAt: startedAt.toISOString()
+          };
+          runtimeEvents.push(call);
           const replayContext = {
             input,
             signal: context.signal,
@@ -414,7 +421,6 @@ function createToolset({
               replay: void 0
             };
             const finishedAt = /* @__PURE__ */ new Date();
-            const normalizedResult = (0, import_harness.toJsonValue)(executionResult.result);
             const replayMetadata = (0, import_replay.normalizeReplayMetadata)(
               executionResult.replay
             );
@@ -424,15 +430,20 @@ function createToolset({
                 executionResult.replay
               );
             }
-            runtimeToolCalls.push({
-              id: execution.toolCallId,
+            call.finishedAt = finishedAt.toISOString();
+            call.durationMs = finishedAt.getTime() - startedAt.getTime();
+            if (replayMetadata) {
+              call.metadata = replayMetadata;
+            }
+            const normalizedResult = (0, import_harness.toJsonValue)(executionResult.result);
+            runtimeEvents.push({
+              type: "tool_result",
+              toolCallId: execution.toolCallId,
               name: toolName,
-              ...normalizedArgs ? { arguments: normalizedArgs } : {},
-              ...normalizedResult !== void 0 ? { result: normalizedResult } : {},
-              startedAt: startedAt.toISOString(),
-              finishedAt: finishedAt.toISOString(),
-              durationMs: finishedAt.getTime() - startedAt.getTime(),
-              ...replayMetadata ? { metadata: replayMetadata } : {}
+              ...normalizedResult !== void 0 ? { content: normalizedResult } : {},
+              startedAt: call.startedAt,
+              finishedAt: call.finishedAt,
+              durationMs: call.durationMs
             });
             return executionResult.result;
           } catch (error) {
@@ -442,15 +453,19 @@ function createToolset({
             if (replay2) {
               replayMetadataByToolCallId.set(execution.toolCallId, replay2);
             }
-            runtimeToolCalls.push({
-              id: execution.toolCallId,
+            call.finishedAt = finishedAt.toISOString();
+            call.durationMs = finishedAt.getTime() - startedAt.getTime();
+            if (replayMetadata) {
+              call.metadata = replayMetadata;
+            }
+            runtimeEvents.push({
+              type: "tool_result",
+              toolCallId: execution.toolCallId,
               name: toolName,
-              ...normalizedArgs ? { arguments: normalizedArgs } : {},
               error: normalizeError(error),
-              startedAt: startedAt.toISOString(),
-              finishedAt: finishedAt.toISOString(),
-              durationMs: finishedAt.getTime() - startedAt.getTime(),
-              ...replayMetadata ? { metadata: replayMetadata } : {}
+              startedAt: call.startedAt,
+              finishedAt: call.finishedAt,
+              durationMs: call.durationMs
             });
             throw error;
           }
@@ -539,26 +554,26 @@ function toOutputValue(value) {
   }
   return void 0;
 }
-function resolveUsage(result, runtimeToolCallCount = 0) {
-  const steps = resolveSteps(result);
-  const usage = resolveLanguageModelUsage(result) ?? resolveStepUsage(steps);
+function resolveUsage(result, runtimeEvents = [], explicitSession) {
+  const steps = readAiSdkSteps(result);
+  const runtimeToolCallCount = countRuntimeToolCalls(runtimeEvents);
+  const explicitSessionToolCallCount = explicitSession ? countSessionToolCalls(explicitSession) : void 0;
+  const usage = readAiSdkUsage(result, steps);
   const lastStep = steps.length > 0 ? steps[steps.length - 1] : void 0;
+  const toolCallCount = explicitSessionToolCallCount ?? (steps.length > 0 ? countStepToolCalls(steps) : runtimeToolCallCount);
   if (!usage) {
-    if (steps.length > 0) {
-      const toolCallCount2 = countStepToolCalls(steps);
+    if (toolCallCount > 0 || steps.length > 0) {
       return {
-        provider: lastStep?.model.provider,
-        model: lastStep?.model.modelId,
-        ...toolCallCount2 > 0 ? { toolCalls: toolCallCount2 } : {}
+        provider: lastStep?.model?.provider,
+        model: lastStep?.model?.modelId,
+        ...toolCallCount > 0 ? { toolCalls: toolCallCount } : {}
       };
     }
-    return runtimeToolCallCount > 0 ? { toolCalls: runtimeToolCallCount } : {};
+    return {};
   }
-  const stepToolCallCount = countStepToolCalls(steps);
-  const toolCallCount = stepToolCallCount > 0 ? stepToolCallCount : runtimeToolCallCount;
   return {
-    provider: lastStep?.model.provider,
-    model: lastStep?.model.modelId,
+    provider: lastStep?.model?.provider,
+    model: lastStep?.model?.modelId,
     inputTokens: usage.inputTokens,
     outputTokens: usage.outputTokens,
     reasoningTokens: usage.outputTokenDetails?.reasoningTokens ?? usage.reasoningTokens,
@@ -571,8 +586,11 @@ function resolveUsage(result, runtimeToolCallCount = 0) {
     })
   };
 }
+function countRuntimeToolCalls(events) {
+  return events.filter((event) => event.type === "tool_call").length;
+}
 function resolveStepUsage(steps) {
-  const usages = steps.map((step) => step.usage).filter((usage) => Boolean(usage));
+  const usages = steps.map((step) => step.usage).filter(isLanguageModelUsage);
   if (usages.length === 0) {
     return void 0;
   }
@@ -626,86 +644,125 @@ function countStepToolCalls(steps) {
     0
   );
 }
-function resolveSession(input, result, output, replayMetadataByToolCallId, runtimeToolCalls = []) {
+function countSessionToolCalls(session) {
+  return session.events.filter((event) => event.type === "tool_call").length;
+}
+function shouldUseRuntimeEvents(result) {
   if ((0, import_harness.isNormalizedSession)(
     result?.session
   )) {
-    return result.session;
+    return false;
   }
-  if ((0, import_harness.isNormalizedSession)(result?.trace)) {
-    return result.trace;
+  return readAiSdkSteps(result).length === 0;
+}
+function resolveSession(input, result, output, replayMetadataByToolCallId, runtimeEvents = [], explicitSession = getResultSession(result)) {
+  if (explicitSession) {
+    return explicitSession;
   }
-  const steps = resolveSteps(result);
-  const messages = [
+  const steps = readAiSdkSteps(result);
+  const events = [
     {
+      type: "message",
       role: "user",
       content: (0, import_harness.normalizeContent)(input)
     }
   ];
-  const stepToolCallIds = /* @__PURE__ */ new Set();
   for (const step of steps) {
-    for (const toolCall of step.toolCalls ?? []) {
-      stepToolCallIds.add(toolCall.toolCallId);
-    }
-    messages.push(...normalizeStep(step, replayMetadataByToolCallId));
+    events.push(...normalizeStep(step, replayMetadataByToolCallId));
   }
-  const unmatchedRuntimeToolCalls = runtimeToolCalls.filter(
-    (call) => call.id === void 0 || !stepToolCallIds.has(call.id)
-  );
-  if (unmatchedRuntimeToolCalls.length > 0) {
-    messages.push(...normalizeRuntimeToolCalls(unmatchedRuntimeToolCalls));
+  if (steps.length === 0) {
+    events.push(...runtimeEvents);
   }
-  if (output !== void 0 && !messages.some(
-    (message) => message.role === "assistant" && message.content !== void 0
+  if (output !== void 0 && !events.some(
+    (event) => event.type === "message" && event.role === "assistant" && event.content !== void 0
   )) {
-    messages.push({
+    events.push({
+      type: "message",
       role: "assistant",
       content: output
     });
   }
   const lastStep = steps.length > 0 ? steps[steps.length - 1] : void 0;
   return {
-    messages,
-    provider: lastStep?.model.provider,
-    model: lastStep?.model.modelId
+    events,
+    provider: lastStep?.model?.provider,
+    model: lastStep?.model?.modelId
   };
 }
-function normalizeRuntimeToolCalls(runtimeToolCalls) {
-  const messages = [
-    {
-      role: "assistant",
-      toolCalls: runtimeToolCalls
-    }
-  ];
-  for (const call of runtimeToolCalls) {
-    if (call.result === void 0 && !call.error) {
-      continue;
-    }
-    const content = call.result !== void 0 ? call.result : call.error && call.error.message.length > 0 ? call.error.message : void 0;
-    messages.push({
-      role: "tool",
-      ...content !== void 0 ? { content } : {},
-      metadata: (0, import_harness.normalizeMetadata)({
-        name: call.name,
-        toolCallId: call.id,
-        isError: Boolean(call.error)
-      })
-    });
-  }
-  return messages;
+function getResultSession(result) {
+  const session = result?.session;
+  return (0, import_harness.isNormalizedSession)(session) ? session : void 0;
 }
-function resolveSteps(result) {
+function resolveFailureSession(input, runtimeEvents) {
+  return {
+    events: [
+      {
+        type: "message",
+        role: "user",
+        content: (0, import_harness.normalizeContent)(input)
+      },
+      ...runtimeEvents
+    ]
+  };
+}
+function readAiSdkSteps(result) {
   if (!result || typeof result !== "object") {
     return [];
   }
-  return Array.isArray(result.steps) ? result.steps ?? [] : [];
+  if (!Object.prototype.hasOwnProperty.call(result, "steps")) {
+    return [];
+  }
+  const steps = result.steps;
+  if (!Array.isArray(steps)) {
+    return [];
+  }
+  if (steps.length > 0 && !steps.every(isAiSdkStepLike)) {
+    return [];
+  }
+  return steps;
+}
+function isAiSdkStepLike(step) {
+  if (!step || typeof step !== "object") {
+    return false;
+  }
+  const record = step;
+  return Array.isArray(record.content) || Array.isArray(record.toolCalls) || Array.isArray(record.toolResults) || Boolean(record.response && typeof record.response === "object") || Boolean(record.usage && typeof record.usage === "object") || typeof record.finishReason === "string" || typeof record.stepNumber === "number" || typeof record.text === "string";
+}
+function readAiSdkUsage(result, steps) {
+  const totalUsage = readUsageField(result, "totalUsage");
+  if (totalUsage) {
+    return totalUsage;
+  }
+  if (steps.length > 0) {
+    return resolveStepUsage(steps);
+  }
+  return readUsageField(result, "usage");
 }
-function resolveLanguageModelUsage(result) {
+function readUsageField(result, field) {
   if (!result || typeof result !== "object") {
     return void 0;
   }
-  const aiResult = result;
-  return aiResult.totalUsage ?? aiResult.usage;
+  const usage = result[field];
+  return isLanguageModelUsage(usage) ? usage : void 0;
+}
+function isLanguageModelUsage(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  const usage = value;
+  return isOptionalFiniteNumber(usage.inputTokens) && isOptionalFiniteNumber(usage.outputTokens) && isOptionalFiniteNumber(usage.reasoningTokens) && isOptionalFiniteNumber(usage.totalTokens) && isOptionalFiniteNumber(usage.cachedInputTokens) && isUsageDetailObject(usage.inputTokenDetails) && isUsageDetailObject(usage.outputTokenDetails);
+}
+function isUsageDetailObject(value) {
+  if (value === void 0) {
+    return true;
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  return Object.values(value).every(isOptionalFiniteNumber);
+}
+function isOptionalFiniteNumber(value) {
+  return value === void 0 || typeof value === "number" && Number.isFinite(value);
 }
 function normalizeStep(step, replayMetadataByToolCallId) {
   const toolResultsById = new Map(
@@ -714,12 +771,15 @@ function normalizeStep(step, replayMetadataByToolCallId) {
       toolResult
     ])
   );
+  const toolErrorsById = new Map(
+    (step.toolCalls ?? []).filter((toolCall) => toolCall.invalid || toolCall.error !== void 0).map((toolCall) => [
+      toolCall.toolCallId,
+      normalizeError(toolCall.error ?? toolCall.invalid)
+    ])
+  );
   const normalizedCalls = (step.toolCalls ?? []).map(
     (toolCall) => normalizeToolCall(toolCall, toolResultsById, replayMetadataByToolCallId)
   );
-  const normalizedCallsById = new Map(
-    normalizedCalls.map((toolCall) => [toolCall.id, toolCall])
-  );
   const assistantMetadata = (0, import_harness.normalizeMetadata)({
     stepNumber: step.stepNumber,
     finishReason: step.finishReason,
@@ -727,24 +787,25 @@ function normalizeStep(step, replayMetadataByToolCallId) {
     reasoningText: step.reasoningText,
     response: step.response
   });
-  const messages = [];
-  if (step.text || normalizedCalls.length > 0 || assistantMetadata) {
-    messages.push({
+  const events = [];
+  if (step.text || assistantMetadata) {
+    events.push({
+      type: "message",
       role: "assistant",
       ...step.text ? { content: step.text } : {},
-      ...normalizedCalls.length > 0 ? { toolCalls: normalizedCalls } : {},
       ...assistantMetadata ? { metadata: assistantMetadata } : {}
     });
   }
+  events.push(...normalizedCalls);
   for (const toolResult of step.toolResults ?? []) {
     const content = toolResult.output === void 0 ? void 0 : (0, import_harness.normalizeContent)(toolResult.output);
-    messages.push({
-      role: "tool",
+    events.push({
+      type: "tool_result",
+      toolCallId: toolResult.toolCallId,
+      name: toolResult.toolName,
       ...content !== void 0 ? { content } : {},
+      ...toolErrorsById.has(toolResult.toolCallId) ? { error: toolErrorsById.get(toolResult.toolCallId) } : {},
       metadata: (0, import_harness.normalizeMetadata)({
-        name: toolResult.toolName,
-        toolCallId: toolResult.toolCallId,
-        isError: Boolean(normalizedCallsById.get(toolResult.toolCallId)?.error),
         preliminary: toolResult.preliminary,
         providerExecuted: toolResult.providerExecuted,
         title: toolResult.title,
@@ -752,27 +813,26 @@ function normalizeStep(step, replayMetadataByToolCallId) {
       })
     });
   }
-  return messages;
+  return events;
 }
 function normalizeToolCall(toolCall, toolResultsById, replayMetadataByToolCallId) {
   const toolResult = toolResultsById.get(toolCall.toolCallId);
   const normalizedArguments = normalizeArguments(toolCall.input);
-  const normalizedResult = toolResult !== void 0 ? (0, import_harness.toJsonValue)(toolResult.output) : void 0;
   const errorValue = toolCall.invalid || toolCall.error !== void 0 ? normalizeError(toolCall.error ?? toolCall.invalid) : void 0;
   const replayMetadata = (0, import_replay.normalizeReplayMetadata)(
     replayMetadataByToolCallId.get(toolCall.toolCallId)
   );
   return {
+    type: "tool_call",
     id: toolCall.toolCallId,
     name: toolCall.toolName,
     ...normalizedArguments ? { arguments: normalizedArguments } : {},
-    ...toolResult && normalizedResult !== void 0 ? { result: normalizedResult } : {},
-    ...errorValue ? { error: errorValue } : {},
     metadata: (0, import_harness.normalizeMetadata)({
       providerExecuted: toolCall.providerExecuted ?? toolResult?.providerExecuted,
       title: toolCall.title ?? toolResult?.title,
       dynamic: toolCall.dynamic,
       invalid: toolCall.invalid,
+      ...errorValue ? { error: errorValue } : {},
       preliminary: toolResult?.preliminary,
       providerMetadata: toolCall.providerMetadata ?? toolResult?.providerMetadata,
       ...replayMetadata ?? {}