@vitest-evals/harness-pi-ai 0.13.0 → 0.14.0

package/README.md

@@ -33,7 +33,7 @@ describeEval("refund agent", { harness }, (it) => {
     expect(result.output).toMatchObject({
       status: "approved",
     });
-    expect(toolCalls(result.session).map((call) => call.name)).toEqual([
+    expect(toolCalls(result).map((call) => call.name)).toEqual([
       "lookupInvoice",
       "createRefund",
     ]);
@@ -164,8 +164,8 @@ const harness = piAiHarness({
 When an agent exposes both a native Pi tool and a runtime tool with the same
 name, a native tool call records in its own cassette namespace. Runtime calls of
 that same name are treated as implementation details while the native tool is
-executing, so delegated runtime calls do not create duplicate trace entries or
-overwrite the native recording.
+executing, so delegated runtime calls do not create duplicate transcript entries
+or overwrite the native recording.
 
 Supported modes:
 

package/dist/index.d.mts

@@ -1,5 +1,5 @@
 import * as vitest_evals_judges from 'vitest-evals/judges';
-import { NormalizedMessage, JsonValue, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
+import { TranscriptMessageEvent, JsonValue, NormalizedError, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
 import { Api, Model, SimpleStreamOptions } from '@mariozechner/pi-ai';
 
@@ -63,13 +63,18 @@ type PiAiAgentResult<TAgent, TInput, TTools extends PiAiToolset<TInput>> = TAgen
 } ? Awaited<TResult> : unknown;
 /** Replay mode alias used by the Pi AI harness package. */
 type PiAiReplayMode = ReplayMode;
-/** Event sink for recording Pi AI transcript messages. */
+/** Event sink for recording Pi AI transcript events. */
 interface PiAiEventSink {
-    message: (message: NormalizedMessage) => void;
+    message: (event: TranscriptMessageEvent) => void;
     system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
     user: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
     assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
-    tool: (name: string, content: JsonValue, metadata?: Record<string, JsonValue>) => void;
+    /** Records a tool-result event linked to an assistant tool call by id. */
+    tool: (toolCallId: string, content: JsonValue, options?: {
+        name?: string;
+        error?: NormalizedError;
+        metadata?: Record<string, JsonValue>;
+    }) => void;
 }
 /** Context passed to instrumented Pi AI tool executions. */
 interface PiAiToolContext<TInput = string> {

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import * as vitest_evals_judges from 'vitest-evals/judges';
-import { NormalizedMessage, JsonValue, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
+import { TranscriptMessageEvent, JsonValue, NormalizedError, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
 import { Api, Model, SimpleStreamOptions } from '@mariozechner/pi-ai';
 
@@ -63,13 +63,18 @@ type PiAiAgentResult<TAgent, TInput, TTools extends PiAiToolset<TInput>> = TAgen
 } ? Awaited<TResult> : unknown;
 /** Replay mode alias used by the Pi AI harness package. */
 type PiAiReplayMode = ReplayMode;
-/** Event sink for recording Pi AI transcript messages. */
+/** Event sink for recording Pi AI transcript events. */
 interface PiAiEventSink {
-    message: (message: NormalizedMessage) => void;
+    message: (event: TranscriptMessageEvent) => void;
     system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
     user: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
     assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
-    tool: (name: string, content: JsonValue, metadata?: Record<string, JsonValue>) => void;
+    /** Records a tool-result event linked to an assistant tool call by id. */
+    tool: (toolCallId: string, content: JsonValue, options?: {
+        name?: string;
+        error?: NormalizedError;
+        metadata?: Record<string, JsonValue>;
+    }) => void;
 }
 /** Context passed to instrumented Pi AI tool executions. */
 interface PiAiToolContext<TInput = string> {

package/dist/index.js

@@ -88,8 +88,9 @@ function piAiHarness(options) {
           input,
           context
         });
-        const messages = [
+        const events = [
           {
+            type: "message",
             role: "user",
             content: (0, import_harness.normalizeContent)(input)
           }
@@ -101,7 +102,7 @@ function piAiHarness(options) {
             agent,
             input,
             context,
-            messages,
+            events,
             options.tools,
             inferredTools.nativeToolsets
           );
@@ -111,7 +112,7 @@ function piAiHarness(options) {
           agent,
           input,
           context,
-          messages,
+          events,
           inferredTools.runtimeTools,
           inferredTools.nativeToolsets
         );
@@ -148,7 +149,7 @@ function validateOptions(options) {
     );
   }
 }
-async function executePiHarnessRun(options, agent, input, context, messages, runtimeTools, nativeToolsets) {
+async function executePiHarnessRun(options, agent, input, context, events, runtimeTools, nativeToolsets) {
   const trace = createTraceRecorder(options.name ?? "pi-ai");
   const executionState = createPiToolExecutionState();
   const runtime = createRuntime({
@@ -157,7 +158,7 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
     tools: runtimeTools,
     toolReplay: options.toolReplay,
     executionState,
-    messages
+    events
   });
   try {
     const result = await withInstrumentedAgentTools(
@@ -166,8 +167,7 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
       {
         input,
         context,
-        messages,
-        toolCalls: runtime.toolCalls,
+        events,
         toolReplay: options.toolReplay,
         executionState
       },
@@ -194,8 +194,8 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
       result: normalizeResult
     };
     const output = options.output ? await options.output(resultArgs) : resolveOutput(normalizeResult);
-    const usage = resolveUsage(normalizeResult, runtime.toolCalls.length);
-    const session = resolveSession(normalizeResult, messages, output, usage);
+    const usage = resolveUsage(normalizeResult, countToolCallEvents(events));
+    const session = resolveSession(normalizeResult, events, output, usage);
     const errors = resolveErrors(normalizeResult);
     const finishedAt = /* @__PURE__ */ new Date();
     return {
@@ -214,8 +214,8 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
       ]
     };
   } catch (error) {
-    const usage = resolveUsage(void 0, runtime.toolCalls.length);
-    const session = resolveSession(void 0, messages, void 0, usage);
+    const usage = resolveUsage(void 0, countToolCallEvents(events));
+    const session = resolveSession(void 0, events, void 0, usage);
     const finishedAt = /* @__PURE__ */ new Date();
     const serializedError = (0, import_harness.serializeError)(error);
     const run = {
@@ -290,15 +290,7 @@ function finishPiAiTrace(trace, options) {
     })
   };
   const modelSpan = createUsageModelSpan(trace, options.usage);
-  const spans = [
-    rootSpan,
-    ...modelSpan ? [modelSpan] : [],
-    ...(0, import_harness.createToolCallSpans)((0, import_harness.toolCalls)(options.session), {
-      traceId: trace.id,
-      parentId: trace.rootSpanId,
-      spanIdPrefix: `${trace.id}:tool`
-    })
-  ];
+  const spans = [rootSpan, ...modelSpan ? [modelSpan] : []];
   return {
     id: trace.id,
     name: trace.name,
@@ -447,6 +439,15 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
         args.executionState,
         tool.name
       );
+      const call = {
+        type: "tool_call",
+        id: toolCallId,
+        name: tool.name,
+        arguments: rawArgs,
+        startedAt: startedAt.toISOString(),
+        metadata: (0, import_replay.normalizeReplayMetadata)(void 0)
+      };
+      args.events.push(call);
       try {
         const execution = await executeNativeToolWithReplay({
           toolName: tool.name,
@@ -457,45 +458,34 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
           context: toolContext
         });
         const finishedAt = /* @__PURE__ */ new Date();
-        const call = {
-          id: toolCallId,
+        call.finishedAt = finishedAt.toISOString();
+        call.durationMs = finishedAt.getTime() - startedAt.getTime();
+        call.metadata = (0, import_replay.normalizeReplayMetadata)(execution.replay);
+        args.events.push({
+          type: "tool_result",
+          toolCallId,
           name: tool.name,
-          arguments: rawArgs,
-          result: execution.normalizedResult,
-          startedAt: startedAt.toISOString(),
-          finishedAt: finishedAt.toISOString(),
-          durationMs: finishedAt.getTime() - startedAt.getTime(),
-          metadata: (0, import_replay.normalizeReplayMetadata)(execution.replay)
-        };
-        args.toolCalls.push(call);
-        args.messages.push({
-          role: "assistant",
-          toolCalls: [call]
-        });
-        args.messages.push({
-          role: "tool",
           content: execution.normalizedResult,
-          metadata: {
-            name: tool.name
-          }
+          startedAt: call.startedAt,
+          finishedAt: call.finishedAt,
+          durationMs: call.durationMs
         });
         return execution.result;
       } catch (error) {
         const finishedAt = /* @__PURE__ */ new Date();
-        const call = {
-          id: toolCallId,
+        call.finishedAt = finishedAt.toISOString();
+        call.durationMs = finishedAt.getTime() - startedAt.getTime();
+        call.metadata = (0, import_replay.normalizeReplayMetadata)(
+          (0, import_replay.getReplayMetadataFromError)(error)
+        );
+        args.events.push({
+          type: "tool_result",
+          toolCallId,
           name: tool.name,
-          arguments: rawArgs,
           error: serializeToolCallError(error),
-          startedAt: startedAt.toISOString(),
-          finishedAt: finishedAt.toISOString(),
-          durationMs: finishedAt.getTime() - startedAt.getTime(),
-          metadata: (0, import_replay.normalizeReplayMetadata)((0, import_replay.getReplayMetadataFromError)(error))
-        };
-        args.toolCalls.push(call);
-        args.messages.push({
-          role: "assistant",
-          toolCalls: [call]
+          startedAt: call.startedAt,
+          finishedAt: call.finishedAt,
+          durationMs: call.durationMs
         });
         throw error;
       } finally {
@@ -570,6 +560,9 @@ function serializeToolCallError(error) {
     ...typeof type === "string" ? { type } : {}
   };
 }
+function createRuntimeToolCallId(toolName, index) {
+  return `runtime:${toolName}:${index + 1}`;
+}
 function getNativeToolExecuteOrigin(execute) {
   const nativeExecute = execute;
   return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;
@@ -638,42 +631,44 @@ function createRuntime({
   tools,
   toolReplay,
   executionState,
-  messages
+  events
 }) {
-  const toolCalls = [];
   const eventSink = {
-    message: (message) => {
-      messages.push(message);
+    message: (event) => {
+      events.push(event);
     },
     system: (content, metadata) => {
-      messages.push({
+      events.push({
+        type: "message",
         role: "system",
         content,
         metadata
       });
     },
     user: (content, metadata) => {
-      messages.push({
+      events.push({
+        type: "message",
         role: "user",
         content,
         metadata
       });
     },
     assistant: (content, metadata) => {
-      messages.push({
+      events.push({
+        type: "message",
         role: "assistant",
         content,
         metadata
       });
     },
-    tool: (name, content, metadata) => {
-      messages.push({
-        role: "tool",
+    tool: (toolCallId, content, options) => {
+      events.push({
+        type: "tool_result",
+        toolCallId,
+        ...options?.name ? { name: options.name } : {},
         content,
-        metadata: {
-          name,
-          ...metadata ?? {}
-        }
+        ...options?.error ? { error: options.error } : {},
+        ...options?.metadata ? { metadata: options.metadata } : {}
       });
     }
   };
@@ -691,7 +686,21 @@ function createRuntime({
           signal: context.signal,
           setArtifact: context.setArtifact
         };
+        const toolCallId = createRuntimeToolCallId(
+          toolName,
+          countToolCallEvents(events)
+        );
+        const call = {
+          type: "tool_call",
+          id: toolCallId,
+          name: toolName,
+          arguments: args,
+          startedAt: startedAt.toISOString()
+        };
         try {
+          if (!isNativeImplementationCall) {
+            events.push(call);
+          }
           const execution = await executeToolWithReplay({
             toolName,
             tool,
@@ -703,26 +712,17 @@ function createRuntime({
           if (isNativeImplementationCall) {
             return execution.result;
           }
-          const call = {
+          call.finishedAt = finishedAt.toISOString();
+          call.durationMs = finishedAt.getTime() - startedAt.getTime();
+          call.metadata = (0, import_replay.normalizeReplayMetadata)(execution.replay);
+          events.push({
+            type: "tool_result",
+            toolCallId,
             name: toolName,
-            arguments: args,
-            result: execution.result,
-            startedAt: startedAt.toISOString(),
-            finishedAt: finishedAt.toISOString(),
-            durationMs: finishedAt.getTime() - startedAt.getTime(),
-            metadata: (0, import_replay.normalizeReplayMetadata)(execution.replay)
-          };
-          toolCalls.push(call);
-          messages.push({
-            role: "assistant",
-            toolCalls: [call]
-          });
-          messages.push({
-            role: "tool",
             content: execution.result,
-            metadata: {
-              name: toolName
-            }
+            startedAt: call.startedAt,
+            finishedAt: call.finishedAt,
+            durationMs: call.durationMs
           });
           return execution.result;
         } catch (error) {
@@ -730,21 +730,19 @@ function createRuntime({
           if (isNativeImplementationCall) {
             throw error;
           }
-          const call = {
+          call.finishedAt = finishedAt.toISOString();
+          call.durationMs = finishedAt.getTime() - startedAt.getTime();
+          call.metadata = (0, import_replay.normalizeReplayMetadata)(
+            (0, import_replay.getReplayMetadataFromError)(error)
+          );
+          events.push({
+            type: "tool_result",
+            toolCallId,
             name: toolName,
-            arguments: args,
             error: serializeToolCallError(error),
-            startedAt: startedAt.toISOString(),
-            finishedAt: finishedAt.toISOString(),
-            durationMs: finishedAt.getTime() - startedAt.getTime(),
-            metadata: (0, import_replay.normalizeReplayMetadata)(
-              (0, import_replay.getReplayMetadataFromError)(error)
-            )
-          };
-          toolCalls.push(call);
-          messages.push({
-            role: "assistant",
-            toolCalls: [call]
+            startedAt: call.startedAt,
+            finishedAt: call.finishedAt,
+            durationMs: call.durationMs
           });
           throw error;
         }
@@ -754,8 +752,7 @@ function createRuntime({
   return {
     tools: runtimeTools,
     events: eventSink,
-    signal: context.signal,
-    toolCalls
+    signal: context.signal
   };
 }
 function resolveOutput(result) {
@@ -874,6 +871,9 @@ function isLegacyNativeToolReplayMarker(value) {
     value && typeof value === "object" && "kind" in value && value.kind === "pi-ai-native-tool-result" && "version" in value && value.version === 1
   );
 }
+function countToolCallEvents(events) {
+  return events.filter((event) => event.type === "tool_call").length;
+}
 function resolveUsage(result, toolCallCount) {
   if (!result || typeof result !== "object") {
     return toolCallCount > 0 ? { toolCalls: toolCallCount } : {};
@@ -929,26 +929,24 @@ function stringField(value) {
 function numberField(value) {
   return typeof value === "number" && Number.isFinite(value) ? value : void 0;
 }
-function resolveSession(result, messages, output, usage) {
+function resolveSession(result, events, output, usage) {
   if ((0, import_harness.isNormalizedSession)(
     result?.session
   )) {
     return result.session;
   }
-  if ((0, import_harness.isNormalizedSession)(result?.trace)) {
-    return result.trace;
-  }
-  const sessionMessages = [...messages];
-  if (output !== void 0 && !sessionMessages.some(
-    (message) => message.role === "assistant" && message.content !== void 0
+  const sessionEvents = [...events];
+  if (output !== void 0 && !sessionEvents.some(
+    (event) => event.type === "message" && event.role === "assistant" && event.content !== void 0
   )) {
-    sessionMessages.push({
+    sessionEvents.push({
+      type: "message",
       role: "assistant",
       content: output
     });
   }
   return {
-    messages: sessionMessages,
+    events: sessionEvents,
     provider: result?.provider ?? usage.provider,
     model: result?.model ?? usage.model
   };