@vitest-evals/harness-pi-ai 0.9.0-beta.1 → 0.9.0-beta.3

package/README.md

@@ -11,12 +11,31 @@ npm install -D vitest-evals @vitest-evals/harness-pi-ai
 ## Usage
 
 ```ts
+import { expect } from "vitest";
 import { piAiHarness } from "@vitest-evals/harness-pi-ai";
+import { describeEval, toolCalls } from "vitest-evals";
 
 const harness = piAiHarness({
   createAgent: () => createRefundAgent(),
+  toolReplay: {
+    lookupInvoice: true,
+  },
   prompt: sharedJudgePrompt,
 });
+
+describeEval("refund agent", { harness }, (it) => {
+  it("approves a refundable invoice", async ({ run }) => {
+    const result = await run("Refund invoice inv_123");
+
+    expect(result.output).toMatchObject({
+      status: "approved",
+    });
+    expect(toolCalls(result.session).map((call) => call.name)).toEqual([
+      "lookupInvoice",
+      "createRefund",
+    ]);
+  });
+});
 ```
 
 `prompt` gives rubric or factuality judges the same provider/model setup
@@ -38,6 +57,26 @@ const harness = piAiHarness({
 If the agent already implements `run(input, runtime)`, you can omit `run` and
 the harness will call that method automatically.
 
+`createAgent` receives the per-run input and harness context before the adapter
+infers and instruments native agent tools. Use that for scenario-specific
+instructions, tool closures, metadata, or seeded artifacts without leaving the
+default replay path:
+
+```ts
+const harness = piAiHarness({
+  createAgent: ({ input, context }) =>
+    createRefundAgent({
+      instructions: buildInstructions(input),
+      metadata: context.metadata,
+      setArtifact: context.setArtifact,
+    }),
+  toolReplay: {
+    lookupInvoice: true,
+  },
+  prompt: sharedJudgePrompt,
+});
+```
+
 You should not need to configure output/session/usage basics for the normal Pi
 path. Pass your agent and let the adapter infer both the toolset and the
 normalized result from common Pi-style return values such as `decision` or
@@ -74,7 +113,7 @@ The adapter provides:
 - a required prompt seam for LLM-backed judges
 - normalized session capture from emitted events and wrapped tool calls
 - usage/output inference for common `pi-ai`-style result objects
-- opt-in tool replay/recording when the tool definition sets `replay: true`
+- opt-in tool replay/recording from harness-level `toolReplay`
 
 See the workspace demo in `apps/demo-pi`.
 
@@ -95,17 +134,24 @@ export default defineConfig({
 });
 ```
 
-Then opt individual tools into recording/replay:
+Then opt individual tools into recording/replay from the harness:
 
 ```ts
-const tools = {
-  lookupInvoice: {
-    replay: true,
-    execute: async ({ invoiceId }) => fetchInvoice(invoiceId),
+const harness = piAiHarness({
+  createAgent: () => createRefundAgent(),
+  toolReplay: {
+    lookupInvoice: true,
   },
-};
+  prompt: sharedJudgePrompt,
+});
 ```
 
+When an agent exposes both a native Pi tool and a runtime tool with the same
+name, a native tool call records in its own cassette namespace. Runtime calls of
+that same name are treated as implementation details while the native tool is
+executing, so delegated runtime calls do not create duplicate trace entries or
+overwrite the native recording.
+
 Supported modes:
 
 - `off`: never read or write recordings

package/dist/index.d.mts

@@ -20,9 +20,10 @@ interface PiAiToolContext<TInput = string, TMetadata extends HarnessMetadata = H
 }
 type PiAiToolRecording<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
 type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;
+type PiAiToolReplayPolicy<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = boolean | PiAiToolReplayConfig<Record<string, JsonValue>, JsonValue, TInput, TMetadata>;
+type PiAiToolReplayPolicies<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolReplayPolicy<TInput, TMetadata>>;
 interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
     description?: string;
-    replay?: boolean | PiAiToolReplayConfig<TArgs, TResult, TInput, TMetadata>;
     execute: (args: TArgs, context: PiAiToolContext<TInput, TMetadata>) => MaybePromise<TResult>;
 }
 type PiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = AnyPiAiToolset<TInput, TMetadata>;
@@ -44,9 +45,14 @@ interface PiAiHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata,
 interface PiAiHarnessResultArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends PiAiToolset<TInput, TMetadata>> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {
     result: TResult;
 }
+interface PiAiCreateAgentArgs<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
+    input: TInput;
+    context: HarnessContext<TMetadata>;
+}
 interface PiAiHarnessBaseOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> {
     agent?: TAgent;
-    createAgent?: () => MaybePromise<TAgent>;
+    createAgent?: (args: PiAiCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>;
+    toolReplay?: PiAiToolReplayPolicies<TInput, TMetadata>;
     normalize?: PiAiHarnessNormalizeOptions<TAgent, TInput, TMetadata, TResult, TTools>;
     prompt: HarnessPrompt;
     name?: string;
@@ -71,4 +77,4 @@ interface PiAiHarnessNormalizeOptions<TAgent, TInput = string, TMetadata extends
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, TMetadata>;
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>): Harness<TInput, TMetadata>;
 
-export { type PiAiEventSink, type PiAiHarnessInferredToolsOptions, type PiAiHarnessNormalizeOptions, type PiAiHarnessOptions, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiHarnessWithToolsOptions, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolset, piAiHarness };
+export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessInferredToolsOptions, type PiAiHarnessNormalizeOptions, type PiAiHarnessOptions, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiHarnessWithToolsOptions, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };

package/dist/index.d.ts

@@ -20,9 +20,10 @@ interface PiAiToolContext<TInput = string, TMetadata extends HarnessMetadata = H
 }
 type PiAiToolRecording<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
 type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;
+type PiAiToolReplayPolicy<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = boolean | PiAiToolReplayConfig<Record<string, JsonValue>, JsonValue, TInput, TMetadata>;
+type PiAiToolReplayPolicies<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolReplayPolicy<TInput, TMetadata>>;
 interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
     description?: string;
-    replay?: boolean | PiAiToolReplayConfig<TArgs, TResult, TInput, TMetadata>;
     execute: (args: TArgs, context: PiAiToolContext<TInput, TMetadata>) => MaybePromise<TResult>;
 }
 type PiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = AnyPiAiToolset<TInput, TMetadata>;
@@ -44,9 +45,14 @@ interface PiAiHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata,
 interface PiAiHarnessResultArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends PiAiToolset<TInput, TMetadata>> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {
     result: TResult;
 }
+interface PiAiCreateAgentArgs<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
+    input: TInput;
+    context: HarnessContext<TMetadata>;
+}
 interface PiAiHarnessBaseOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> {
     agent?: TAgent;
-    createAgent?: () => MaybePromise<TAgent>;
+    createAgent?: (args: PiAiCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>;
+    toolReplay?: PiAiToolReplayPolicies<TInput, TMetadata>;
     normalize?: PiAiHarnessNormalizeOptions<TAgent, TInput, TMetadata, TResult, TTools>;
     prompt: HarnessPrompt;
     name?: string;
@@ -71,4 +77,4 @@ interface PiAiHarnessNormalizeOptions<TAgent, TInput = string, TMetadata extends
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, TMetadata>;
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>): Harness<TInput, TMetadata>;
 
-export { type PiAiEventSink, type PiAiHarnessInferredToolsOptions, type PiAiHarnessNormalizeOptions, type PiAiHarnessOptions, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiHarnessWithToolsOptions, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolset, piAiHarness };
+export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessInferredToolsOptions, type PiAiHarnessNormalizeOptions, type PiAiHarnessOptions, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiHarnessWithToolsOptions, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };

package/dist/index.js

@@ -31,7 +31,10 @@ function piAiHarness(options) {
     name: options.name ?? "pi-ai",
     prompt: options.prompt,
     run: async (input, context) => {
-      const agent = await resolveAgent(options);
+      const agent = await resolveAgent(options, {
+        input,
+        context
+      });
       const messages = [
         {
           role: "user",
@@ -65,10 +68,13 @@ function piAiHarness(options) {
   };
 }
 async function executePiHarnessRun(options, agent, input, context, messages, runtimeTools, nativeToolsets) {
+  const executionState = createPiToolExecutionState();
   const runtime = createRuntime({
     input,
     context,
     tools: runtimeTools,
+    toolReplay: options.toolReplay,
+    executionState,
     messages
   });
   try {
@@ -79,7 +85,9 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
         input,
         context,
         messages,
-        toolCalls: runtime.toolCalls
+        toolCalls: runtime.toolCalls,
+        toolReplay: options.toolReplay,
+        executionState
       },
       () => runAgent(options, {
         agent,
@@ -125,12 +133,12 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
     throw (0, import_harness.attachHarnessRunToError)(error, run);
   }
 }
-async function resolveAgent(options) {
+async function resolveAgent(options, args) {
   if (options.agent !== void 0) {
     return options.agent;
   }
   if (options.createAgent) {
-    return options.createAgent();
+    return options.createAgent(args);
   }
   throw new Error(
     "piAiHarness requires either an agent instance or a createAgent() function."
@@ -257,12 +265,16 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
         signal: args.context.signal,
         setArtifact: args.context.setArtifact
       };
+      const leaveNativeTool = enterNativeToolExecution(
+        args.executionState,
+        tool.name
+      );
       try {
         const execution = await executeNativeToolWithReplay({
           toolName: tool.name,
           toolCallId,
           execute: originalExecute,
-          replay: tool.replay,
+          replay: args.toolReplay?.[tool.name],
           args: rawArgs,
           context: toolContext
         });
@@ -306,6 +318,8 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
           toolCalls: [call]
         });
         throw error;
+      } finally {
+        leaveNativeTool();
       }
     };
     instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute;
@@ -382,6 +396,28 @@ function getNativeToolExecuteOrigin(execute) {
   const nativeExecute = execute;
   return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;
 }
+function createPiToolExecutionState() {
+  return {
+    activeNativeToolNames: /* @__PURE__ */ new Map()
+  };
+}
+function enterNativeToolExecution(state, toolName) {
+  state.activeNativeToolNames.set(
+    toolName,
+    (state.activeNativeToolNames.get(toolName) ?? 0) + 1
+  );
+  return () => {
+    const nextCount = (state.activeNativeToolNames.get(toolName) ?? 1) - 1;
+    if (nextCount <= 0) {
+      state.activeNativeToolNames.delete(toolName);
+      return;
+    }
+    state.activeNativeToolNames.set(toolName, nextCount);
+  };
+}
+function hasActiveNativeToolExecution(state, toolName) {
+  return (state.activeNativeToolNames.get(toolName) ?? 0) > 0;
+}
 async function executeNativeToolWithReplay({
   toolName,
   toolCallId,
@@ -393,7 +429,7 @@ async function executeNativeToolWithReplay({
   let didExecute = false;
   let liveResult;
   const execution = await (0, import_replay.executeWithReplay)({
-    toolName,
+    toolName: createNativeReplayToolName(toolName),
     args,
     context,
     execute: async (toolArgs) => {
@@ -415,10 +451,15 @@ async function executeNativeToolWithReplay({
     replay: execution.replay
   };
 }
+function createNativeReplayToolName(toolName) {
+  return `${toolName}.native`;
+}
 function createRuntime({
   input,
   context,
   tools,
+  toolReplay,
+  executionState,
   messages
 }) {
   const toolCalls = [];
@@ -463,6 +504,10 @@ function createRuntime({
       toolName,
       async (args) => {
         const startedAt = /* @__PURE__ */ new Date();
+        const isNativeImplementationCall = hasActiveNativeToolExecution(
+          executionState,
+          toolName
+        );
         const toolContext = {
           input,
           metadata: context.metadata,
@@ -473,10 +518,14 @@ function createRuntime({
           const execution = await executeToolWithReplay({
             toolName,
             tool,
+            replay: isNativeImplementationCall ? void 0 : toolReplay?.[toolName],
             args,
             context: toolContext
           });
           const finishedAt = /* @__PURE__ */ new Date();
+          if (isNativeImplementationCall) {
+            return execution.result;
+          }
           const call = {
             name: toolName,
             arguments: args,
@@ -501,6 +550,9 @@ function createRuntime({
           return execution.result;
         } catch (error) {
           const finishedAt = /* @__PURE__ */ new Date();
+          if (isNativeImplementationCall) {
+            throw error;
+          }
           const call = {
             name: toolName,
             arguments: args,
@@ -653,6 +705,7 @@ function resolveErrors(result) {
 async function executeToolWithReplay({
   toolName,
   tool,
+  replay,
   args,
   context
 }) {
@@ -660,8 +713,8 @@ async function executeToolWithReplay({
     toolName,
     args,
     context,
-    execute: tool.execute,
-    replay: tool.replay
+    execute: (toolArgs, toolContext) => tool.execute(toolArgs, toolContext),
+    replay
   });
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type {\n  Harness,\n  HarnessContext,\n  HarnessMetadata,\n  HarnessPrompt,\n  HarnessRun,\n  JsonValue,\n  NormalizedMessage,\n  NormalizedSession,\n  TimingSummary,\n  ToolCallRecord,\n  UsageSummary,\n} from \"vitest-evals/harness\";\nimport {\n  attachHarnessRunToError,\n  isHarnessRun,\n  isNormalizedSession,\n  normalizeContent,\n  resolveHarnessRunErrors,\n  serializeError,\n  toJsonValue,\n} from \"vitest-evals/harness\";\nimport {\n  executeWithReplay,\n  getReplayMetadataFromError,\n  normalizeReplayMetadata,\n} from \"vitest-evals/replay\";\nimport type {\n  ReplayMode,\n  ToolRecording,\n  ToolReplayConfig,\n} from \"vitest-evals/replay\";\n\ntype MaybePromise<T> = T | Promise<T>;\ntype AnyPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;\ntype InferredPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<\n  string,\n  PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>\n>;\n\ntype PiAgentToolLike<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  replay?:\n    | boolean\n    | PiAiToolReplayConfig<\n        Record<string, JsonValue>,\n        JsonValue,\n        TInput,\n        TMetadata\n      >;\n  execute: (toolCallId: string, args: Record<string, JsonValue>) => unknown;\n};\n\nconst ORIGINAL_NATIVE_EXECUTE = Symbol(\"vitest-evals.originalNativeExecute\");\n\ntype NativeToolExecute<\n  TInput,\n  TMetadata extends HarnessMetadata,\n> = PiAgentToolLike<TInput, TMetadata>[\"execute\"] & {\n  [ORIGINAL_NATIVE_EXECUTE]?: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n};\n\nexport type PiAiReplayMode = ReplayMode;\n\nexport interface PiAiEventSink {\n  message: (message: NormalizedMessage) => void;\n  system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  user: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  tool: (\n    name: string,\n    content: JsonValue,\n    metadata?: Record<string, JsonValue>,\n  ) => void;\n}\n\nexport interface PiAiToolContext<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  input: TInput;\n  metadata: HarnessContext<TMetadata>[\"metadata\"];\n  signal?: AbortSignal;\n  setArtifact: HarnessContext<TMetadata>[\"setArtifact\"];\n}\n\nexport type PiAiToolRecording<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n> = ToolRecording<TArgs, TResult>;\n\nexport type PiAiToolReplayConfig<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;\n\nexport interface PiAiToolDefinition<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  description?: string;\n  replay?: boolean | PiAiToolReplayConfig<TArgs, TResult, TInput, TMetadata>;\n  execute: (\n    args: TArgs,\n    context: PiAiToolContext<TInput, TMetadata>,\n  ) => MaybePromise<TResult>;\n}\n\nexport type PiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = AnyPiAiToolset<TInput, TMetadata>;\n\ntype ToolArgs<TTool> = TTool extends PiAiToolDefinition<\n  infer TArgs,\n  infer _TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TArgs\n  : never;\n\ntype ToolResult<TTool> = TTool extends PiAiToolDefinition<\n  infer _TArgs,\n  infer TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TResult\n  : never;\n\nexport type PiAiRuntime<\n  TTools extends PiAiToolset<TInput, TMetadata>,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  tools: {\n    [K in keyof TTools]: (\n      args: ToolArgs<TTools[K]>,\n    ) => Promise<ToolResult<TTools[K]>>;\n  };\n  events: PiAiEventSink;\n  signal?: AbortSignal;\n};\n\nexport interface PiAiHarnessRunArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> {\n  agent: TAgent;\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  runtime: PiAiRuntime<TTools, TInput, TMetadata>;\n}\n\nexport interface PiAiHarnessResultArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {\n  result: TResult;\n}\n\ninterface PiAiHarnessBaseOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  agent?: TAgent;\n  createAgent?: () => MaybePromise<TAgent>;\n  normalize?: PiAiHarnessNormalizeOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >;\n  prompt: HarnessPrompt;\n  name?: string;\n}\n\nexport interface PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> {\n  tools: TTools;\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport interface PiAiHarnessInferredToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n> extends PiAiHarnessBaseOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    InferredPiAiToolset<TInput, TMetadata>\n  > {\n  tools?: undefined;\n  run?: (\n    args: PiAiHarnessRunArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      InferredPiAiToolset<TInput, TMetadata>\n    >,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport type PiAiHarnessOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> =\n  | PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>\n  | PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>;\n\ntype PiAiHarnessRunOptions<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> & {\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\ntype PiAiRunnableAgent<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = {\n  run: (\n    input: TInput,\n    runtime: PiAiRuntime<TTools, TInput, TMetadata>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\nexport interface PiAiHarnessNormalizeOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  session?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<NormalizedSession>;\n  output?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<JsonValue | undefined>;\n  usage?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<UsageSummary>;\n  timings?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<TimingSummary | undefined>;\n  errors?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<Array<Record<string, JsonValue>>>;\n}\n\ntype InferredToolSurfaces<TInput, TMetadata extends HarnessMetadata> = {\n  runtimeTools?: InferredPiAiToolset<TInput, TMetadata>;\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>;\n};\n\n/** Adapts a Pi agent runtime into a normalized vitest-evals harness. */\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessWithToolsOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n>(\n  options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): Harness<TInput, TMetadata> {\n  return {\n    name: options.name ?? \"pi-ai\",\n    prompt: options.prompt,\n    run: async (input, context) => {\n      const agent = await resolveAgent(options);\n      const messages: NormalizedMessage[] = [\n        {\n          role: \"user\",\n          content: normalizeContent(input),\n        },\n      ];\n      const inferredTools = resolveInferredToolSurfaces<TInput, TMetadata>(\n        agent,\n      );\n\n      if (hasExplicitToolset(options)) {\n        return executePiHarnessRun(\n          options,\n          agent,\n          input,\n          context,\n          messages,\n          options.tools,\n          inferredTools.nativeToolsets,\n        );\n      }\n\n      return executePiHarnessRun(\n        options,\n        agent,\n        input,\n        context,\n        messages,\n        inferredTools.runtimeTools,\n        inferredTools.nativeToolsets,\n      );\n    },\n  };\n}\n\nasync function executePiHarnessRun<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  agent: TAgent,\n  input: TInput,\n  context: HarnessContext<TMetadata>,\n  messages: NormalizedMessage[],\n  runtimeTools: TTools | undefined,\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n): Promise<HarnessRun> {\n  const runtime = createRuntime({\n    input,\n    context,\n    tools: runtimeTools,\n    messages,\n  });\n\n  try {\n    const result = await withInstrumentedAgentTools(\n      agent,\n      nativeToolsets,\n      {\n        input,\n        context,\n        messages,\n        toolCalls: runtime.toolCalls,\n      },\n      () =>\n        runAgent(options, {\n          agent,\n          input,\n          context,\n          runtime,\n        }),\n    );\n\n    if (isHarnessRun(result) && !hasResultOverrides(options)) {\n      if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {\n        result.artifacts = context.artifacts;\n      }\n      return result;\n    }\n\n    const normalizeResult = result as TResult;\n    const resultArgs = {\n      agent,\n      input,\n      context,\n      runtime,\n      result: normalizeResult,\n    } satisfies PiAiHarnessResultArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      TResult,\n      TTools\n    >;\n\n    const output = options.normalize?.output\n      ? await options.normalize.output(resultArgs)\n      : resolveOutput(normalizeResult);\n    const usage = options.normalize?.usage\n      ? await options.normalize.usage(resultArgs)\n      : resolveUsage(normalizeResult, runtime.toolCalls.length);\n    const session = options.normalize?.session\n      ? await options.normalize.session(resultArgs)\n      : resolveSession(normalizeResult, messages, output, usage);\n\n    return {\n      session,\n      output,\n      usage,\n      timings: options.normalize?.timings\n        ? await options.normalize.timings(resultArgs)\n        : undefined,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: options.normalize?.errors\n        ? await options.normalize.errors(resultArgs)\n        : resolveErrors(normalizeResult),\n    };\n  } catch (error) {\n    const usage = resolveUsage(undefined, runtime.toolCalls.length);\n    const run = {\n      session: resolveSession(undefined, messages, undefined, usage),\n      output: undefined,\n      usage,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: [serializeError(error)],\n    } satisfies HarnessRun;\n\n    throw attachHarnessRunToError(error, run);\n  }\n}\n\nasync function resolveAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>) {\n  if (options.agent !== undefined) {\n    return options.agent;\n  }\n\n  if (options.createAgent) {\n    return options.createAgent();\n  }\n\n  throw new Error(\n    \"piAiHarness requires either an agent instance or a createAgent() function.\",\n  );\n}\n\nfunction hasResultOverrides<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>) {\n  return Boolean(\n    options.normalize?.output ??\n      options.normalize?.session ??\n      options.normalize?.usage ??\n      options.normalize?.timings ??\n      options.normalize?.errors,\n  );\n}\n\nfunction resolveInferredToolSurfaces<TInput, TMetadata extends HarnessMetadata>(\n  agent: unknown,\n): InferredToolSurfaces<TInput, TMetadata> {\n  let runtimeTools: InferredPiAiToolset<TInput, TMetadata> | undefined;\n  const nativeToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nextRuntimeTools = getRuntimeToolset<TInput, TMetadata>(candidate);\n    if (runtimeTools === undefined && nextRuntimeTools !== undefined) {\n      runtimeTools = nextRuntimeTools;\n    }\n\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      nativeToolsets.push(nativeTools);\n    }\n  }\n\n  return {\n    ...(runtimeTools ? { runtimeTools } : {}),\n    ...(nativeToolsets.length > 0 ? { nativeToolsets } : {}),\n  };\n}\n\nfunction getAgentToolCandidates(agent: unknown): object[] {\n  const roots = getAgentRoots(agent);\n  const candidates: object[] = [];\n  const seen = new Set<object>();\n\n  for (const root of roots) {\n    addUniqueObject(candidates, seen, root);\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"state\"));\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"initialState\"));\n  }\n\n  return candidates;\n}\n\nfunction getAgentRoots(agent: unknown): object[] {\n  return [asObject(agent)]\n    .concat(asObject(getObjectProperty(agent, \"agent\")))\n    .filter((value): value is object => value !== undefined);\n}\n\nfunction addUniqueObject(\n  candidates: object[],\n  seen: Set<object>,\n  value: unknown,\n) {\n  if (!value || typeof value !== \"object\" || seen.has(value)) {\n    return;\n  }\n\n  seen.add(value);\n  candidates.push(value);\n}\n\nfunction asObject(value: unknown): object | undefined {\n  return value && typeof value === \"object\" ? value : undefined;\n}\n\nfunction getObjectProperty(value: unknown, key: string): unknown {\n  return value && typeof value === \"object\"\n    ? (value as Record<string, unknown>)[key]\n    : undefined;\n}\n\nfunction getRuntimeToolset<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): InferredPiAiToolset<TInput, TMetadata> | undefined {\n  const candidate =\n    getObjectProperty(value, \"tools\") ?? getObjectProperty(value, \"toolset\");\n\n  return isPiAiToolset(candidate)\n    ? (candidate as InferredPiAiToolset<TInput, TMetadata>)\n    : undefined;\n}\n\nfunction getNativeToolArray<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): PiAgentToolLike<TInput, TMetadata>[] | undefined {\n  const candidate = getObjectProperty(value, \"tools\");\n  if (isAgentToolArray(candidate)) {\n    return candidate as PiAgentToolLike<TInput, TMetadata>[];\n  }\n\n  return undefined;\n}\n\nasync function runAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n): Promise<TResult | HarnessRun> {\n  if (options.run) {\n    return options.run(args);\n  }\n\n  if (hasPiAiRunMethod<TInput, TMetadata, TResult, TTools>(args.agent)) {\n    return args.agent.run(args.input, args.runtime);\n  }\n\n  throw new Error(\n    \"piAiHarness requires a run() function unless the provided agent exposes run(input, runtime).\",\n  );\n}\n\nfunction hasExplicitToolset<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): options is PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput,\n  TMetadata,\n  TResult,\n  TTools\n> {\n  return options.tools !== undefined;\n}\n\nfunction hasPiAiRunMethod<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  agent: unknown,\n): agent is PiAiRunnableAgent<TInput, TMetadata, TResult, TTools> {\n  if (!agent || typeof agent !== \"object\") {\n    return false;\n  }\n\n  return (\n    \"run\" in agent && typeof (agent as { run?: unknown }).run === \"function\"\n  );\n}\n\nfunction isPiAiToolset(value: unknown): value is PiAiToolset {\n  if (!value || typeof value !== \"object\" || Array.isArray(value)) {\n    return false;\n  }\n\n  const tools = Object.values(value);\n  return (\n    tools.length > 0 &&\n    tools.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nfunction isAgentToolArray(value: unknown): value is PiAgentToolLike[] {\n  return (\n    Array.isArray(value) &&\n    value.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"name\" in tool &&\n          typeof (tool as { name?: unknown }).name === \"string\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nasync function withInstrumentedAgentTools<\n  TResult,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(\n  agent: unknown,\n  toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> | undefined,\n  args: {\n    input: TInput;\n    context: HarnessContext<TMetadata>;\n    messages: NormalizedMessage[];\n    toolCalls: ToolCallRecord[];\n  },\n  callback: () => Promise<TResult>,\n) {\n  if (!toolsets || toolsets.length === 0) {\n    return callback();\n  }\n\n  const originalExecutions = new Map<\n    PiAgentToolLike<TInput, TMetadata>,\n    PiAgentToolLike<TInput, TMetadata>[\"execute\"]\n  >();\n  const originalResets = new Map<ResettableAgent, ResettableAgent[\"reset\"]>();\n\n  const patchTool = (tool: PiAgentToolLike<TInput, TMetadata>) => {\n    if (originalExecutions.has(tool)) {\n      return;\n    }\n\n    const originalExecute = getNativeToolExecuteOrigin(tool.execute);\n    originalExecutions.set(tool, originalExecute);\n    const instrumentedExecute: NativeToolExecute<TInput, TMetadata> = async (\n      toolCallId: string,\n      rawArgs: Record<string, JsonValue>,\n    ) => {\n      const startedAt = new Date();\n      const toolContext = {\n        input: args.input,\n        metadata: args.context.metadata,\n        signal: args.context.signal,\n        setArtifact: args.context.setArtifact,\n      } satisfies PiAiToolContext<TInput, TMetadata>;\n\n      try {\n        const execution = await executeNativeToolWithReplay({\n          toolName: tool.name,\n          toolCallId,\n          execute: originalExecute,\n          replay: tool.replay,\n          args: rawArgs,\n          context: toolContext,\n        });\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          result: execution.normalizedResult,\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(execution.replay),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        args.messages.push({\n          role: \"tool\",\n          content: execution.normalizedResult,\n          metadata: {\n            name: tool.name,\n          },\n        });\n        return execution.result;\n      } catch (error) {\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          error: serializeToolCallError(error),\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(getReplayMetadataFromError(error)),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        throw error;\n      }\n    };\n    instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute;\n    tool.execute = instrumentedExecute;\n  };\n\n  const patchToolsets = (\n    nextToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n  ) => {\n    for (const toolset of nextToolsets) {\n      for (const tool of toolset) {\n        patchTool(tool);\n      }\n    }\n  };\n\n  patchToolsets(toolsets);\n\n  for (const target of getAgentResetTargets(agent)) {\n    const originalReset = target.reset;\n    originalResets.set(target, originalReset);\n    target.reset = function patchedReset(this: ResettableAgent, ...resetArgs) {\n      const resetResult = originalReset.apply(this, resetArgs);\n\n      if (isPromiseLike(resetResult)) {\n        return resetResult.finally(() => {\n          patchToolsets(\n            resolveInferredNativeToolsets<TInput, TMetadata>(agent),\n          );\n        });\n      }\n\n      patchToolsets(resolveInferredNativeToolsets<TInput, TMetadata>(agent));\n      return resetResult;\n    };\n  }\n\n  try {\n    return await callback();\n  } finally {\n    for (const [target, originalReset] of originalResets) {\n      target.reset = originalReset;\n    }\n\n    for (const [tool, originalExecute] of originalExecutions) {\n      tool.execute = originalExecute;\n    }\n  }\n}\n\ntype ResettableAgent = {\n  reset: (...args: unknown[]) => unknown;\n};\n\nfunction getAgentResetTargets(agent: unknown): ResettableAgent[] {\n  return getAgentRoots(agent).filter(isResettableAgent);\n}\n\nfunction isResettableAgent(value: object): value is ResettableAgent {\n  return \"reset\" in value && typeof value.reset === \"function\";\n}\n\nfunction resolveInferredNativeToolsets<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(agent: unknown): Array<PiAgentToolLike<TInput, TMetadata>[]> {\n  const toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      toolsets.push(nativeTools);\n    }\n  }\n\n  return toolsets;\n}\n\nfunction isPromiseLike(value: unknown): value is Promise<unknown> {\n  return Boolean(\n    value && typeof (value as { then?: unknown }).then === \"function\",\n  );\n}\n\nfunction serializeToolCallError(\n  error: unknown,\n): NonNullable<ToolCallRecord[\"error\"]> {\n  const serialized = serializeError(error);\n  const { message, type, ...details } = serialized;\n\n  return {\n    ...details,\n    message: typeof message === \"string\" ? message : String(message),\n    ...(typeof type === \"string\" ? { type } : {}),\n  };\n}\n\nfunction getNativeToolExecuteOrigin<TInput, TMetadata extends HarnessMetadata>(\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"],\n) {\n  const nativeExecute = execute as NativeToolExecute<TInput, TMetadata>;\n  return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;\n}\n\nasync function executeNativeToolWithReplay<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  toolCallId,\n  execute,\n  replay,\n  args,\n  context,\n}: {\n  toolName: string;\n  toolCallId: string;\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n  replay: PiAgentToolLike<TInput, TMetadata>[\"replay\"];\n  args: Record<string, JsonValue>;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  let didExecute = false;\n  let liveResult: unknown;\n\n  const execution = await executeWithReplay({\n    toolName,\n    args,\n    context,\n    execute: async (toolArgs) => {\n      didExecute = true;\n      liveResult = await execute(toolCallId, toolArgs);\n      return createNativeToolReplayEnvelope(liveResult);\n    },\n    replay,\n  });\n\n  if (didExecute) {\n    return {\n      result: liveResult,\n      normalizedResult: normalizeReplayToolResult(liveResult),\n      replay: execution.replay,\n    };\n  }\n\n  return {\n    ...resolveNativeToolReplayResult(execution.result),\n    replay: execution.replay,\n  };\n}\n\nfunction createRuntime<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>({\n  input,\n  context,\n  tools,\n  messages,\n}: {\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  tools: TTools | undefined;\n  messages: NormalizedMessage[];\n}): PiAiRuntime<TTools, TInput, TMetadata> & {\n  toolCalls: ToolCallRecord[];\n} {\n  const toolCalls: ToolCallRecord[] = [];\n  const eventSink: PiAiEventSink = {\n    message: (message) => {\n      messages.push(message);\n    },\n    system: (content, metadata) => {\n      messages.push({\n        role: \"system\",\n        content,\n        metadata,\n      });\n    },\n    user: (content, metadata) => {\n      messages.push({\n        role: \"user\",\n        content,\n        metadata,\n      });\n    },\n    assistant: (content, metadata) => {\n      messages.push({\n        role: \"assistant\",\n        content,\n        metadata,\n      });\n    },\n    tool: (name, content, metadata) => {\n      messages.push({\n        role: \"tool\",\n        content,\n        metadata: {\n          name,\n          ...(metadata ?? {}),\n        },\n      });\n    },\n  };\n\n  const runtimeTools = Object.fromEntries(\n    Object.entries(tools ?? {}).map(([toolName, tool]) => [\n      toolName,\n      async (args: Record<string, JsonValue>) => {\n        const startedAt = new Date();\n        const toolContext = {\n          input,\n          metadata: context.metadata,\n          signal: context.signal,\n          setArtifact: context.setArtifact,\n        } satisfies PiAiToolContext<TInput, TMetadata>;\n\n        try {\n          const execution = await executeToolWithReplay({\n            toolName,\n            tool,\n            args,\n            context: toolContext,\n          });\n          const finishedAt = new Date();\n          const call = {\n            name: toolName,\n            arguments: args,\n            result: execution.result,\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(execution.replay),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          messages.push({\n            role: \"tool\",\n            content: execution.result,\n            metadata: {\n              name: toolName,\n            },\n          });\n          return execution.result;\n        } catch (error) {\n          const finishedAt = new Date();\n          const call = {\n            name: toolName,\n            arguments: args,\n            error: serializeToolCallError(error),\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(\n              getReplayMetadataFromError(error),\n            ),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          throw error;\n        }\n      },\n    ]),\n  ) as unknown as PiAiRuntime<TTools, TInput, TMetadata>[\"tools\"];\n\n  return {\n    tools: runtimeTools,\n    events: eventSink,\n    signal: context.signal,\n    toolCalls,\n  };\n}\n\nfunction resolveOutput(result: unknown): JsonValue | undefined {\n  if (!result || typeof result !== \"object\") {\n    return toJsonValue(result);\n  }\n\n  const candidates = [\n    \"output\",\n    \"decision\",\n    \"result\",\n    \"final\",\n  ] satisfies string[];\n\n  for (const key of candidates) {\n    const value = (result as Record<string, unknown>)[key];\n    const normalized = toJsonValue(value);\n    if (normalized !== undefined) {\n      return normalized;\n    }\n  }\n\n  return undefined;\n}\n\nfunction normalizeToolResult(result: unknown): JsonValue | undefined {\n  const details =\n    result && typeof result === \"object\"\n      ? toJsonValue((result as { details?: unknown }).details)\n      : undefined;\n  if (details !== undefined) {\n    return details;\n  }\n\n  return (\n    toJsonValue(result) ?? (result === undefined ? undefined : String(result))\n  );\n}\n\nfunction normalizeReplayToolResult(result: unknown): JsonValue {\n  return normalizeToolResult(result) ?? null;\n}\n\ntype NativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 2;\n  };\n  agentResult: JsonValue;\n  normalizedResult: JsonValue;\n};\n\ntype LegacyNativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 1;\n  };\n  normalizedResult: JsonValue;\n  agentResult?: JsonValue;\n};\n\nfunction createNativeToolReplayEnvelope(\n  result: unknown,\n): NativeToolReplayEnvelope {\n  const normalizedResult = normalizeReplayToolResult(result);\n\n  return {\n    __vitestEvals: {\n      kind: \"pi-ai-native-tool-result\",\n      version: 2,\n    },\n    agentResult: toJsonValue(result) ?? normalizedResult,\n    normalizedResult,\n  };\n}\n\nfunction resolveNativeToolReplayResult(result: JsonValue) {\n  if (isNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  if (isLegacyNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult ?? result.normalizedResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  return {\n    result,\n    normalizedResult: normalizeReplayToolResult(result),\n  };\n}\n\nfunction isNativeToolReplayEnvelope(\n  value: unknown,\n): value is NativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"agentResult\" in value &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isNativeToolReplayMarker(\n  value: unknown,\n): value is NativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 2,\n  );\n}\n\nfunction isLegacyNativeToolReplayEnvelope(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isLegacyNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isLegacyNativeToolReplayMarker(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 1,\n  );\n}\n\nfunction resolveUsage(result: unknown, toolCallCount: number): UsageSummary {\n  if (!result || typeof result !== \"object\") {\n    return toolCallCount > 0 ? { toolCalls: toolCallCount } : {};\n  }\n\n  const usageValue =\n    (result as Record<string, unknown>).usage ??\n    (result as Record<string, unknown>).metrics;\n\n  const usage =\n    usageValue && typeof usageValue === \"object\"\n      ? ({ ...(usageValue as Record<string, unknown>) } as UsageSummary)\n      : {};\n\n  if (usage.toolCalls === undefined && toolCallCount > 0) {\n    usage.toolCalls = toolCallCount;\n  }\n\n  return usage;\n}\n\nfunction resolveSession(\n  result: unknown,\n  messages: NormalizedMessage[],\n  output: JsonValue | undefined,\n  usage: UsageSummary,\n): NormalizedSession {\n  if (\n    isNormalizedSession(\n      (result as Record<string, unknown> | undefined)?.session,\n    )\n  ) {\n    return (result as { session: NormalizedSession }).session;\n  }\n\n  if (\n    isNormalizedSession((result as Record<string, unknown> | undefined)?.trace)\n  ) {\n    return (result as { trace: NormalizedSession }).trace;\n  }\n\n  const sessionMessages = [...messages];\n  if (\n    output !== undefined &&\n    !sessionMessages.some(\n      (message) =>\n        message.role === \"assistant\" && message.content !== undefined,\n    )\n  ) {\n    sessionMessages.push({\n      role: \"assistant\",\n      content: output,\n    });\n  }\n\n  return {\n    messages: sessionMessages,\n    outputText: typeof output === \"string\" ? output : undefined,\n    provider:\n      ((result as Record<string, unknown> | undefined)?.provider as\n        | string\n        | undefined) ?? usage.provider,\n    model:\n      ((result as Record<string, unknown> | undefined)?.model as\n        | string\n        | undefined) ?? usage.model,\n  };\n}\n\nfunction resolveErrors(result: unknown): Array<Record<string, JsonValue>> {\n  return resolveHarnessRunErrors(result);\n}\n\nasync function executeToolWithReplay<\n  TArgs extends Record<string, JsonValue>,\n  TResult extends JsonValue,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  tool,\n  args,\n  context,\n}: {\n  toolName: string;\n  tool: PiAiToolDefinition<TArgs, TResult, TInput, TMetadata>;\n  args: TArgs;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  return executeWithReplay({\n    toolName,\n    args,\n    context,\n    execute: tool.execute,\n    replay: tool.replay,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAaA,qBAQO;AACP,oBAIO;AAoCP,IAAM,0BAA0B,OAAO,oCAAoC;AAoRpE,SAAS,YAUd,SAC4B;AAC5B,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,QAAQ,QAAQ;AAAA,IAChB,KAAK,OAAO,OAAO,YAAY;AAC7B,YAAM,QAAQ,MAAM,aAAa,OAAO;AACxC,YAAM,WAAgC;AAAA,QACpC;AAAA,UACE,MAAM;AAAA,UACN,aAAS,iCAAiB,KAAK;AAAA,QACjC;AAAA,MACF;AACA,YAAM,gBAAgB;AAAA,QACpB;AAAA,MACF;AAEA,UAAI,mBAAmB,OAAO,GAAG;AAC/B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,UACR,cAAc;AAAA,QAChB;AAAA,MACF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,QACd,cAAc;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,oBAOb,SACA,OACA,OACA,SACA,UACA,cACA,gBACqB;AACrB,QAAM,UAAU,cAAc;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP;AAAA,EACF,CAAC;AAED,MAAI;AACF,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,WAAW,QAAQ;AAAA,MACrB;AAAA,MACA,MACE,SAAS,SAAS;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACL;AAEA,YAAI,6BAAa,MAAM,KAAK,CAAC,mBAAmB,OAAO,GAAG;AACxD,UAAI,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,KAAK,CAAC,OAAO,WAAW;AAClE,eAAO,YAAY,QAAQ;AAAA,MAC7B;AACA,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB;AACxB,UAAM,aAAa;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IACV;AAQA,UAAM,SAAS,QAAQ,WAAW,SAC9B,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AACjC,UAAM,QAAQ,QAAQ,WAAW,QAC7B,MAAM,QAAQ,UAAU,MAAM,UAAU,IACxC,aAAa,iBAAiB,QAAQ,UAAU,MAAM;AAC1D,UAAM,UAAU,QAAQ,WAAW,UAC/B,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C,eAAe,iBAAiB,UAAU,QAAQ,KAAK;AAE3D,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS,QAAQ,WAAW,UACxB,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C;AAAA,MACJ,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,QAAQ,WAAW,SACvB,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AAAA,IACnC;AAAA,EACF,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,QAAW,QAAQ,UAAU,MAAM;AAC9D,UAAM,MAAM;AAAA,MACV,SAAS,eAAe,QAAW,UAAU,QAAW,KAAK;AAAA,MAC7D,QAAQ;AAAA,MACR;AAAA,MACA,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,KAAC,+BAAe,KAAK,CAAC;AAAA,IAChC;AAEA,cAAM,wCAAwB,OAAO,GAAG;AAAA,EAC1C;AACF;AAEA,eAAe,aAMb,SAAyE;AACzE,MAAI,QAAQ,UAAU,QAAW;AAC/B,WAAO,QAAQ;AAAA,EACjB;AAEA,MAAI,QAAQ,aAAa;AACvB,WAAO,QAAQ,YAAY;AAAA,EAC7B;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAMP,SAA4E;AAC5E,SAAO;AAAA,IACL,QAAQ,WAAW,UACjB,QAAQ,WAAW,WACnB,QAAQ,WAAW,SACnB,QAAQ,WAAW,WACnB,QAAQ,WAAW;AAAA,EACvB;AACF;AAEA,SAAS,4BACP,OACyC;AACzC,MAAI;AACJ,QAAM,iBAA8D,CAAC;AACrE,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,mBAAmB,kBAAqC,SAAS;AACvE,QAAI,iBAAiB,UAAa,qBAAqB,QAAW;AAChE,qBAAe;AAAA,IACjB;AAEA,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,qBAAe,KAAK,WAAW;AAAA,IACjC;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,IACvC,GAAI,eAAe,SAAS,IAAI,EAAE,eAAe,IAAI,CAAC;AAAA,EACxD;AACF;AAEA,SAAS,uBAAuB,OAA0B;AACxD,QAAM,QAAQ,cAAc,KAAK;AACjC,QAAM,aAAuB,CAAC;AAC9B,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,QAAQ,OAAO;AACxB,oBAAgB,YAAY,MAAM,IAAI;AACtC,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,OAAO,CAAC;AAClE,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,cAAc,CAAC;AAAA,EAC3E;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA0B;AAC/C,SAAO,CAAC,SAAS,KAAK,CAAC,EACpB,OAAO,SAAS,kBAAkB,OAAO,OAAO,CAAC,CAAC,EAClD,OAAO,CAAC,UAA2B,UAAU,MAAS;AAC3D;AAEA,SAAS,gBACP,YACA,MACA,OACA;AACA,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,KAAK,IAAI,KAAK,GAAG;AAC1D;AAAA,EACF;AAEA,OAAK,IAAI,KAAK;AACd,aAAW,KAAK,KAAK;AACvB;AAEA,SAAS,SAAS,OAAoC;AACpD,SAAO,SAAS,OAAO,UAAU,WAAW,QAAQ;AACtD;AAEA,SAAS,kBAAkB,OAAgB,KAAsB;AAC/D,SAAO,SAAS,OAAO,UAAU,WAC5B,MAAkC,GAAG,IACtC;AACN;AAEA,SAAS,kBACP,OACoD;AACpD,QAAM,YACJ,kBAAkB,OAAO,OAAO,KAAK,kBAAkB,OAAO,SAAS;AAEzE,SAAO,cAAc,SAAS,IACzB,YACD;AACN;AAEA,SAAS,mBACP,OACkD;AAClD,QAAM,YAAY,kBAAkB,OAAO,OAAO;AAClD,MAAI,iBAAiB,SAAS,GAAG;AAC/B,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEA,eAAe,SAOb,SACA,MAC+B;AAC/B,MAAI,QAAQ,KAAK;AACf,WAAO,QAAQ,IAAI,IAAI;AAAA,EACzB;AAEA,MAAI,iBAAqD,KAAK,KAAK,GAAG;AACpE,WAAO,KAAK,MAAM,IAAI,KAAK,OAAO,KAAK,OAAO;AAAA,EAChD;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAOP,SAOA;AACA,SAAO,QAAQ,UAAU;AAC3B;AAEA,SAAS,iBAMP,OACgE;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,SACE,SAAS,SAAS,OAAQ,MAA4B,QAAQ;AAElE;AAEA,SAAS,cAAc,OAAsC;AAC3D,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AAC/D,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,OAAO,OAAO,KAAK;AACjC,SACE,MAAM,SAAS,KACf,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,SAAS,iBAAiB,OAA4C;AACpE,SACE,MAAM,QAAQ,KAAK,KACnB,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,UAAU,QACV,OAAQ,KAA4B,SAAS,YAC7C,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,eAAe,2BAKb,OACA,UACA,MAMA,UACA;AACA,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO,SAAS;AAAA,EAClB;AAEA,QAAM,qBAAqB,oBAAI,IAG7B;AACF,QAAM,iBAAiB,oBAAI,IAA+C;AAE1E,QAAM,YAAY,CAAC,SAA6C;AAC9D,QAAI,mBAAmB,IAAI,IAAI,GAAG;AAChC;AAAA,IACF;AAEA,UAAM,kBAAkB,2BAA2B,KAAK,OAAO;AAC/D,uBAAmB,IAAI,MAAM,eAAe;AAC5C,UAAM,sBAA4D,OAChE,YACA,YACG;AACH,YAAM,YAAY,oBAAI,KAAK;AAC3B,YAAM,cAAc;AAAA,QAClB,OAAO,KAAK;AAAA,QACZ,UAAU,KAAK,QAAQ;AAAA,QACvB,QAAQ,KAAK,QAAQ;AAAA,QACrB,aAAa,KAAK,QAAQ;AAAA,MAC5B;AAEA,UAAI;AACF,cAAM,YAAY,MAAM,4BAA4B;AAAA,UAClD,UAAU,KAAK;AAAA,UACf;AAAA,UACA,SAAS;AAAA,UACT,QAAQ,KAAK;AAAA,UACb,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AACD,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,QAAQ,UAAU;AAAA,UAClB,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,cAAU,uCAAwB,UAAU,MAAM;AAAA,QACpD;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,SAAS,UAAU;AAAA,UACnB,UAAU;AAAA,YACR,MAAM,KAAK;AAAA,UACb;AAAA,QACF,CAAC;AACD,eAAO,UAAU;AAAA,MACnB,SAAS,OAAO;AACd,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,OAAO,uBAAuB,KAAK;AAAA,UACnC,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,cAAU,2CAAwB,0CAA2B,KAAK,CAAC;AAAA,QACrE;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,cAAM;AAAA,MACR;AAAA,IACF;AACA,wBAAoB,uBAAuB,IAAI;AAC/C,SAAK,UAAU;AAAA,EACjB;AAEA,QAAM,gBAAgB,CACpB,iBACG;AACH,eAAW,WAAW,cAAc;AAClC,iBAAW,QAAQ,SAAS;AAC1B,kBAAU,IAAI;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,gBAAc,QAAQ;AAEtB,aAAW,UAAU,qBAAqB,KAAK,GAAG;AAChD,UAAM,gBAAgB,OAAO;AAC7B,mBAAe,IAAI,QAAQ,aAAa;AACxC,WAAO,QAAQ,SAAS,gBAAuC,WAAW;AACxE,YAAM,cAAc,cAAc,MAAM,MAAM,SAAS;AAEvD,UAAI,cAAc,WAAW,GAAG;AAC9B,eAAO,YAAY,QAAQ,MAAM;AAC/B;AAAA,YACE,8BAAiD,KAAK;AAAA,UACxD;AAAA,QACF,CAAC;AAAA,MACH;AAEA,oBAAc,8BAAiD,KAAK,CAAC;AACrE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI;AACF,WAAO,MAAM,SAAS;AAAA,EACxB,UAAE;AACA,eAAW,CAAC,QAAQ,aAAa,KAAK,gBAAgB;AACpD,aAAO,QAAQ;AAAA,IACjB;AAEA,eAAW,CAAC,MAAM,eAAe,KAAK,oBAAoB;AACxD,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AACF;AAMA,SAAS,qBAAqB,OAAmC;AAC/D,SAAO,cAAc,KAAK,EAAE,OAAO,iBAAiB;AACtD;AAEA,SAAS,kBAAkB,OAAyC;AAClE,SAAO,WAAW,SAAS,OAAO,MAAM,UAAU;AACpD;AAEA,SAAS,8BAGP,OAA6D;AAC7D,QAAM,WAAwD,CAAC;AAC/D,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,eAAS,KAAK,WAAW;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA2C;AAChE,SAAO;AAAA,IACL,SAAS,OAAQ,MAA6B,SAAS;AAAA,EACzD;AACF;AAEA,SAAS,uBACP,OACsC;AACtC,QAAM,iBAAa,+BAAe,KAAK;AACvC,QAAM,EAAE,SAAS,MAAM,GAAG,QAAQ,IAAI;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,SAAS,OAAO,YAAY,WAAW,UAAU,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAAA,EAC7C;AACF;AAEA,SAAS,2BACP,SACA;AACA,QAAM,gBAAgB;AACtB,SAAO,cAAc,uBAAuB,KAAK;AACnD;AAEA,eAAe,4BAGb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAOG;AACD,MAAI,aAAa;AACjB,MAAI;AAEJ,QAAM,YAAY,UAAM,iCAAkB;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,OAAO,aAAa;AAC3B,mBAAa;AACb,mBAAa,MAAM,QAAQ,YAAY,QAAQ;AAC/C,aAAO,+BAA+B,UAAU;AAAA,IAClD;AAAA,IACA;AAAA,EACF,CAAC;AAED,MAAI,YAAY;AACd,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,kBAAkB,0BAA0B,UAAU;AAAA,MACtD,QAAQ,UAAU;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG,8BAA8B,UAAU,MAAM;AAAA,IACjD,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,cAIP;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAOE;AACA,QAAM,YAA8B,CAAC;AACrC,QAAM,YAA2B;AAAA,IAC/B,SAAS,CAAC,YAAY;AACpB,eAAS,KAAK,OAAO;AAAA,IACvB;AAAA,IACA,QAAQ,CAAC,SAAS,aAAa;AAC7B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,SAAS,aAAa;AAC3B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,WAAW,CAAC,SAAS,aAAa;AAChC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,MAAM,SAAS,aAAa;AACjC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA,UAAU;AAAA,UACR;AAAA,UACA,GAAI,YAAY,CAAC;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,eAAe,OAAO;AAAA,IAC1B,OAAO,QAAQ,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM;AAAA,MACpD;AAAA,MACA,OAAO,SAAoC;AACzC,cAAM,YAAY,oBAAI,KAAK;AAC3B,cAAM,cAAc;AAAA,UAClB;AAAA,UACA,UAAU,QAAQ;AAAA,UAClB,QAAQ,QAAQ;AAAA,UAChB,aAAa,QAAQ;AAAA,QACvB;AAEA,YAAI;AACF,gBAAM,YAAY,MAAM,sBAAsB;AAAA,YAC5C;AAAA,YACA;AAAA,YACA;AAAA,YACA,SAAS;AAAA,UACX,CAAC;AACD,gBAAM,aAAa,oBAAI,KAAK;AAC5B,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,QAAQ,UAAU;AAAA,YAClB,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,cAAU,uCAAwB,UAAU,MAAM;AAAA,UACpD;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,SAAS,UAAU;AAAA,YACnB,UAAU;AAAA,cACR,MAAM;AAAA,YACR;AAAA,UACF,CAAC;AACD,iBAAO,UAAU;AAAA,QACnB,SAAS,OAAO;AACd,gBAAM,aAAa,oBAAI,KAAK;AAC5B,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,OAAO,uBAAuB,KAAK;AAAA,YACnC,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,cAAU;AAAA,kBACR,0CAA2B,KAAK;AAAA,YAClC;AAAA,UACF;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,QAAQ,QAAQ;AAAA,IAChB;AAAA,EACF;AACF;AAEA,SAAS,cAAc,QAAwC;AAC7D,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,eAAO,4BAAY,MAAM;AAAA,EAC3B;AAEA,QAAM,aAAa;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,aAAW,OAAO,YAAY;AAC5B,UAAM,QAAS,OAAmC,GAAG;AACrD,UAAM,iBAAa,4BAAY,KAAK;AACpC,QAAI,eAAe,QAAW;AAC5B,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,QAAwC;AACnE,QAAM,UACJ,UAAU,OAAO,WAAW,eACxB,4BAAa,OAAiC,OAAO,IACrD;AACN,MAAI,YAAY,QAAW;AACzB,WAAO;AAAA,EACT;AAEA,aACE,4BAAY,MAAM,MAAM,WAAW,SAAY,SAAY,OAAO,MAAM;AAE5E;AAEA,SAAS,0BAA0B,QAA4B;AAC7D,SAAO,oBAAoB,MAAM,KAAK;AACxC;AAoBA,SAAS,+BACP,QAC0B;AAC1B,QAAM,mBAAmB,0BAA0B,MAAM;AAEzD,SAAO;AAAA,IACL,eAAe;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,iBAAa,4BAAY,MAAM,KAAK;AAAA,IACpC;AAAA,EACF;AACF;AAEA,SAAS,8BAA8B,QAAmB;AACxD,MAAI,2BAA2B,MAAM,GAAG;AACtC,WAAO;AAAA,MACL,QAAQ,OAAO;AAAA,MACf,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,iCAAiC,MAAM,GAAG;AAC5C,WAAO;AAAA,MACL,QAAQ,OAAO,eAAe,OAAO;AAAA,MACrC,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,kBAAkB,0BAA0B,MAAM;AAAA,EACpD;AACF;AAEA,SAAS,2BACP,OACmC;AACnC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,iBAAiB,SACjB,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,yBACP,OACoD;AACpD,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,iCACP,OACyC;AACzC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,+BACP,OAC0D;AAC1D,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,aAAa,QAAiB,eAAqC;AAC1E,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,WAAO,gBAAgB,IAAI,EAAE,WAAW,cAAc,IAAI,CAAC;AAAA,EAC7D;AAEA,QAAM,aACH,OAAmC,SACnC,OAAmC;AAEtC,QAAM,QACJ,cAAc,OAAO,eAAe,WAC/B,EAAE,GAAI,WAAuC,IAC9C,CAAC;AAEP,MAAI,MAAM,cAAc,UAAa,gBAAgB,GAAG;AACtD,UAAM,YAAY;AAAA,EACpB;AAEA,SAAO;AACT;AAEA,SAAS,eACP,QACA,UACA,QACA,OACmB;AACnB,UACE;AAAA,IACG,QAAgD;AAAA,EACnD,GACA;AACA,WAAQ,OAA0C;AAAA,EACpD;AAEA,UACE,oCAAqB,QAAgD,KAAK,GAC1E;AACA,WAAQ,OAAwC;AAAA,EAClD;AAEA,QAAM,kBAAkB,CAAC,GAAG,QAAQ;AACpC,MACE,WAAW,UACX,CAAC,gBAAgB;AAAA,IACf,CAAC,YACC,QAAQ,SAAS,eAAe,QAAQ,YAAY;AAAA,EACxD,GACA;AACA,oBAAgB,KAAK;AAAA,MACnB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,UAAU;AAAA,IACV,YAAY,OAAO,WAAW,WAAW,SAAS;AAAA,IAClD,UACI,QAAgD,YAEhC,MAAM;AAAA,IAC1B,OACI,QAAgD,SAEhC,MAAM;AAAA,EAC5B;AACF;AAEA,SAAS,cAAc,QAAmD;AACxE,aAAO,wCAAwB,MAAM;AACvC;AAEA,eAAe,sBAKb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAKG;AACD,aAAO,iCAAkB;AAAA,IACvB;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,KAAK;AAAA,IACd,QAAQ,KAAK;AAAA,EACf,CAAC;AACH;","names":[]}
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type {\n  Harness,\n  HarnessContext,\n  HarnessMetadata,\n  HarnessPrompt,\n  HarnessRun,\n  JsonValue,\n  NormalizedMessage,\n  NormalizedSession,\n  TimingSummary,\n  ToolCallRecord,\n  UsageSummary,\n} from \"vitest-evals/harness\";\nimport {\n  attachHarnessRunToError,\n  isHarnessRun,\n  isNormalizedSession,\n  normalizeContent,\n  resolveHarnessRunErrors,\n  serializeError,\n  toJsonValue,\n} from \"vitest-evals/harness\";\nimport {\n  executeWithReplay,\n  getReplayMetadataFromError,\n  normalizeReplayMetadata,\n} from \"vitest-evals/replay\";\nimport type {\n  ReplayMode,\n  ToolRecording,\n  ToolReplayConfig,\n} from \"vitest-evals/replay\";\n\ntype MaybePromise<T> = T | Promise<T>;\ntype AnyPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;\ntype InferredPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<\n  string,\n  PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>\n>;\n\ntype PiAgentToolLike<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  execute: (toolCallId: string, args: Record<string, JsonValue>) => unknown;\n};\n\nconst ORIGINAL_NATIVE_EXECUTE = Symbol(\"vitest-evals.originalNativeExecute\");\n\ntype NativeToolExecute<\n  TInput,\n  TMetadata extends HarnessMetadata,\n> = PiAgentToolLike<TInput, TMetadata>[\"execute\"] & {\n  [ORIGINAL_NATIVE_EXECUTE]?: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n};\n\nexport type PiAiReplayMode = ReplayMode;\n\nexport interface PiAiEventSink {\n  message: (message: NormalizedMessage) => void;\n  system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  user: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  tool: (\n    name: string,\n    content: JsonValue,\n    metadata?: Record<string, JsonValue>,\n  ) => void;\n}\n\nexport interface PiAiToolContext<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  input: TInput;\n  metadata: HarnessContext<TMetadata>[\"metadata\"];\n  signal?: AbortSignal;\n  setArtifact: HarnessContext<TMetadata>[\"setArtifact\"];\n}\n\nexport type PiAiToolRecording<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n> = ToolRecording<TArgs, TResult>;\n\nexport type PiAiToolReplayConfig<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;\n\nexport type PiAiToolReplayPolicy<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> =\n  | boolean\n  | PiAiToolReplayConfig<\n      Record<string, JsonValue>,\n      JsonValue,\n      TInput,\n      TMetadata\n    >;\n\nexport type PiAiToolReplayPolicies<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<string, PiAiToolReplayPolicy<TInput, TMetadata>>;\n\nexport interface PiAiToolDefinition<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  description?: string;\n  execute: (\n    args: TArgs,\n    context: PiAiToolContext<TInput, TMetadata>,\n  ) => MaybePromise<TResult>;\n}\n\nexport type PiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = AnyPiAiToolset<TInput, TMetadata>;\n\ntype ToolArgs<TTool> = TTool extends PiAiToolDefinition<\n  infer TArgs,\n  infer _TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TArgs\n  : never;\n\ntype ToolResult<TTool> = TTool extends PiAiToolDefinition<\n  infer _TArgs,\n  infer TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TResult\n  : never;\n\nexport type PiAiRuntime<\n  TTools extends PiAiToolset<TInput, TMetadata>,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  tools: {\n    [K in keyof TTools]: (\n      args: ToolArgs<TTools[K]>,\n    ) => Promise<ToolResult<TTools[K]>>;\n  };\n  events: PiAiEventSink;\n  signal?: AbortSignal;\n};\n\nexport interface PiAiHarnessRunArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> {\n  agent: TAgent;\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  runtime: PiAiRuntime<TTools, TInput, TMetadata>;\n}\n\nexport interface PiAiHarnessResultArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {\n  result: TResult;\n}\n\nexport interface PiAiCreateAgentArgs<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n}\n\ninterface PiAiHarnessBaseOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  agent?: TAgent;\n  createAgent?: (\n    args: PiAiCreateAgentArgs<TInput, TMetadata>,\n  ) => MaybePromise<TAgent>;\n  toolReplay?: PiAiToolReplayPolicies<TInput, TMetadata>;\n  normalize?: PiAiHarnessNormalizeOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >;\n  prompt: HarnessPrompt;\n  name?: string;\n}\n\nexport interface PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> {\n  tools: TTools;\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport interface PiAiHarnessInferredToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n> extends PiAiHarnessBaseOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    InferredPiAiToolset<TInput, TMetadata>\n  > {\n  tools?: undefined;\n  run?: (\n    args: PiAiHarnessRunArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      InferredPiAiToolset<TInput, TMetadata>\n    >,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport type PiAiHarnessOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> =\n  | PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>\n  | PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>;\n\ntype PiAiHarnessRunOptions<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> & {\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\ntype PiAiRunnableAgent<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = {\n  run: (\n    input: TInput,\n    runtime: PiAiRuntime<TTools, TInput, TMetadata>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\nexport interface PiAiHarnessNormalizeOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  session?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<NormalizedSession>;\n  output?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<JsonValue | undefined>;\n  usage?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<UsageSummary>;\n  timings?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<TimingSummary | undefined>;\n  errors?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<Array<Record<string, JsonValue>>>;\n}\n\ntype InferredToolSurfaces<TInput, TMetadata extends HarnessMetadata> = {\n  runtimeTools?: InferredPiAiToolset<TInput, TMetadata>;\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>;\n};\n\ntype PiToolExecutionState = {\n  activeNativeToolNames: Map<string, number>;\n};\n\n/** Adapts a Pi agent runtime into a normalized vitest-evals harness. */\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessWithToolsOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n>(\n  options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): Harness<TInput, TMetadata> {\n  return {\n    name: options.name ?? \"pi-ai\",\n    prompt: options.prompt,\n    run: async (input, context) => {\n      const agent = await resolveAgent(options, {\n        input,\n        context,\n      });\n      const messages: NormalizedMessage[] = [\n        {\n          role: \"user\",\n          content: normalizeContent(input),\n        },\n      ];\n      const inferredTools = resolveInferredToolSurfaces<TInput, TMetadata>(\n        agent,\n      );\n\n      if (hasExplicitToolset(options)) {\n        return executePiHarnessRun(\n          options,\n          agent,\n          input,\n          context,\n          messages,\n          options.tools,\n          inferredTools.nativeToolsets,\n        );\n      }\n\n      return executePiHarnessRun(\n        options,\n        agent,\n        input,\n        context,\n        messages,\n        inferredTools.runtimeTools,\n        inferredTools.nativeToolsets,\n      );\n    },\n  };\n}\n\nasync function executePiHarnessRun<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  agent: TAgent,\n  input: TInput,\n  context: HarnessContext<TMetadata>,\n  messages: NormalizedMessage[],\n  runtimeTools: TTools | undefined,\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n): Promise<HarnessRun> {\n  const executionState = createPiToolExecutionState();\n  const runtime = createRuntime({\n    input,\n    context,\n    tools: runtimeTools,\n    toolReplay: options.toolReplay,\n    executionState,\n    messages,\n  });\n\n  try {\n    const result = await withInstrumentedAgentTools(\n      agent,\n      nativeToolsets,\n      {\n        input,\n        context,\n        messages,\n        toolCalls: runtime.toolCalls,\n        toolReplay: options.toolReplay,\n        executionState,\n      },\n      () =>\n        runAgent(options, {\n          agent,\n          input,\n          context,\n          runtime,\n        }),\n    );\n\n    if (isHarnessRun(result) && !hasResultOverrides(options)) {\n      if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {\n        result.artifacts = context.artifacts;\n      }\n      return result;\n    }\n\n    const normalizeResult = result as TResult;\n    const resultArgs = {\n      agent,\n      input,\n      context,\n      runtime,\n      result: normalizeResult,\n    } satisfies PiAiHarnessResultArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      TResult,\n      TTools\n    >;\n\n    const output = options.normalize?.output\n      ? await options.normalize.output(resultArgs)\n      : resolveOutput(normalizeResult);\n    const usage = options.normalize?.usage\n      ? await options.normalize.usage(resultArgs)\n      : resolveUsage(normalizeResult, runtime.toolCalls.length);\n    const session = options.normalize?.session\n      ? await options.normalize.session(resultArgs)\n      : resolveSession(normalizeResult, messages, output, usage);\n\n    return {\n      session,\n      output,\n      usage,\n      timings: options.normalize?.timings\n        ? await options.normalize.timings(resultArgs)\n        : undefined,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: options.normalize?.errors\n        ? await options.normalize.errors(resultArgs)\n        : resolveErrors(normalizeResult),\n    };\n  } catch (error) {\n    const usage = resolveUsage(undefined, runtime.toolCalls.length);\n    const run = {\n      session: resolveSession(undefined, messages, undefined, usage),\n      output: undefined,\n      usage,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: [serializeError(error)],\n    } satisfies HarnessRun;\n\n    throw attachHarnessRunToError(error, run);\n  }\n}\n\nasync function resolveAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  args: PiAiCreateAgentArgs<TInput, TMetadata>,\n) {\n  if (options.agent !== undefined) {\n    return options.agent;\n  }\n\n  if (options.createAgent) {\n    return options.createAgent(args);\n  }\n\n  throw new Error(\n    \"piAiHarness requires either an agent instance or a createAgent() function.\",\n  );\n}\n\nfunction hasResultOverrides<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>) {\n  return Boolean(\n    options.normalize?.output ??\n      options.normalize?.session ??\n      options.normalize?.usage ??\n      options.normalize?.timings ??\n      options.normalize?.errors,\n  );\n}\n\nfunction resolveInferredToolSurfaces<TInput, TMetadata extends HarnessMetadata>(\n  agent: unknown,\n): InferredToolSurfaces<TInput, TMetadata> {\n  let runtimeTools: InferredPiAiToolset<TInput, TMetadata> | undefined;\n  const nativeToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nextRuntimeTools = getRuntimeToolset<TInput, TMetadata>(candidate);\n    if (runtimeTools === undefined && nextRuntimeTools !== undefined) {\n      runtimeTools = nextRuntimeTools;\n    }\n\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      nativeToolsets.push(nativeTools);\n    }\n  }\n\n  return {\n    ...(runtimeTools ? { runtimeTools } : {}),\n    ...(nativeToolsets.length > 0 ? { nativeToolsets } : {}),\n  };\n}\n\nfunction getAgentToolCandidates(agent: unknown): object[] {\n  const roots = getAgentRoots(agent);\n  const candidates: object[] = [];\n  const seen = new Set<object>();\n\n  for (const root of roots) {\n    addUniqueObject(candidates, seen, root);\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"state\"));\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"initialState\"));\n  }\n\n  return candidates;\n}\n\nfunction getAgentRoots(agent: unknown): object[] {\n  return [asObject(agent)]\n    .concat(asObject(getObjectProperty(agent, \"agent\")))\n    .filter((value): value is object => value !== undefined);\n}\n\nfunction addUniqueObject(\n  candidates: object[],\n  seen: Set<object>,\n  value: unknown,\n) {\n  if (!value || typeof value !== \"object\" || seen.has(value)) {\n    return;\n  }\n\n  seen.add(value);\n  candidates.push(value);\n}\n\nfunction asObject(value: unknown): object | undefined {\n  return value && typeof value === \"object\" ? value : undefined;\n}\n\nfunction getObjectProperty(value: unknown, key: string): unknown {\n  return value && typeof value === \"object\"\n    ? (value as Record<string, unknown>)[key]\n    : undefined;\n}\n\nfunction getRuntimeToolset<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): InferredPiAiToolset<TInput, TMetadata> | undefined {\n  const candidate =\n    getObjectProperty(value, \"tools\") ?? getObjectProperty(value, \"toolset\");\n\n  return isPiAiToolset(candidate)\n    ? (candidate as InferredPiAiToolset<TInput, TMetadata>)\n    : undefined;\n}\n\nfunction getNativeToolArray<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): PiAgentToolLike<TInput, TMetadata>[] | undefined {\n  const candidate = getObjectProperty(value, \"tools\");\n  if (isAgentToolArray(candidate)) {\n    return candidate as PiAgentToolLike<TInput, TMetadata>[];\n  }\n\n  return undefined;\n}\n\nasync function runAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n): Promise<TResult | HarnessRun> {\n  if (options.run) {\n    return options.run(args);\n  }\n\n  if (hasPiAiRunMethod<TInput, TMetadata, TResult, TTools>(args.agent)) {\n    return args.agent.run(args.input, args.runtime);\n  }\n\n  throw new Error(\n    \"piAiHarness requires a run() function unless the provided agent exposes run(input, runtime).\",\n  );\n}\n\nfunction hasExplicitToolset<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): options is PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput,\n  TMetadata,\n  TResult,\n  TTools\n> {\n  return options.tools !== undefined;\n}\n\nfunction hasPiAiRunMethod<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  agent: unknown,\n): agent is PiAiRunnableAgent<TInput, TMetadata, TResult, TTools> {\n  if (!agent || typeof agent !== \"object\") {\n    return false;\n  }\n\n  return (\n    \"run\" in agent && typeof (agent as { run?: unknown }).run === \"function\"\n  );\n}\n\nfunction isPiAiToolset(value: unknown): value is PiAiToolset {\n  if (!value || typeof value !== \"object\" || Array.isArray(value)) {\n    return false;\n  }\n\n  const tools = Object.values(value);\n  return (\n    tools.length > 0 &&\n    tools.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nfunction isAgentToolArray(value: unknown): value is PiAgentToolLike[] {\n  return (\n    Array.isArray(value) &&\n    value.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"name\" in tool &&\n          typeof (tool as { name?: unknown }).name === \"string\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nasync function withInstrumentedAgentTools<\n  TResult,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(\n  agent: unknown,\n  toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> | undefined,\n  args: {\n    input: TInput;\n    context: HarnessContext<TMetadata>;\n    messages: NormalizedMessage[];\n    toolCalls: ToolCallRecord[];\n    toolReplay: PiAiToolReplayPolicies<TInput, TMetadata> | undefined;\n    executionState: PiToolExecutionState;\n  },\n  callback: () => Promise<TResult>,\n) {\n  if (!toolsets || toolsets.length === 0) {\n    return callback();\n  }\n\n  const originalExecutions = new Map<\n    PiAgentToolLike<TInput, TMetadata>,\n    PiAgentToolLike<TInput, TMetadata>[\"execute\"]\n  >();\n  const originalResets = new Map<ResettableAgent, ResettableAgent[\"reset\"]>();\n\n  const patchTool = (tool: PiAgentToolLike<TInput, TMetadata>) => {\n    if (originalExecutions.has(tool)) {\n      return;\n    }\n\n    const originalExecute = getNativeToolExecuteOrigin(tool.execute);\n    originalExecutions.set(tool, originalExecute);\n    const instrumentedExecute: NativeToolExecute<TInput, TMetadata> = async (\n      toolCallId: string,\n      rawArgs: Record<string, JsonValue>,\n    ) => {\n      const startedAt = new Date();\n      const toolContext = {\n        input: args.input,\n        metadata: args.context.metadata,\n        signal: args.context.signal,\n        setArtifact: args.context.setArtifact,\n      } satisfies PiAiToolContext<TInput, TMetadata>;\n      const leaveNativeTool = enterNativeToolExecution(\n        args.executionState,\n        tool.name,\n      );\n\n      try {\n        const execution = await executeNativeToolWithReplay({\n          toolName: tool.name,\n          toolCallId,\n          execute: originalExecute,\n          replay: args.toolReplay?.[tool.name],\n          args: rawArgs,\n          context: toolContext,\n        });\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          result: execution.normalizedResult,\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(execution.replay),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        args.messages.push({\n          role: \"tool\",\n          content: execution.normalizedResult,\n          metadata: {\n            name: tool.name,\n          },\n        });\n        return execution.result;\n      } catch (error) {\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          error: serializeToolCallError(error),\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(getReplayMetadataFromError(error)),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        throw error;\n      } finally {\n        leaveNativeTool();\n      }\n    };\n    instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute;\n    tool.execute = instrumentedExecute;\n  };\n\n  const patchToolsets = (\n    nextToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n  ) => {\n    for (const toolset of nextToolsets) {\n      for (const tool of toolset) {\n        patchTool(tool);\n      }\n    }\n  };\n\n  patchToolsets(toolsets);\n\n  for (const target of getAgentResetTargets(agent)) {\n    const originalReset = target.reset;\n    originalResets.set(target, originalReset);\n    target.reset = function patchedReset(this: ResettableAgent, ...resetArgs) {\n      const resetResult = originalReset.apply(this, resetArgs);\n\n      if (isPromiseLike(resetResult)) {\n        return resetResult.finally(() => {\n          patchToolsets(\n            resolveInferredNativeToolsets<TInput, TMetadata>(agent),\n          );\n        });\n      }\n\n      patchToolsets(resolveInferredNativeToolsets<TInput, TMetadata>(agent));\n      return resetResult;\n    };\n  }\n\n  try {\n    return await callback();\n  } finally {\n    for (const [target, originalReset] of originalResets) {\n      target.reset = originalReset;\n    }\n\n    for (const [tool, originalExecute] of originalExecutions) {\n      tool.execute = originalExecute;\n    }\n  }\n}\n\ntype ResettableAgent = {\n  reset: (...args: unknown[]) => unknown;\n};\n\nfunction getAgentResetTargets(agent: unknown): ResettableAgent[] {\n  return getAgentRoots(agent).filter(isResettableAgent);\n}\n\nfunction isResettableAgent(value: object): value is ResettableAgent {\n  return \"reset\" in value && typeof value.reset === \"function\";\n}\n\nfunction resolveInferredNativeToolsets<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(agent: unknown): Array<PiAgentToolLike<TInput, TMetadata>[]> {\n  const toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      toolsets.push(nativeTools);\n    }\n  }\n\n  return toolsets;\n}\n\nfunction isPromiseLike(value: unknown): value is Promise<unknown> {\n  return Boolean(\n    value && typeof (value as { then?: unknown }).then === \"function\",\n  );\n}\n\nfunction serializeToolCallError(\n  error: unknown,\n): NonNullable<ToolCallRecord[\"error\"]> {\n  const serialized = serializeError(error);\n  const { message, type, ...details } = serialized;\n\n  return {\n    ...details,\n    message: typeof message === \"string\" ? message : String(message),\n    ...(typeof type === \"string\" ? { type } : {}),\n  };\n}\n\nfunction getNativeToolExecuteOrigin<TInput, TMetadata extends HarnessMetadata>(\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"],\n) {\n  const nativeExecute = execute as NativeToolExecute<TInput, TMetadata>;\n  return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;\n}\n\nfunction createPiToolExecutionState(): PiToolExecutionState {\n  return {\n    activeNativeToolNames: new Map(),\n  };\n}\n\nfunction enterNativeToolExecution(\n  state: PiToolExecutionState,\n  toolName: string,\n) {\n  state.activeNativeToolNames.set(\n    toolName,\n    (state.activeNativeToolNames.get(toolName) ?? 0) + 1,\n  );\n\n  return () => {\n    const nextCount = (state.activeNativeToolNames.get(toolName) ?? 1) - 1;\n    if (nextCount <= 0) {\n      state.activeNativeToolNames.delete(toolName);\n      return;\n    }\n\n    state.activeNativeToolNames.set(toolName, nextCount);\n  };\n}\n\nfunction hasActiveNativeToolExecution(\n  state: PiToolExecutionState,\n  toolName: string,\n) {\n  return (state.activeNativeToolNames.get(toolName) ?? 0) > 0;\n}\n\nasync function executeNativeToolWithReplay<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  toolCallId,\n  execute,\n  replay,\n  args,\n  context,\n}: {\n  toolName: string;\n  toolCallId: string;\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n  replay: PiAiToolReplayPolicy<TInput, TMetadata> | undefined;\n  args: Record<string, JsonValue>;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  let didExecute = false;\n  let liveResult: unknown;\n\n  const execution = await executeWithReplay({\n    toolName: createNativeReplayToolName(toolName),\n    args,\n    context,\n    execute: async (toolArgs) => {\n      didExecute = true;\n      liveResult = await execute(toolCallId, toolArgs);\n      return createNativeToolReplayEnvelope(liveResult);\n    },\n    replay,\n  });\n\n  if (didExecute) {\n    return {\n      result: liveResult,\n      normalizedResult: normalizeReplayToolResult(liveResult),\n      replay: execution.replay,\n    };\n  }\n\n  return {\n    ...resolveNativeToolReplayResult(execution.result),\n    replay: execution.replay,\n  };\n}\n\nfunction createNativeReplayToolName(toolName: string) {\n  return `${toolName}.native`;\n}\n\nfunction createRuntime<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>({\n  input,\n  context,\n  tools,\n  toolReplay,\n  executionState,\n  messages,\n}: {\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  tools: TTools | undefined;\n  toolReplay: PiAiToolReplayPolicies<TInput, TMetadata> | undefined;\n  executionState: PiToolExecutionState;\n  messages: NormalizedMessage[];\n}): PiAiRuntime<TTools, TInput, TMetadata> & {\n  toolCalls: ToolCallRecord[];\n} {\n  const toolCalls: ToolCallRecord[] = [];\n  const eventSink: PiAiEventSink = {\n    message: (message) => {\n      messages.push(message);\n    },\n    system: (content, metadata) => {\n      messages.push({\n        role: \"system\",\n        content,\n        metadata,\n      });\n    },\n    user: (content, metadata) => {\n      messages.push({\n        role: \"user\",\n        content,\n        metadata,\n      });\n    },\n    assistant: (content, metadata) => {\n      messages.push({\n        role: \"assistant\",\n        content,\n        metadata,\n      });\n    },\n    tool: (name, content, metadata) => {\n      messages.push({\n        role: \"tool\",\n        content,\n        metadata: {\n          name,\n          ...(metadata ?? {}),\n        },\n      });\n    },\n  };\n\n  const runtimeTools = Object.fromEntries(\n    Object.entries(tools ?? {}).map(([toolName, tool]) => [\n      toolName,\n      async (args: Record<string, JsonValue>) => {\n        const startedAt = new Date();\n        const isNativeImplementationCall = hasActiveNativeToolExecution(\n          executionState,\n          toolName,\n        );\n        const toolContext = {\n          input,\n          metadata: context.metadata,\n          signal: context.signal,\n          setArtifact: context.setArtifact,\n        } satisfies PiAiToolContext<TInput, TMetadata>;\n\n        try {\n          const execution = await executeToolWithReplay({\n            toolName,\n            tool,\n            replay: isNativeImplementationCall\n              ? undefined\n              : toolReplay?.[toolName],\n            args,\n            context: toolContext,\n          });\n          const finishedAt = new Date();\n\n          if (isNativeImplementationCall) {\n            return execution.result;\n          }\n\n          const call = {\n            name: toolName,\n            arguments: args,\n            result: execution.result,\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(execution.replay),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          messages.push({\n            role: \"tool\",\n            content: execution.result,\n            metadata: {\n              name: toolName,\n            },\n          });\n          return execution.result;\n        } catch (error) {\n          const finishedAt = new Date();\n          if (isNativeImplementationCall) {\n            throw error;\n          }\n\n          const call = {\n            name: toolName,\n            arguments: args,\n            error: serializeToolCallError(error),\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(\n              getReplayMetadataFromError(error),\n            ),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          throw error;\n        }\n      },\n    ]),\n  ) as unknown as PiAiRuntime<TTools, TInput, TMetadata>[\"tools\"];\n\n  return {\n    tools: runtimeTools,\n    events: eventSink,\n    signal: context.signal,\n    toolCalls,\n  };\n}\n\nfunction resolveOutput(result: unknown): JsonValue | undefined {\n  if (!result || typeof result !== \"object\") {\n    return toJsonValue(result);\n  }\n\n  const candidates = [\n    \"output\",\n    \"decision\",\n    \"result\",\n    \"final\",\n  ] satisfies string[];\n\n  for (const key of candidates) {\n    const value = (result as Record<string, unknown>)[key];\n    const normalized = toJsonValue(value);\n    if (normalized !== undefined) {\n      return normalized;\n    }\n  }\n\n  return undefined;\n}\n\nfunction normalizeToolResult(result: unknown): JsonValue | undefined {\n  const details =\n    result && typeof result === \"object\"\n      ? toJsonValue((result as { details?: unknown }).details)\n      : undefined;\n  if (details !== undefined) {\n    return details;\n  }\n\n  return (\n    toJsonValue(result) ?? (result === undefined ? undefined : String(result))\n  );\n}\n\nfunction normalizeReplayToolResult(result: unknown): JsonValue {\n  return normalizeToolResult(result) ?? null;\n}\n\ntype NativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 2;\n  };\n  agentResult: JsonValue;\n  normalizedResult: JsonValue;\n};\n\ntype LegacyNativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 1;\n  };\n  normalizedResult: JsonValue;\n  agentResult?: JsonValue;\n};\n\nfunction createNativeToolReplayEnvelope(\n  result: unknown,\n): NativeToolReplayEnvelope {\n  const normalizedResult = normalizeReplayToolResult(result);\n\n  return {\n    __vitestEvals: {\n      kind: \"pi-ai-native-tool-result\",\n      version: 2,\n    },\n    agentResult: toJsonValue(result) ?? normalizedResult,\n    normalizedResult,\n  };\n}\n\nfunction resolveNativeToolReplayResult(result: JsonValue) {\n  if (isNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  if (isLegacyNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult ?? result.normalizedResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  return {\n    result,\n    normalizedResult: normalizeReplayToolResult(result),\n  };\n}\n\nfunction isNativeToolReplayEnvelope(\n  value: unknown,\n): value is NativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"agentResult\" in value &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isNativeToolReplayMarker(\n  value: unknown,\n): value is NativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 2,\n  );\n}\n\nfunction isLegacyNativeToolReplayEnvelope(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isLegacyNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isLegacyNativeToolReplayMarker(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 1,\n  );\n}\n\nfunction resolveUsage(result: unknown, toolCallCount: number): UsageSummary {\n  if (!result || typeof result !== \"object\") {\n    return toolCallCount > 0 ? { toolCalls: toolCallCount } : {};\n  }\n\n  const usageValue =\n    (result as Record<string, unknown>).usage ??\n    (result as Record<string, unknown>).metrics;\n\n  const usage =\n    usageValue && typeof usageValue === \"object\"\n      ? ({ ...(usageValue as Record<string, unknown>) } as UsageSummary)\n      : {};\n\n  if (usage.toolCalls === undefined && toolCallCount > 0) {\n    usage.toolCalls = toolCallCount;\n  }\n\n  return usage;\n}\n\nfunction resolveSession(\n  result: unknown,\n  messages: NormalizedMessage[],\n  output: JsonValue | undefined,\n  usage: UsageSummary,\n): NormalizedSession {\n  if (\n    isNormalizedSession(\n      (result as Record<string, unknown> | undefined)?.session,\n    )\n  ) {\n    return (result as { session: NormalizedSession }).session;\n  }\n\n  if (\n    isNormalizedSession((result as Record<string, unknown> | undefined)?.trace)\n  ) {\n    return (result as { trace: NormalizedSession }).trace;\n  }\n\n  const sessionMessages = [...messages];\n  if (\n    output !== undefined &&\n    !sessionMessages.some(\n      (message) =>\n        message.role === \"assistant\" && message.content !== undefined,\n    )\n  ) {\n    sessionMessages.push({\n      role: \"assistant\",\n      content: output,\n    });\n  }\n\n  return {\n    messages: sessionMessages,\n    outputText: typeof output === \"string\" ? output : undefined,\n    provider:\n      ((result as Record<string, unknown> | undefined)?.provider as\n        | string\n        | undefined) ?? usage.provider,\n    model:\n      ((result as Record<string, unknown> | undefined)?.model as\n        | string\n        | undefined) ?? usage.model,\n  };\n}\n\nfunction resolveErrors(result: unknown): Array<Record<string, JsonValue>> {\n  return resolveHarnessRunErrors(result);\n}\n\nasync function executeToolWithReplay<\n  TArgs extends Record<string, JsonValue>,\n  TResult extends JsonValue,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  tool,\n  replay,\n  args,\n  context,\n}: {\n  toolName: string;\n  tool: PiAiToolDefinition<TArgs, TResult, TInput, TMetadata>;\n  replay: PiAiToolReplayPolicy<TInput, TMetadata> | undefined;\n  args: TArgs;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  return executeWithReplay<\n    Record<string, JsonValue>,\n    JsonValue,\n    PiAiToolContext<TInput, TMetadata>\n  >({\n    toolName,\n    args,\n    context,\n    execute: (toolArgs, toolContext) =>\n      tool.execute(toolArgs as TArgs, toolContext),\n    replay,\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAaA,qBAQO;AACP,oBAIO;AA4BP,IAAM,0BAA0B,OAAO,oCAAoC;AAmTpE,SAAS,YAUd,SAC4B;AAC5B,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,QAAQ,QAAQ;AAAA,IAChB,KAAK,OAAO,OAAO,YAAY;AAC7B,YAAM,QAAQ,MAAM,aAAa,SAAS;AAAA,QACxC;AAAA,QACA;AAAA,MACF,CAAC;AACD,YAAM,WAAgC;AAAA,QACpC;AAAA,UACE,MAAM;AAAA,UACN,aAAS,iCAAiB,KAAK;AAAA,QACjC;AAAA,MACF;AACA,YAAM,gBAAgB;AAAA,QACpB;AAAA,MACF;AAEA,UAAI,mBAAmB,OAAO,GAAG;AAC/B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,UACR,cAAc;AAAA,QAChB;AAAA,MACF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,QACd,cAAc;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,oBAOb,SACA,OACA,OACA,SACA,UACA,cACA,gBACqB;AACrB,QAAM,iBAAiB,2BAA2B;AAClD,QAAM,UAAU,cAAc;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,YAAY,QAAQ;AAAA,IACpB;AAAA,IACA;AAAA,EACF,CAAC;AAED,MAAI;AACF,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB;AAAA,MACF;AAAA,MACA,MACE,SAAS,SAAS;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACL;AAEA,YAAI,6BAAa,MAAM,KAAK,CAAC,mBAAmB,OAAO,GAAG;AACxD,UAAI,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,KAAK,CAAC,OAAO,WAAW;AAClE,eAAO,YAAY,QAAQ;AAAA,MAC7B;AACA,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB;AACxB,UAAM,aAAa;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IACV;AAQA,UAAM,SAAS,QAAQ,WAAW,SAC9B,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AACjC,UAAM,QAAQ,QAAQ,WAAW,QAC7B,MAAM,QAAQ,UAAU,MAAM,UAAU,IACxC,aAAa,iBAAiB,QAAQ,UAAU,MAAM;AAC1D,UAAM,UAAU,QAAQ,WAAW,UAC/B,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C,eAAe,iBAAiB,UAAU,QAAQ,KAAK;AAE3D,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS,QAAQ,WAAW,UACxB,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C;AAAA,MACJ,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,QAAQ,WAAW,SACvB,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AAAA,IACnC;AAAA,EACF,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,QAAW,QAAQ,UAAU,MAAM;AAC9D,UAAM,MAAM;AAAA,MACV,SAAS,eAAe,QAAW,UAAU,QAAW,KAAK;AAAA,MAC7D,QAAQ;AAAA,MACR;AAAA,MACA,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,KAAC,+BAAe,KAAK,CAAC;AAAA,IAChC;AAEA,cAAM,wCAAwB,OAAO,GAAG;AAAA,EAC1C;AACF;AAEA,eAAe,aAOb,SACA,MACA;AACA,MAAI,QAAQ,UAAU,QAAW;AAC/B,WAAO,QAAQ;AAAA,EACjB;AAEA,MAAI,QAAQ,aAAa;AACvB,WAAO,QAAQ,YAAY,IAAI;AAAA,EACjC;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAMP,SAA4E;AAC5E,SAAO;AAAA,IACL,QAAQ,WAAW,UACjB,QAAQ,WAAW,WACnB,QAAQ,WAAW,SACnB,QAAQ,WAAW,WACnB,QAAQ,WAAW;AAAA,EACvB;AACF;AAEA,SAAS,4BACP,OACyC;AACzC,MAAI;AACJ,QAAM,iBAA8D,CAAC;AACrE,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,mBAAmB,kBAAqC,SAAS;AACvE,QAAI,iBAAiB,UAAa,qBAAqB,QAAW;AAChE,qBAAe;AAAA,IACjB;AAEA,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,qBAAe,KAAK,WAAW;AAAA,IACjC;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,IACvC,GAAI,eAAe,SAAS,IAAI,EAAE,eAAe,IAAI,CAAC;AAAA,EACxD;AACF;AAEA,SAAS,uBAAuB,OAA0B;AACxD,QAAM,QAAQ,cAAc,KAAK;AACjC,QAAM,aAAuB,CAAC;AAC9B,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,QAAQ,OAAO;AACxB,oBAAgB,YAAY,MAAM,IAAI;AACtC,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,OAAO,CAAC;AAClE,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,cAAc,CAAC;AAAA,EAC3E;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA0B;AAC/C,SAAO,CAAC,SAAS,KAAK,CAAC,EACpB,OAAO,SAAS,kBAAkB,OAAO,OAAO,CAAC,CAAC,EAClD,OAAO,CAAC,UAA2B,UAAU,MAAS;AAC3D;AAEA,SAAS,gBACP,YACA,MACA,OACA;AACA,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,KAAK,IAAI,KAAK,GAAG;AAC1D;AAAA,EACF;AAEA,OAAK,IAAI,KAAK;AACd,aAAW,KAAK,KAAK;AACvB;AAEA,SAAS,SAAS,OAAoC;AACpD,SAAO,SAAS,OAAO,UAAU,WAAW,QAAQ;AACtD;AAEA,SAAS,kBAAkB,OAAgB,KAAsB;AAC/D,SAAO,SAAS,OAAO,UAAU,WAC5B,MAAkC,GAAG,IACtC;AACN;AAEA,SAAS,kBACP,OACoD;AACpD,QAAM,YACJ,kBAAkB,OAAO,OAAO,KAAK,kBAAkB,OAAO,SAAS;AAEzE,SAAO,cAAc,SAAS,IACzB,YACD;AACN;AAEA,SAAS,mBACP,OACkD;AAClD,QAAM,YAAY,kBAAkB,OAAO,OAAO;AAClD,MAAI,iBAAiB,SAAS,GAAG;AAC/B,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEA,eAAe,SAOb,SACA,MAC+B;AAC/B,MAAI,QAAQ,KAAK;AACf,WAAO,QAAQ,IAAI,IAAI;AAAA,EACzB;AAEA,MAAI,iBAAqD,KAAK,KAAK,GAAG;AACpE,WAAO,KAAK,MAAM,IAAI,KAAK,OAAO,KAAK,OAAO;AAAA,EAChD;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAOP,SAOA;AACA,SAAO,QAAQ,UAAU;AAC3B;AAEA,SAAS,iBAMP,OACgE;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,SACE,SAAS,SAAS,OAAQ,MAA4B,QAAQ;AAElE;AAEA,SAAS,cAAc,OAAsC;AAC3D,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AAC/D,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,OAAO,OAAO,KAAK;AACjC,SACE,MAAM,SAAS,KACf,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,SAAS,iBAAiB,OAA4C;AACpE,SACE,MAAM,QAAQ,KAAK,KACnB,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,UAAU,QACV,OAAQ,KAA4B,SAAS,YAC7C,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,eAAe,2BAKb,OACA,UACA,MAQA,UACA;AACA,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO,SAAS;AAAA,EAClB;AAEA,QAAM,qBAAqB,oBAAI,IAG7B;AACF,QAAM,iBAAiB,oBAAI,IAA+C;AAE1E,QAAM,YAAY,CAAC,SAA6C;AAC9D,QAAI,mBAAmB,IAAI,IAAI,GAAG;AAChC;AAAA,IACF;AAEA,UAAM,kBAAkB,2BAA2B,KAAK,OAAO;AAC/D,uBAAmB,IAAI,MAAM,eAAe;AAC5C,UAAM,sBAA4D,OAChE,YACA,YACG;AACH,YAAM,YAAY,oBAAI,KAAK;AAC3B,YAAM,cAAc;AAAA,QAClB,OAAO,KAAK;AAAA,QACZ,UAAU,KAAK,QAAQ;AAAA,QACvB,QAAQ,KAAK,QAAQ;AAAA,QACrB,aAAa,KAAK,QAAQ;AAAA,MAC5B;AACA,YAAM,kBAAkB;AAAA,QACtB,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAEA,UAAI;AACF,cAAM,YAAY,MAAM,4BAA4B;AAAA,UAClD,UAAU,KAAK;AAAA,UACf;AAAA,UACA,SAAS;AAAA,UACT,QAAQ,KAAK,aAAa,KAAK,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AACD,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,QAAQ,UAAU;AAAA,UAClB,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,cAAU,uCAAwB,UAAU,MAAM;AAAA,QACpD;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,SAAS,UAAU;AAAA,UACnB,UAAU;AAAA,YACR,MAAM,KAAK;AAAA,UACb;AAAA,QACF,CAAC;AACD,eAAO,UAAU;AAAA,MACnB,SAAS,OAAO;AACd,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,OAAO,uBAAuB,KAAK;AAAA,UACnC,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,cAAU,2CAAwB,0CAA2B,KAAK,CAAC;AAAA,QACrE;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,cAAM;AAAA,MACR,UAAE;AACA,wBAAgB;AAAA,MAClB;AAAA,IACF;AACA,wBAAoB,uBAAuB,IAAI;AAC/C,SAAK,UAAU;AAAA,EACjB;AAEA,QAAM,gBAAgB,CACpB,iBACG;AACH,eAAW,WAAW,cAAc;AAClC,iBAAW,QAAQ,SAAS;AAC1B,kBAAU,IAAI;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,gBAAc,QAAQ;AAEtB,aAAW,UAAU,qBAAqB,KAAK,GAAG;AAChD,UAAM,gBAAgB,OAAO;AAC7B,mBAAe,IAAI,QAAQ,aAAa;AACxC,WAAO,QAAQ,SAAS,gBAAuC,WAAW;AACxE,YAAM,cAAc,cAAc,MAAM,MAAM,SAAS;AAEvD,UAAI,cAAc,WAAW,GAAG;AAC9B,eAAO,YAAY,QAAQ,MAAM;AAC/B;AAAA,YACE,8BAAiD,KAAK;AAAA,UACxD;AAAA,QACF,CAAC;AAAA,MACH;AAEA,oBAAc,8BAAiD,KAAK,CAAC;AACrE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI;AACF,WAAO,MAAM,SAAS;AAAA,EACxB,UAAE;AACA,eAAW,CAAC,QAAQ,aAAa,KAAK,gBAAgB;AACpD,aAAO,QAAQ;AAAA,IACjB;AAEA,eAAW,CAAC,MAAM,eAAe,KAAK,oBAAoB;AACxD,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AACF;AAMA,SAAS,qBAAqB,OAAmC;AAC/D,SAAO,cAAc,KAAK,EAAE,OAAO,iBAAiB;AACtD;AAEA,SAAS,kBAAkB,OAAyC;AAClE,SAAO,WAAW,SAAS,OAAO,MAAM,UAAU;AACpD;AAEA,SAAS,8BAGP,OAA6D;AAC7D,QAAM,WAAwD,CAAC;AAC/D,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,eAAS,KAAK,WAAW;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA2C;AAChE,SAAO;AAAA,IACL,SAAS,OAAQ,MAA6B,SAAS;AAAA,EACzD;AACF;AAEA,SAAS,uBACP,OACsC;AACtC,QAAM,iBAAa,+BAAe,KAAK;AACvC,QAAM,EAAE,SAAS,MAAM,GAAG,QAAQ,IAAI;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,SAAS,OAAO,YAAY,WAAW,UAAU,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAAA,EAC7C;AACF;AAEA,SAAS,2BACP,SACA;AACA,QAAM,gBAAgB;AACtB,SAAO,cAAc,uBAAuB,KAAK;AACnD;AAEA,SAAS,6BAAmD;AAC1D,SAAO;AAAA,IACL,uBAAuB,oBAAI,IAAI;AAAA,EACjC;AACF;AAEA,SAAS,yBACP,OACA,UACA;AACA,QAAM,sBAAsB;AAAA,IAC1B;AAAA,KACC,MAAM,sBAAsB,IAAI,QAAQ,KAAK,KAAK;AAAA,EACrD;AAEA,SAAO,MAAM;AACX,UAAM,aAAa,MAAM,sBAAsB,IAAI,QAAQ,KAAK,KAAK;AACrE,QAAI,aAAa,GAAG;AAClB,YAAM,sBAAsB,OAAO,QAAQ;AAC3C;AAAA,IACF;AAEA,UAAM,sBAAsB,IAAI,UAAU,SAAS;AAAA,EACrD;AACF;AAEA,SAAS,6BACP,OACA,UACA;AACA,UAAQ,MAAM,sBAAsB,IAAI,QAAQ,KAAK,KAAK;AAC5D;AAEA,eAAe,4BAGb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAOG;AACD,MAAI,aAAa;AACjB,MAAI;AAEJ,QAAM,YAAY,UAAM,iCAAkB;AAAA,IACxC,UAAU,2BAA2B,QAAQ;AAAA,IAC7C;AAAA,IACA;AAAA,IACA,SAAS,OAAO,aAAa;AAC3B,mBAAa;AACb,mBAAa,MAAM,QAAQ,YAAY,QAAQ;AAC/C,aAAO,+BAA+B,UAAU;AAAA,IAClD;AAAA,IACA;AAAA,EACF,CAAC;AAED,MAAI,YAAY;AACd,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,kBAAkB,0BAA0B,UAAU;AAAA,MACtD,QAAQ,UAAU;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG,8BAA8B,UAAU,MAAM;AAAA,IACjD,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,2BAA2B,UAAkB;AACpD,SAAO,GAAG,QAAQ;AACpB;AAEA,SAAS,cAIP;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GASE;AACA,QAAM,YAA8B,CAAC;AACrC,QAAM,YAA2B;AAAA,IAC/B,SAAS,CAAC,YAAY;AACpB,eAAS,KAAK,OAAO;AAAA,IACvB;AAAA,IACA,QAAQ,CAAC,SAAS,aAAa;AAC7B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,SAAS,aAAa;AAC3B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,WAAW,CAAC,SAAS,aAAa;AAChC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,MAAM,SAAS,aAAa;AACjC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA,UAAU;AAAA,UACR;AAAA,UACA,GAAI,YAAY,CAAC;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,eAAe,OAAO;AAAA,IAC1B,OAAO,QAAQ,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM;AAAA,MACpD;AAAA,MACA,OAAO,SAAoC;AACzC,cAAM,YAAY,oBAAI,KAAK;AAC3B,cAAM,6BAA6B;AAAA,UACjC;AAAA,UACA;AAAA,QACF;AACA,cAAM,cAAc;AAAA,UAClB;AAAA,UACA,UAAU,QAAQ;AAAA,UAClB,QAAQ,QAAQ;AAAA,UAChB,aAAa,QAAQ;AAAA,QACvB;AAEA,YAAI;AACF,gBAAM,YAAY,MAAM,sBAAsB;AAAA,YAC5C;AAAA,YACA;AAAA,YACA,QAAQ,6BACJ,SACA,aAAa,QAAQ;AAAA,YACzB;AAAA,YACA,SAAS;AAAA,UACX,CAAC;AACD,gBAAM,aAAa,oBAAI,KAAK;AAE5B,cAAI,4BAA4B;AAC9B,mBAAO,UAAU;AAAA,UACnB;AAEA,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,QAAQ,UAAU;AAAA,YAClB,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,cAAU,uCAAwB,UAAU,MAAM;AAAA,UACpD;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,SAAS,UAAU;AAAA,YACnB,UAAU;AAAA,cACR,MAAM;AAAA,YACR;AAAA,UACF,CAAC;AACD,iBAAO,UAAU;AAAA,QACnB,SAAS,OAAO;AACd,gBAAM,aAAa,oBAAI,KAAK;AAC5B,cAAI,4BAA4B;AAC9B,kBAAM;AAAA,UACR;AAEA,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,OAAO,uBAAuB,KAAK;AAAA,YACnC,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,cAAU;AAAA,kBACR,0CAA2B,KAAK;AAAA,YAClC;AAAA,UACF;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,QAAQ,QAAQ;AAAA,IAChB;AAAA,EACF;AACF;AAEA,SAAS,cAAc,QAAwC;AAC7D,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,eAAO,4BAAY,MAAM;AAAA,EAC3B;AAEA,QAAM,aAAa;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,aAAW,OAAO,YAAY;AAC5B,UAAM,QAAS,OAAmC,GAAG;AACrD,UAAM,iBAAa,4BAAY,KAAK;AACpC,QAAI,eAAe,QAAW;AAC5B,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,QAAwC;AACnE,QAAM,UACJ,UAAU,OAAO,WAAW,eACxB,4BAAa,OAAiC,OAAO,IACrD;AACN,MAAI,YAAY,QAAW;AACzB,WAAO;AAAA,EACT;AAEA,aACE,4BAAY,MAAM,MAAM,WAAW,SAAY,SAAY,OAAO,MAAM;AAE5E;AAEA,SAAS,0BAA0B,QAA4B;AAC7D,SAAO,oBAAoB,MAAM,KAAK;AACxC;AAoBA,SAAS,+BACP,QAC0B;AAC1B,QAAM,mBAAmB,0BAA0B,MAAM;AAEzD,SAAO;AAAA,IACL,eAAe;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,iBAAa,4BAAY,MAAM,KAAK;AAAA,IACpC;AAAA,EACF;AACF;AAEA,SAAS,8BAA8B,QAAmB;AACxD,MAAI,2BAA2B,MAAM,GAAG;AACtC,WAAO;AAAA,MACL,QAAQ,OAAO;AAAA,MACf,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,iCAAiC,MAAM,GAAG;AAC5C,WAAO;AAAA,MACL,QAAQ,OAAO,eAAe,OAAO;AAAA,MACrC,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,kBAAkB,0BAA0B,MAAM;AAAA,EACpD;AACF;AAEA,SAAS,2BACP,OACmC;AACnC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,iBAAiB,SACjB,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,yBACP,OACoD;AACpD,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,iCACP,OACyC;AACzC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,+BACP,OAC0D;AAC1D,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,aAAa,QAAiB,eAAqC;AAC1E,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,WAAO,gBAAgB,IAAI,EAAE,WAAW,cAAc,IAAI,CAAC;AAAA,EAC7D;AAEA,QAAM,aACH,OAAmC,SACnC,OAAmC;AAEtC,QAAM,QACJ,cAAc,OAAO,eAAe,WAC/B,EAAE,GAAI,WAAuC,IAC9C,CAAC;AAEP,MAAI,MAAM,cAAc,UAAa,gBAAgB,GAAG;AACtD,UAAM,YAAY;AAAA,EACpB;AAEA,SAAO;AACT;AAEA,SAAS,eACP,QACA,UACA,QACA,OACmB;AACnB,UACE;AAAA,IACG,QAAgD;AAAA,EACnD,GACA;AACA,WAAQ,OAA0C;AAAA,EACpD;AAEA,UACE,oCAAqB,QAAgD,KAAK,GAC1E;AACA,WAAQ,OAAwC;AAAA,EAClD;AAEA,QAAM,kBAAkB,CAAC,GAAG,QAAQ;AACpC,MACE,WAAW,UACX,CAAC,gBAAgB;AAAA,IACf,CAAC,YACC,QAAQ,SAAS,eAAe,QAAQ,YAAY;AAAA,EACxD,GACA;AACA,oBAAgB,KAAK;AAAA,MACnB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,UAAU;AAAA,IACV,YAAY,OAAO,WAAW,WAAW,SAAS;AAAA,IAClD,UACI,QAAgD,YAEhC,MAAM;AAAA,IAC1B,OACI,QAAgD,SAEhC,MAAM;AAAA,EAC5B;AACF;AAEA,SAAS,cAAc,QAAmD;AACxE,aAAO,wCAAwB,MAAM;AACvC;AAEA,eAAe,sBAKb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAMG;AACD,aAAO,iCAIL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,CAAC,UAAU,gBAClB,KAAK,QAAQ,UAAmB,WAAW;AAAA,IAC7C;AAAA,EACF,CAAC;AACH;","names":[]}

package/dist/index.mjs

@@ -19,7 +19,10 @@ function piAiHarness(options) {
     name: options.name ?? "pi-ai",
     prompt: options.prompt,
     run: async (input, context) => {
-      const agent = await resolveAgent(options);
+      const agent = await resolveAgent(options, {
+        input,
+        context
+      });
       const messages = [
         {
           role: "user",
@@ -53,10 +56,13 @@ function piAiHarness(options) {
   };
 }
 async function executePiHarnessRun(options, agent, input, context, messages, runtimeTools, nativeToolsets) {
+  const executionState = createPiToolExecutionState();
   const runtime = createRuntime({
     input,
     context,
     tools: runtimeTools,
+    toolReplay: options.toolReplay,
+    executionState,
     messages
   });
   try {
@@ -67,7 +73,9 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
         input,
         context,
         messages,
-        toolCalls: runtime.toolCalls
+        toolCalls: runtime.toolCalls,
+        toolReplay: options.toolReplay,
+        executionState
       },
       () => runAgent(options, {
         agent,
@@ -113,12 +121,12 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
     throw attachHarnessRunToError(error, run);
   }
 }
-async function resolveAgent(options) {
+async function resolveAgent(options, args) {
   if (options.agent !== void 0) {
     return options.agent;
   }
   if (options.createAgent) {
-    return options.createAgent();
+    return options.createAgent(args);
   }
   throw new Error(
     "piAiHarness requires either an agent instance or a createAgent() function."
@@ -245,12 +253,16 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
         signal: args.context.signal,
         setArtifact: args.context.setArtifact
       };
+      const leaveNativeTool = enterNativeToolExecution(
+        args.executionState,
+        tool.name
+      );
       try {
         const execution = await executeNativeToolWithReplay({
           toolName: tool.name,
           toolCallId,
           execute: originalExecute,
-          replay: tool.replay,
+          replay: args.toolReplay?.[tool.name],
           args: rawArgs,
           context: toolContext
         });
@@ -294,6 +306,8 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
           toolCalls: [call]
         });
         throw error;
+      } finally {
+        leaveNativeTool();
       }
     };
     instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute;
@@ -370,6 +384,28 @@ function getNativeToolExecuteOrigin(execute) {
   const nativeExecute = execute;
   return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;
 }
+function createPiToolExecutionState() {
+  return {
+    activeNativeToolNames: /* @__PURE__ */ new Map()
+  };
+}
+function enterNativeToolExecution(state, toolName) {
+  state.activeNativeToolNames.set(
+    toolName,
+    (state.activeNativeToolNames.get(toolName) ?? 0) + 1
+  );
+  return () => {
+    const nextCount = (state.activeNativeToolNames.get(toolName) ?? 1) - 1;
+    if (nextCount <= 0) {
+      state.activeNativeToolNames.delete(toolName);
+      return;
+    }
+    state.activeNativeToolNames.set(toolName, nextCount);
+  };
+}
+function hasActiveNativeToolExecution(state, toolName) {
+  return (state.activeNativeToolNames.get(toolName) ?? 0) > 0;
+}
 async function executeNativeToolWithReplay({
   toolName,
   toolCallId,
@@ -381,7 +417,7 @@ async function executeNativeToolWithReplay({
   let didExecute = false;
   let liveResult;
   const execution = await executeWithReplay({
-    toolName,
+    toolName: createNativeReplayToolName(toolName),
     args,
     context,
     execute: async (toolArgs) => {
@@ -403,10 +439,15 @@ async function executeNativeToolWithReplay({
     replay: execution.replay
   };
 }
+function createNativeReplayToolName(toolName) {
+  return `${toolName}.native`;
+}
 function createRuntime({
   input,
   context,
   tools,
+  toolReplay,
+  executionState,
   messages
 }) {
   const toolCalls = [];
@@ -451,6 +492,10 @@ function createRuntime({
       toolName,
       async (args) => {
         const startedAt = /* @__PURE__ */ new Date();
+        const isNativeImplementationCall = hasActiveNativeToolExecution(
+          executionState,
+          toolName
+        );
         const toolContext = {
           input,
           metadata: context.metadata,
@@ -461,10 +506,14 @@ function createRuntime({
           const execution = await executeToolWithReplay({
             toolName,
             tool,
+            replay: isNativeImplementationCall ? void 0 : toolReplay?.[toolName],
             args,
             context: toolContext
           });
           const finishedAt = /* @__PURE__ */ new Date();
+          if (isNativeImplementationCall) {
+            return execution.result;
+          }
           const call = {
             name: toolName,
             arguments: args,
@@ -489,6 +538,9 @@ function createRuntime({
           return execution.result;
         } catch (error) {
           const finishedAt = /* @__PURE__ */ new Date();
+          if (isNativeImplementationCall) {
+            throw error;
+          }
           const call = {
             name: toolName,
             arguments: args,
@@ -641,6 +693,7 @@ function resolveErrors(result) {
 async function executeToolWithReplay({
   toolName,
   tool,
+  replay,
   args,
   context
 }) {
@@ -648,8 +701,8 @@ async function executeToolWithReplay({
     toolName,
     args,
     context,
-    execute: tool.execute,
-    replay: tool.replay
+    execute: (toolArgs, toolContext) => tool.execute(toolArgs, toolContext),
+    replay
   });
 }
 export {

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type {\n  Harness,\n  HarnessContext,\n  HarnessMetadata,\n  HarnessPrompt,\n  HarnessRun,\n  JsonValue,\n  NormalizedMessage,\n  NormalizedSession,\n  TimingSummary,\n  ToolCallRecord,\n  UsageSummary,\n} from \"vitest-evals/harness\";\nimport {\n  attachHarnessRunToError,\n  isHarnessRun,\n  isNormalizedSession,\n  normalizeContent,\n  resolveHarnessRunErrors,\n  serializeError,\n  toJsonValue,\n} from \"vitest-evals/harness\";\nimport {\n  executeWithReplay,\n  getReplayMetadataFromError,\n  normalizeReplayMetadata,\n} from \"vitest-evals/replay\";\nimport type {\n  ReplayMode,\n  ToolRecording,\n  ToolReplayConfig,\n} from \"vitest-evals/replay\";\n\ntype MaybePromise<T> = T | Promise<T>;\ntype AnyPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;\ntype InferredPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<\n  string,\n  PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>\n>;\n\ntype PiAgentToolLike<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  replay?:\n    | boolean\n    | PiAiToolReplayConfig<\n        Record<string, JsonValue>,\n        JsonValue,\n        TInput,\n        TMetadata\n      >;\n  execute: (toolCallId: string, args: Record<string, JsonValue>) => unknown;\n};\n\nconst ORIGINAL_NATIVE_EXECUTE = Symbol(\"vitest-evals.originalNativeExecute\");\n\ntype NativeToolExecute<\n  TInput,\n  TMetadata extends HarnessMetadata,\n> = PiAgentToolLike<TInput, TMetadata>[\"execute\"] & {\n  [ORIGINAL_NATIVE_EXECUTE]?: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n};\n\nexport type PiAiReplayMode = ReplayMode;\n\nexport interface PiAiEventSink {\n  message: (message: NormalizedMessage) => void;\n  system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  user: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  tool: (\n    name: string,\n    content: JsonValue,\n    metadata?: Record<string, JsonValue>,\n  ) => void;\n}\n\nexport interface PiAiToolContext<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  input: TInput;\n  metadata: HarnessContext<TMetadata>[\"metadata\"];\n  signal?: AbortSignal;\n  setArtifact: HarnessContext<TMetadata>[\"setArtifact\"];\n}\n\nexport type PiAiToolRecording<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n> = ToolRecording<TArgs, TResult>;\n\nexport type PiAiToolReplayConfig<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;\n\nexport interface PiAiToolDefinition<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  description?: string;\n  replay?: boolean | PiAiToolReplayConfig<TArgs, TResult, TInput, TMetadata>;\n  execute: (\n    args: TArgs,\n    context: PiAiToolContext<TInput, TMetadata>,\n  ) => MaybePromise<TResult>;\n}\n\nexport type PiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = AnyPiAiToolset<TInput, TMetadata>;\n\ntype ToolArgs<TTool> = TTool extends PiAiToolDefinition<\n  infer TArgs,\n  infer _TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TArgs\n  : never;\n\ntype ToolResult<TTool> = TTool extends PiAiToolDefinition<\n  infer _TArgs,\n  infer TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TResult\n  : never;\n\nexport type PiAiRuntime<\n  TTools extends PiAiToolset<TInput, TMetadata>,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  tools: {\n    [K in keyof TTools]: (\n      args: ToolArgs<TTools[K]>,\n    ) => Promise<ToolResult<TTools[K]>>;\n  };\n  events: PiAiEventSink;\n  signal?: AbortSignal;\n};\n\nexport interface PiAiHarnessRunArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> {\n  agent: TAgent;\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  runtime: PiAiRuntime<TTools, TInput, TMetadata>;\n}\n\nexport interface PiAiHarnessResultArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {\n  result: TResult;\n}\n\ninterface PiAiHarnessBaseOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  agent?: TAgent;\n  createAgent?: () => MaybePromise<TAgent>;\n  normalize?: PiAiHarnessNormalizeOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >;\n  prompt: HarnessPrompt;\n  name?: string;\n}\n\nexport interface PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> {\n  tools: TTools;\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport interface PiAiHarnessInferredToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n> extends PiAiHarnessBaseOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    InferredPiAiToolset<TInput, TMetadata>\n  > {\n  tools?: undefined;\n  run?: (\n    args: PiAiHarnessRunArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      InferredPiAiToolset<TInput, TMetadata>\n    >,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport type PiAiHarnessOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> =\n  | PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>\n  | PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>;\n\ntype PiAiHarnessRunOptions<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> & {\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\ntype PiAiRunnableAgent<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = {\n  run: (\n    input: TInput,\n    runtime: PiAiRuntime<TTools, TInput, TMetadata>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\nexport interface PiAiHarnessNormalizeOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  session?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<NormalizedSession>;\n  output?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<JsonValue | undefined>;\n  usage?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<UsageSummary>;\n  timings?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<TimingSummary | undefined>;\n  errors?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<Array<Record<string, JsonValue>>>;\n}\n\ntype InferredToolSurfaces<TInput, TMetadata extends HarnessMetadata> = {\n  runtimeTools?: InferredPiAiToolset<TInput, TMetadata>;\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>;\n};\n\n/** Adapts a Pi agent runtime into a normalized vitest-evals harness. */\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessWithToolsOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n>(\n  options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): Harness<TInput, TMetadata> {\n  return {\n    name: options.name ?? \"pi-ai\",\n    prompt: options.prompt,\n    run: async (input, context) => {\n      const agent = await resolveAgent(options);\n      const messages: NormalizedMessage[] = [\n        {\n          role: \"user\",\n          content: normalizeContent(input),\n        },\n      ];\n      const inferredTools = resolveInferredToolSurfaces<TInput, TMetadata>(\n        agent,\n      );\n\n      if (hasExplicitToolset(options)) {\n        return executePiHarnessRun(\n          options,\n          agent,\n          input,\n          context,\n          messages,\n          options.tools,\n          inferredTools.nativeToolsets,\n        );\n      }\n\n      return executePiHarnessRun(\n        options,\n        agent,\n        input,\n        context,\n        messages,\n        inferredTools.runtimeTools,\n        inferredTools.nativeToolsets,\n      );\n    },\n  };\n}\n\nasync function executePiHarnessRun<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  agent: TAgent,\n  input: TInput,\n  context: HarnessContext<TMetadata>,\n  messages: NormalizedMessage[],\n  runtimeTools: TTools | undefined,\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n): Promise<HarnessRun> {\n  const runtime = createRuntime({\n    input,\n    context,\n    tools: runtimeTools,\n    messages,\n  });\n\n  try {\n    const result = await withInstrumentedAgentTools(\n      agent,\n      nativeToolsets,\n      {\n        input,\n        context,\n        messages,\n        toolCalls: runtime.toolCalls,\n      },\n      () =>\n        runAgent(options, {\n          agent,\n          input,\n          context,\n          runtime,\n        }),\n    );\n\n    if (isHarnessRun(result) && !hasResultOverrides(options)) {\n      if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {\n        result.artifacts = context.artifacts;\n      }\n      return result;\n    }\n\n    const normalizeResult = result as TResult;\n    const resultArgs = {\n      agent,\n      input,\n      context,\n      runtime,\n      result: normalizeResult,\n    } satisfies PiAiHarnessResultArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      TResult,\n      TTools\n    >;\n\n    const output = options.normalize?.output\n      ? await options.normalize.output(resultArgs)\n      : resolveOutput(normalizeResult);\n    const usage = options.normalize?.usage\n      ? await options.normalize.usage(resultArgs)\n      : resolveUsage(normalizeResult, runtime.toolCalls.length);\n    const session = options.normalize?.session\n      ? await options.normalize.session(resultArgs)\n      : resolveSession(normalizeResult, messages, output, usage);\n\n    return {\n      session,\n      output,\n      usage,\n      timings: options.normalize?.timings\n        ? await options.normalize.timings(resultArgs)\n        : undefined,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: options.normalize?.errors\n        ? await options.normalize.errors(resultArgs)\n        : resolveErrors(normalizeResult),\n    };\n  } catch (error) {\n    const usage = resolveUsage(undefined, runtime.toolCalls.length);\n    const run = {\n      session: resolveSession(undefined, messages, undefined, usage),\n      output: undefined,\n      usage,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: [serializeError(error)],\n    } satisfies HarnessRun;\n\n    throw attachHarnessRunToError(error, run);\n  }\n}\n\nasync function resolveAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>) {\n  if (options.agent !== undefined) {\n    return options.agent;\n  }\n\n  if (options.createAgent) {\n    return options.createAgent();\n  }\n\n  throw new Error(\n    \"piAiHarness requires either an agent instance or a createAgent() function.\",\n  );\n}\n\nfunction hasResultOverrides<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>) {\n  return Boolean(\n    options.normalize?.output ??\n      options.normalize?.session ??\n      options.normalize?.usage ??\n      options.normalize?.timings ??\n      options.normalize?.errors,\n  );\n}\n\nfunction resolveInferredToolSurfaces<TInput, TMetadata extends HarnessMetadata>(\n  agent: unknown,\n): InferredToolSurfaces<TInput, TMetadata> {\n  let runtimeTools: InferredPiAiToolset<TInput, TMetadata> | undefined;\n  const nativeToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nextRuntimeTools = getRuntimeToolset<TInput, TMetadata>(candidate);\n    if (runtimeTools === undefined && nextRuntimeTools !== undefined) {\n      runtimeTools = nextRuntimeTools;\n    }\n\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      nativeToolsets.push(nativeTools);\n    }\n  }\n\n  return {\n    ...(runtimeTools ? { runtimeTools } : {}),\n    ...(nativeToolsets.length > 0 ? { nativeToolsets } : {}),\n  };\n}\n\nfunction getAgentToolCandidates(agent: unknown): object[] {\n  const roots = getAgentRoots(agent);\n  const candidates: object[] = [];\n  const seen = new Set<object>();\n\n  for (const root of roots) {\n    addUniqueObject(candidates, seen, root);\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"state\"));\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"initialState\"));\n  }\n\n  return candidates;\n}\n\nfunction getAgentRoots(agent: unknown): object[] {\n  return [asObject(agent)]\n    .concat(asObject(getObjectProperty(agent, \"agent\")))\n    .filter((value): value is object => value !== undefined);\n}\n\nfunction addUniqueObject(\n  candidates: object[],\n  seen: Set<object>,\n  value: unknown,\n) {\n  if (!value || typeof value !== \"object\" || seen.has(value)) {\n    return;\n  }\n\n  seen.add(value);\n  candidates.push(value);\n}\n\nfunction asObject(value: unknown): object | undefined {\n  return value && typeof value === \"object\" ? value : undefined;\n}\n\nfunction getObjectProperty(value: unknown, key: string): unknown {\n  return value && typeof value === \"object\"\n    ? (value as Record<string, unknown>)[key]\n    : undefined;\n}\n\nfunction getRuntimeToolset<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): InferredPiAiToolset<TInput, TMetadata> | undefined {\n  const candidate =\n    getObjectProperty(value, \"tools\") ?? getObjectProperty(value, \"toolset\");\n\n  return isPiAiToolset(candidate)\n    ? (candidate as InferredPiAiToolset<TInput, TMetadata>)\n    : undefined;\n}\n\nfunction getNativeToolArray<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): PiAgentToolLike<TInput, TMetadata>[] | undefined {\n  const candidate = getObjectProperty(value, \"tools\");\n  if (isAgentToolArray(candidate)) {\n    return candidate as PiAgentToolLike<TInput, TMetadata>[];\n  }\n\n  return undefined;\n}\n\nasync function runAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n): Promise<TResult | HarnessRun> {\n  if (options.run) {\n    return options.run(args);\n  }\n\n  if (hasPiAiRunMethod<TInput, TMetadata, TResult, TTools>(args.agent)) {\n    return args.agent.run(args.input, args.runtime);\n  }\n\n  throw new Error(\n    \"piAiHarness requires a run() function unless the provided agent exposes run(input, runtime).\",\n  );\n}\n\nfunction hasExplicitToolset<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): options is PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput,\n  TMetadata,\n  TResult,\n  TTools\n> {\n  return options.tools !== undefined;\n}\n\nfunction hasPiAiRunMethod<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  agent: unknown,\n): agent is PiAiRunnableAgent<TInput, TMetadata, TResult, TTools> {\n  if (!agent || typeof agent !== \"object\") {\n    return false;\n  }\n\n  return (\n    \"run\" in agent && typeof (agent as { run?: unknown }).run === \"function\"\n  );\n}\n\nfunction isPiAiToolset(value: unknown): value is PiAiToolset {\n  if (!value || typeof value !== \"object\" || Array.isArray(value)) {\n    return false;\n  }\n\n  const tools = Object.values(value);\n  return (\n    tools.length > 0 &&\n    tools.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nfunction isAgentToolArray(value: unknown): value is PiAgentToolLike[] {\n  return (\n    Array.isArray(value) &&\n    value.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"name\" in tool &&\n          typeof (tool as { name?: unknown }).name === \"string\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nasync function withInstrumentedAgentTools<\n  TResult,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(\n  agent: unknown,\n  toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> | undefined,\n  args: {\n    input: TInput;\n    context: HarnessContext<TMetadata>;\n    messages: NormalizedMessage[];\n    toolCalls: ToolCallRecord[];\n  },\n  callback: () => Promise<TResult>,\n) {\n  if (!toolsets || toolsets.length === 0) {\n    return callback();\n  }\n\n  const originalExecutions = new Map<\n    PiAgentToolLike<TInput, TMetadata>,\n    PiAgentToolLike<TInput, TMetadata>[\"execute\"]\n  >();\n  const originalResets = new Map<ResettableAgent, ResettableAgent[\"reset\"]>();\n\n  const patchTool = (tool: PiAgentToolLike<TInput, TMetadata>) => {\n    if (originalExecutions.has(tool)) {\n      return;\n    }\n\n    const originalExecute = getNativeToolExecuteOrigin(tool.execute);\n    originalExecutions.set(tool, originalExecute);\n    const instrumentedExecute: NativeToolExecute<TInput, TMetadata> = async (\n      toolCallId: string,\n      rawArgs: Record<string, JsonValue>,\n    ) => {\n      const startedAt = new Date();\n      const toolContext = {\n        input: args.input,\n        metadata: args.context.metadata,\n        signal: args.context.signal,\n        setArtifact: args.context.setArtifact,\n      } satisfies PiAiToolContext<TInput, TMetadata>;\n\n      try {\n        const execution = await executeNativeToolWithReplay({\n          toolName: tool.name,\n          toolCallId,\n          execute: originalExecute,\n          replay: tool.replay,\n          args: rawArgs,\n          context: toolContext,\n        });\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          result: execution.normalizedResult,\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(execution.replay),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        args.messages.push({\n          role: \"tool\",\n          content: execution.normalizedResult,\n          metadata: {\n            name: tool.name,\n          },\n        });\n        return execution.result;\n      } catch (error) {\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          error: serializeToolCallError(error),\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(getReplayMetadataFromError(error)),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        throw error;\n      }\n    };\n    instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute;\n    tool.execute = instrumentedExecute;\n  };\n\n  const patchToolsets = (\n    nextToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n  ) => {\n    for (const toolset of nextToolsets) {\n      for (const tool of toolset) {\n        patchTool(tool);\n      }\n    }\n  };\n\n  patchToolsets(toolsets);\n\n  for (const target of getAgentResetTargets(agent)) {\n    const originalReset = target.reset;\n    originalResets.set(target, originalReset);\n    target.reset = function patchedReset(this: ResettableAgent, ...resetArgs) {\n      const resetResult = originalReset.apply(this, resetArgs);\n\n      if (isPromiseLike(resetResult)) {\n        return resetResult.finally(() => {\n          patchToolsets(\n            resolveInferredNativeToolsets<TInput, TMetadata>(agent),\n          );\n        });\n      }\n\n      patchToolsets(resolveInferredNativeToolsets<TInput, TMetadata>(agent));\n      return resetResult;\n    };\n  }\n\n  try {\n    return await callback();\n  } finally {\n    for (const [target, originalReset] of originalResets) {\n      target.reset = originalReset;\n    }\n\n    for (const [tool, originalExecute] of originalExecutions) {\n      tool.execute = originalExecute;\n    }\n  }\n}\n\ntype ResettableAgent = {\n  reset: (...args: unknown[]) => unknown;\n};\n\nfunction getAgentResetTargets(agent: unknown): ResettableAgent[] {\n  return getAgentRoots(agent).filter(isResettableAgent);\n}\n\nfunction isResettableAgent(value: object): value is ResettableAgent {\n  return \"reset\" in value && typeof value.reset === \"function\";\n}\n\nfunction resolveInferredNativeToolsets<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(agent: unknown): Array<PiAgentToolLike<TInput, TMetadata>[]> {\n  const toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      toolsets.push(nativeTools);\n    }\n  }\n\n  return toolsets;\n}\n\nfunction isPromiseLike(value: unknown): value is Promise<unknown> {\n  return Boolean(\n    value && typeof (value as { then?: unknown }).then === \"function\",\n  );\n}\n\nfunction serializeToolCallError(\n  error: unknown,\n): NonNullable<ToolCallRecord[\"error\"]> {\n  const serialized = serializeError(error);\n  const { message, type, ...details } = serialized;\n\n  return {\n    ...details,\n    message: typeof message === \"string\" ? message : String(message),\n    ...(typeof type === \"string\" ? { type } : {}),\n  };\n}\n\nfunction getNativeToolExecuteOrigin<TInput, TMetadata extends HarnessMetadata>(\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"],\n) {\n  const nativeExecute = execute as NativeToolExecute<TInput, TMetadata>;\n  return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;\n}\n\nasync function executeNativeToolWithReplay<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  toolCallId,\n  execute,\n  replay,\n  args,\n  context,\n}: {\n  toolName: string;\n  toolCallId: string;\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n  replay: PiAgentToolLike<TInput, TMetadata>[\"replay\"];\n  args: Record<string, JsonValue>;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  let didExecute = false;\n  let liveResult: unknown;\n\n  const execution = await executeWithReplay({\n    toolName,\n    args,\n    context,\n    execute: async (toolArgs) => {\n      didExecute = true;\n      liveResult = await execute(toolCallId, toolArgs);\n      return createNativeToolReplayEnvelope(liveResult);\n    },\n    replay,\n  });\n\n  if (didExecute) {\n    return {\n      result: liveResult,\n      normalizedResult: normalizeReplayToolResult(liveResult),\n      replay: execution.replay,\n    };\n  }\n\n  return {\n    ...resolveNativeToolReplayResult(execution.result),\n    replay: execution.replay,\n  };\n}\n\nfunction createRuntime<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>({\n  input,\n  context,\n  tools,\n  messages,\n}: {\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  tools: TTools | undefined;\n  messages: NormalizedMessage[];\n}): PiAiRuntime<TTools, TInput, TMetadata> & {\n  toolCalls: ToolCallRecord[];\n} {\n  const toolCalls: ToolCallRecord[] = [];\n  const eventSink: PiAiEventSink = {\n    message: (message) => {\n      messages.push(message);\n    },\n    system: (content, metadata) => {\n      messages.push({\n        role: \"system\",\n        content,\n        metadata,\n      });\n    },\n    user: (content, metadata) => {\n      messages.push({\n        role: \"user\",\n        content,\n        metadata,\n      });\n    },\n    assistant: (content, metadata) => {\n      messages.push({\n        role: \"assistant\",\n        content,\n        metadata,\n      });\n    },\n    tool: (name, content, metadata) => {\n      messages.push({\n        role: \"tool\",\n        content,\n        metadata: {\n          name,\n          ...(metadata ?? {}),\n        },\n      });\n    },\n  };\n\n  const runtimeTools = Object.fromEntries(\n    Object.entries(tools ?? {}).map(([toolName, tool]) => [\n      toolName,\n      async (args: Record<string, JsonValue>) => {\n        const startedAt = new Date();\n        const toolContext = {\n          input,\n          metadata: context.metadata,\n          signal: context.signal,\n          setArtifact: context.setArtifact,\n        } satisfies PiAiToolContext<TInput, TMetadata>;\n\n        try {\n          const execution = await executeToolWithReplay({\n            toolName,\n            tool,\n            args,\n            context: toolContext,\n          });\n          const finishedAt = new Date();\n          const call = {\n            name: toolName,\n            arguments: args,\n            result: execution.result,\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(execution.replay),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          messages.push({\n            role: \"tool\",\n            content: execution.result,\n            metadata: {\n              name: toolName,\n            },\n          });\n          return execution.result;\n        } catch (error) {\n          const finishedAt = new Date();\n          const call = {\n            name: toolName,\n            arguments: args,\n            error: serializeToolCallError(error),\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(\n              getReplayMetadataFromError(error),\n            ),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          throw error;\n        }\n      },\n    ]),\n  ) as unknown as PiAiRuntime<TTools, TInput, TMetadata>[\"tools\"];\n\n  return {\n    tools: runtimeTools,\n    events: eventSink,\n    signal: context.signal,\n    toolCalls,\n  };\n}\n\nfunction resolveOutput(result: unknown): JsonValue | undefined {\n  if (!result || typeof result !== \"object\") {\n    return toJsonValue(result);\n  }\n\n  const candidates = [\n    \"output\",\n    \"decision\",\n    \"result\",\n    \"final\",\n  ] satisfies string[];\n\n  for (const key of candidates) {\n    const value = (result as Record<string, unknown>)[key];\n    const normalized = toJsonValue(value);\n    if (normalized !== undefined) {\n      return normalized;\n    }\n  }\n\n  return undefined;\n}\n\nfunction normalizeToolResult(result: unknown): JsonValue | undefined {\n  const details =\n    result && typeof result === \"object\"\n      ? toJsonValue((result as { details?: unknown }).details)\n      : undefined;\n  if (details !== undefined) {\n    return details;\n  }\n\n  return (\n    toJsonValue(result) ?? (result === undefined ? undefined : String(result))\n  );\n}\n\nfunction normalizeReplayToolResult(result: unknown): JsonValue {\n  return normalizeToolResult(result) ?? null;\n}\n\ntype NativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 2;\n  };\n  agentResult: JsonValue;\n  normalizedResult: JsonValue;\n};\n\ntype LegacyNativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 1;\n  };\n  normalizedResult: JsonValue;\n  agentResult?: JsonValue;\n};\n\nfunction createNativeToolReplayEnvelope(\n  result: unknown,\n): NativeToolReplayEnvelope {\n  const normalizedResult = normalizeReplayToolResult(result);\n\n  return {\n    __vitestEvals: {\n      kind: \"pi-ai-native-tool-result\",\n      version: 2,\n    },\n    agentResult: toJsonValue(result) ?? normalizedResult,\n    normalizedResult,\n  };\n}\n\nfunction resolveNativeToolReplayResult(result: JsonValue) {\n  if (isNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  if (isLegacyNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult ?? result.normalizedResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  return {\n    result,\n    normalizedResult: normalizeReplayToolResult(result),\n  };\n}\n\nfunction isNativeToolReplayEnvelope(\n  value: unknown,\n): value is NativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"agentResult\" in value &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isNativeToolReplayMarker(\n  value: unknown,\n): value is NativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 2,\n  );\n}\n\nfunction isLegacyNativeToolReplayEnvelope(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isLegacyNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isLegacyNativeToolReplayMarker(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 1,\n  );\n}\n\nfunction resolveUsage(result: unknown, toolCallCount: number): UsageSummary {\n  if (!result || typeof result !== \"object\") {\n    return toolCallCount > 0 ? { toolCalls: toolCallCount } : {};\n  }\n\n  const usageValue =\n    (result as Record<string, unknown>).usage ??\n    (result as Record<string, unknown>).metrics;\n\n  const usage =\n    usageValue && typeof usageValue === \"object\"\n      ? ({ ...(usageValue as Record<string, unknown>) } as UsageSummary)\n      : {};\n\n  if (usage.toolCalls === undefined && toolCallCount > 0) {\n    usage.toolCalls = toolCallCount;\n  }\n\n  return usage;\n}\n\nfunction resolveSession(\n  result: unknown,\n  messages: NormalizedMessage[],\n  output: JsonValue | undefined,\n  usage: UsageSummary,\n): NormalizedSession {\n  if (\n    isNormalizedSession(\n      (result as Record<string, unknown> | undefined)?.session,\n    )\n  ) {\n    return (result as { session: NormalizedSession }).session;\n  }\n\n  if (\n    isNormalizedSession((result as Record<string, unknown> | undefined)?.trace)\n  ) {\n    return (result as { trace: NormalizedSession }).trace;\n  }\n\n  const sessionMessages = [...messages];\n  if (\n    output !== undefined &&\n    !sessionMessages.some(\n      (message) =>\n        message.role === \"assistant\" && message.content !== undefined,\n    )\n  ) {\n    sessionMessages.push({\n      role: \"assistant\",\n      content: output,\n    });\n  }\n\n  return {\n    messages: sessionMessages,\n    outputText: typeof output === \"string\" ? output : undefined,\n    provider:\n      ((result as Record<string, unknown> | undefined)?.provider as\n        | string\n        | undefined) ?? usage.provider,\n    model:\n      ((result as Record<string, unknown> | undefined)?.model as\n        | string\n        | undefined) ?? usage.model,\n  };\n}\n\nfunction resolveErrors(result: unknown): Array<Record<string, JsonValue>> {\n  return resolveHarnessRunErrors(result);\n}\n\nasync function executeToolWithReplay<\n  TArgs extends Record<string, JsonValue>,\n  TResult extends JsonValue,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  tool,\n  args,\n  context,\n}: {\n  toolName: string;\n  tool: PiAiToolDefinition<TArgs, TResult, TInput, TMetadata>;\n  args: TArgs;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  return executeWithReplay({\n    toolName,\n    args,\n    context,\n    execute: tool.execute,\n    replay: tool.replay,\n  });\n}\n"],"mappings":";AAaA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAoCP,IAAM,0BAA0B,OAAO,oCAAoC;AAoRpE,SAAS,YAUd,SAC4B;AAC5B,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,QAAQ,QAAQ;AAAA,IAChB,KAAK,OAAO,OAAO,YAAY;AAC7B,YAAM,QAAQ,MAAM,aAAa,OAAO;AACxC,YAAM,WAAgC;AAAA,QACpC;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,KAAK;AAAA,QACjC;AAAA,MACF;AACA,YAAM,gBAAgB;AAAA,QACpB;AAAA,MACF;AAEA,UAAI,mBAAmB,OAAO,GAAG;AAC/B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,UACR,cAAc;AAAA,QAChB;AAAA,MACF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,QACd,cAAc;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,oBAOb,SACA,OACA,OACA,SACA,UACA,cACA,gBACqB;AACrB,QAAM,UAAU,cAAc;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP;AAAA,EACF,CAAC;AAED,MAAI;AACF,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,WAAW,QAAQ;AAAA,MACrB;AAAA,MACA,MACE,SAAS,SAAS;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACL;AAEA,QAAI,aAAa,MAAM,KAAK,CAAC,mBAAmB,OAAO,GAAG;AACxD,UAAI,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,KAAK,CAAC,OAAO,WAAW;AAClE,eAAO,YAAY,QAAQ;AAAA,MAC7B;AACA,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB;AACxB,UAAM,aAAa;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IACV;AAQA,UAAM,SAAS,QAAQ,WAAW,SAC9B,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AACjC,UAAM,QAAQ,QAAQ,WAAW,QAC7B,MAAM,QAAQ,UAAU,MAAM,UAAU,IACxC,aAAa,iBAAiB,QAAQ,UAAU,MAAM;AAC1D,UAAM,UAAU,QAAQ,WAAW,UAC/B,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C,eAAe,iBAAiB,UAAU,QAAQ,KAAK;AAE3D,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS,QAAQ,WAAW,UACxB,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C;AAAA,MACJ,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,QAAQ,WAAW,SACvB,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AAAA,IACnC;AAAA,EACF,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,QAAW,QAAQ,UAAU,MAAM;AAC9D,UAAM,MAAM;AAAA,MACV,SAAS,eAAe,QAAW,UAAU,QAAW,KAAK;AAAA,MAC7D,QAAQ;AAAA,MACR;AAAA,MACA,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,CAAC,eAAe,KAAK,CAAC;AAAA,IAChC;AAEA,UAAM,wBAAwB,OAAO,GAAG;AAAA,EAC1C;AACF;AAEA,eAAe,aAMb,SAAyE;AACzE,MAAI,QAAQ,UAAU,QAAW;AAC/B,WAAO,QAAQ;AAAA,EACjB;AAEA,MAAI,QAAQ,aAAa;AACvB,WAAO,QAAQ,YAAY;AAAA,EAC7B;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAMP,SAA4E;AAC5E,SAAO;AAAA,IACL,QAAQ,WAAW,UACjB,QAAQ,WAAW,WACnB,QAAQ,WAAW,SACnB,QAAQ,WAAW,WACnB,QAAQ,WAAW;AAAA,EACvB;AACF;AAEA,SAAS,4BACP,OACyC;AACzC,MAAI;AACJ,QAAM,iBAA8D,CAAC;AACrE,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,mBAAmB,kBAAqC,SAAS;AACvE,QAAI,iBAAiB,UAAa,qBAAqB,QAAW;AAChE,qBAAe;AAAA,IACjB;AAEA,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,qBAAe,KAAK,WAAW;AAAA,IACjC;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,IACvC,GAAI,eAAe,SAAS,IAAI,EAAE,eAAe,IAAI,CAAC;AAAA,EACxD;AACF;AAEA,SAAS,uBAAuB,OAA0B;AACxD,QAAM,QAAQ,cAAc,KAAK;AACjC,QAAM,aAAuB,CAAC;AAC9B,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,QAAQ,OAAO;AACxB,oBAAgB,YAAY,MAAM,IAAI;AACtC,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,OAAO,CAAC;AAClE,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,cAAc,CAAC;AAAA,EAC3E;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA0B;AAC/C,SAAO,CAAC,SAAS,KAAK,CAAC,EACpB,OAAO,SAAS,kBAAkB,OAAO,OAAO,CAAC,CAAC,EAClD,OAAO,CAAC,UAA2B,UAAU,MAAS;AAC3D;AAEA,SAAS,gBACP,YACA,MACA,OACA;AACA,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,KAAK,IAAI,KAAK,GAAG;AAC1D;AAAA,EACF;AAEA,OAAK,IAAI,KAAK;AACd,aAAW,KAAK,KAAK;AACvB;AAEA,SAAS,SAAS,OAAoC;AACpD,SAAO,SAAS,OAAO,UAAU,WAAW,QAAQ;AACtD;AAEA,SAAS,kBAAkB,OAAgB,KAAsB;AAC/D,SAAO,SAAS,OAAO,UAAU,WAC5B,MAAkC,GAAG,IACtC;AACN;AAEA,SAAS,kBACP,OACoD;AACpD,QAAM,YACJ,kBAAkB,OAAO,OAAO,KAAK,kBAAkB,OAAO,SAAS;AAEzE,SAAO,cAAc,SAAS,IACzB,YACD;AACN;AAEA,SAAS,mBACP,OACkD;AAClD,QAAM,YAAY,kBAAkB,OAAO,OAAO;AAClD,MAAI,iBAAiB,SAAS,GAAG;AAC/B,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEA,eAAe,SAOb,SACA,MAC+B;AAC/B,MAAI,QAAQ,KAAK;AACf,WAAO,QAAQ,IAAI,IAAI;AAAA,EACzB;AAEA,MAAI,iBAAqD,KAAK,KAAK,GAAG;AACpE,WAAO,KAAK,MAAM,IAAI,KAAK,OAAO,KAAK,OAAO;AAAA,EAChD;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAOP,SAOA;AACA,SAAO,QAAQ,UAAU;AAC3B;AAEA,SAAS,iBAMP,OACgE;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,SACE,SAAS,SAAS,OAAQ,MAA4B,QAAQ;AAElE;AAEA,SAAS,cAAc,OAAsC;AAC3D,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AAC/D,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,OAAO,OAAO,KAAK;AACjC,SACE,MAAM,SAAS,KACf,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,SAAS,iBAAiB,OAA4C;AACpE,SACE,MAAM,QAAQ,KAAK,KACnB,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,UAAU,QACV,OAAQ,KAA4B,SAAS,YAC7C,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,eAAe,2BAKb,OACA,UACA,MAMA,UACA;AACA,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO,SAAS;AAAA,EAClB;AAEA,QAAM,qBAAqB,oBAAI,IAG7B;AACF,QAAM,iBAAiB,oBAAI,IAA+C;AAE1E,QAAM,YAAY,CAAC,SAA6C;AAC9D,QAAI,mBAAmB,IAAI,IAAI,GAAG;AAChC;AAAA,IACF;AAEA,UAAM,kBAAkB,2BAA2B,KAAK,OAAO;AAC/D,uBAAmB,IAAI,MAAM,eAAe;AAC5C,UAAM,sBAA4D,OAChE,YACA,YACG;AACH,YAAM,YAAY,oBAAI,KAAK;AAC3B,YAAM,cAAc;AAAA,QAClB,OAAO,KAAK;AAAA,QACZ,UAAU,KAAK,QAAQ;AAAA,QACvB,QAAQ,KAAK,QAAQ;AAAA,QACrB,aAAa,KAAK,QAAQ;AAAA,MAC5B;AAEA,UAAI;AACF,cAAM,YAAY,MAAM,4BAA4B;AAAA,UAClD,UAAU,KAAK;AAAA,UACf;AAAA,UACA,SAAS;AAAA,UACT,QAAQ,KAAK;AAAA,UACb,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AACD,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,QAAQ,UAAU;AAAA,UAClB,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,UAAU,wBAAwB,UAAU,MAAM;AAAA,QACpD;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,SAAS,UAAU;AAAA,UACnB,UAAU;AAAA,YACR,MAAM,KAAK;AAAA,UACb;AAAA,QACF,CAAC;AACD,eAAO,UAAU;AAAA,MACnB,SAAS,OAAO;AACd,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,OAAO,uBAAuB,KAAK;AAAA,UACnC,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,UAAU,wBAAwB,2BAA2B,KAAK,CAAC;AAAA,QACrE;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,cAAM;AAAA,MACR;AAAA,IACF;AACA,wBAAoB,uBAAuB,IAAI;AAC/C,SAAK,UAAU;AAAA,EACjB;AAEA,QAAM,gBAAgB,CACpB,iBACG;AACH,eAAW,WAAW,cAAc;AAClC,iBAAW,QAAQ,SAAS;AAC1B,kBAAU,IAAI;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,gBAAc,QAAQ;AAEtB,aAAW,UAAU,qBAAqB,KAAK,GAAG;AAChD,UAAM,gBAAgB,OAAO;AAC7B,mBAAe,IAAI,QAAQ,aAAa;AACxC,WAAO,QAAQ,SAAS,gBAAuC,WAAW;AACxE,YAAM,cAAc,cAAc,MAAM,MAAM,SAAS;AAEvD,UAAI,cAAc,WAAW,GAAG;AAC9B,eAAO,YAAY,QAAQ,MAAM;AAC/B;AAAA,YACE,8BAAiD,KAAK;AAAA,UACxD;AAAA,QACF,CAAC;AAAA,MACH;AAEA,oBAAc,8BAAiD,KAAK,CAAC;AACrE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI;AACF,WAAO,MAAM,SAAS;AAAA,EACxB,UAAE;AACA,eAAW,CAAC,QAAQ,aAAa,KAAK,gBAAgB;AACpD,aAAO,QAAQ;AAAA,IACjB;AAEA,eAAW,CAAC,MAAM,eAAe,KAAK,oBAAoB;AACxD,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AACF;AAMA,SAAS,qBAAqB,OAAmC;AAC/D,SAAO,cAAc,KAAK,EAAE,OAAO,iBAAiB;AACtD;AAEA,SAAS,kBAAkB,OAAyC;AAClE,SAAO,WAAW,SAAS,OAAO,MAAM,UAAU;AACpD;AAEA,SAAS,8BAGP,OAA6D;AAC7D,QAAM,WAAwD,CAAC;AAC/D,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,eAAS,KAAK,WAAW;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA2C;AAChE,SAAO;AAAA,IACL,SAAS,OAAQ,MAA6B,SAAS;AAAA,EACzD;AACF;AAEA,SAAS,uBACP,OACsC;AACtC,QAAM,aAAa,eAAe,KAAK;AACvC,QAAM,EAAE,SAAS,MAAM,GAAG,QAAQ,IAAI;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,SAAS,OAAO,YAAY,WAAW,UAAU,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAAA,EAC7C;AACF;AAEA,SAAS,2BACP,SACA;AACA,QAAM,gBAAgB;AACtB,SAAO,cAAc,uBAAuB,KAAK;AACnD;AAEA,eAAe,4BAGb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAOG;AACD,MAAI,aAAa;AACjB,MAAI;AAEJ,QAAM,YAAY,MAAM,kBAAkB;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,OAAO,aAAa;AAC3B,mBAAa;AACb,mBAAa,MAAM,QAAQ,YAAY,QAAQ;AAC/C,aAAO,+BAA+B,UAAU;AAAA,IAClD;AAAA,IACA;AAAA,EACF,CAAC;AAED,MAAI,YAAY;AACd,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,kBAAkB,0BAA0B,UAAU;AAAA,MACtD,QAAQ,UAAU;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG,8BAA8B,UAAU,MAAM;AAAA,IACjD,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,cAIP;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAOE;AACA,QAAM,YAA8B,CAAC;AACrC,QAAM,YAA2B;AAAA,IAC/B,SAAS,CAAC,YAAY;AACpB,eAAS,KAAK,OAAO;AAAA,IACvB;AAAA,IACA,QAAQ,CAAC,SAAS,aAAa;AAC7B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,SAAS,aAAa;AAC3B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,WAAW,CAAC,SAAS,aAAa;AAChC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,MAAM,SAAS,aAAa;AACjC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA,UAAU;AAAA,UACR;AAAA,UACA,GAAI,YAAY,CAAC;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,eAAe,OAAO;AAAA,IAC1B,OAAO,QAAQ,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM;AAAA,MACpD;AAAA,MACA,OAAO,SAAoC;AACzC,cAAM,YAAY,oBAAI,KAAK;AAC3B,cAAM,cAAc;AAAA,UAClB;AAAA,UACA,UAAU,QAAQ;AAAA,UAClB,QAAQ,QAAQ;AAAA,UAChB,aAAa,QAAQ;AAAA,QACvB;AAEA,YAAI;AACF,gBAAM,YAAY,MAAM,sBAAsB;AAAA,YAC5C;AAAA,YACA;AAAA,YACA;AAAA,YACA,SAAS;AAAA,UACX,CAAC;AACD,gBAAM,aAAa,oBAAI,KAAK;AAC5B,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,QAAQ,UAAU;AAAA,YAClB,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,UAAU,wBAAwB,UAAU,MAAM;AAAA,UACpD;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,SAAS,UAAU;AAAA,YACnB,UAAU;AAAA,cACR,MAAM;AAAA,YACR;AAAA,UACF,CAAC;AACD,iBAAO,UAAU;AAAA,QACnB,SAAS,OAAO;AACd,gBAAM,aAAa,oBAAI,KAAK;AAC5B,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,OAAO,uBAAuB,KAAK;AAAA,YACnC,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,UAAU;AAAA,cACR,2BAA2B,KAAK;AAAA,YAClC;AAAA,UACF;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,QAAQ,QAAQ;AAAA,IAChB;AAAA,EACF;AACF;AAEA,SAAS,cAAc,QAAwC;AAC7D,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,WAAO,YAAY,MAAM;AAAA,EAC3B;AAEA,QAAM,aAAa;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,aAAW,OAAO,YAAY;AAC5B,UAAM,QAAS,OAAmC,GAAG;AACrD,UAAM,aAAa,YAAY,KAAK;AACpC,QAAI,eAAe,QAAW;AAC5B,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,QAAwC;AACnE,QAAM,UACJ,UAAU,OAAO,WAAW,WACxB,YAAa,OAAiC,OAAO,IACrD;AACN,MAAI,YAAY,QAAW;AACzB,WAAO;AAAA,EACT;AAEA,SACE,YAAY,MAAM,MAAM,WAAW,SAAY,SAAY,OAAO,MAAM;AAE5E;AAEA,SAAS,0BAA0B,QAA4B;AAC7D,SAAO,oBAAoB,MAAM,KAAK;AACxC;AAoBA,SAAS,+BACP,QAC0B;AAC1B,QAAM,mBAAmB,0BAA0B,MAAM;AAEzD,SAAO;AAAA,IACL,eAAe;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,aAAa,YAAY,MAAM,KAAK;AAAA,IACpC;AAAA,EACF;AACF;AAEA,SAAS,8BAA8B,QAAmB;AACxD,MAAI,2BAA2B,MAAM,GAAG;AACtC,WAAO;AAAA,MACL,QAAQ,OAAO;AAAA,MACf,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,iCAAiC,MAAM,GAAG;AAC5C,WAAO;AAAA,MACL,QAAQ,OAAO,eAAe,OAAO;AAAA,MACrC,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,kBAAkB,0BAA0B,MAAM;AAAA,EACpD;AACF;AAEA,SAAS,2BACP,OACmC;AACnC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,iBAAiB,SACjB,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,yBACP,OACoD;AACpD,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,iCACP,OACyC;AACzC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,+BACP,OAC0D;AAC1D,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,aAAa,QAAiB,eAAqC;AAC1E,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,WAAO,gBAAgB,IAAI,EAAE,WAAW,cAAc,IAAI,CAAC;AAAA,EAC7D;AAEA,QAAM,aACH,OAAmC,SACnC,OAAmC;AAEtC,QAAM,QACJ,cAAc,OAAO,eAAe,WAC/B,EAAE,GAAI,WAAuC,IAC9C,CAAC;AAEP,MAAI,MAAM,cAAc,UAAa,gBAAgB,GAAG;AACtD,UAAM,YAAY;AAAA,EACpB;AAEA,SAAO;AACT;AAEA,SAAS,eACP,QACA,UACA,QACA,OACmB;AACnB,MACE;AAAA,IACG,QAAgD;AAAA,EACnD,GACA;AACA,WAAQ,OAA0C;AAAA,EACpD;AAEA,MACE,oBAAqB,QAAgD,KAAK,GAC1E;AACA,WAAQ,OAAwC;AAAA,EAClD;AAEA,QAAM,kBAAkB,CAAC,GAAG,QAAQ;AACpC,MACE,WAAW,UACX,CAAC,gBAAgB;AAAA,IACf,CAAC,YACC,QAAQ,SAAS,eAAe,QAAQ,YAAY;AAAA,EACxD,GACA;AACA,oBAAgB,KAAK;AAAA,MACnB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,UAAU;AAAA,IACV,YAAY,OAAO,WAAW,WAAW,SAAS;AAAA,IAClD,UACI,QAAgD,YAEhC,MAAM;AAAA,IAC1B,OACI,QAAgD,SAEhC,MAAM;AAAA,EAC5B;AACF;AAEA,SAAS,cAAc,QAAmD;AACxE,SAAO,wBAAwB,MAAM;AACvC;AAEA,eAAe,sBAKb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAKG;AACD,SAAO,kBAAkB;AAAA,IACvB;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,KAAK;AAAA,IACd,QAAQ,KAAK;AAAA,EACf,CAAC;AACH;","names":[]}
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type {\n  Harness,\n  HarnessContext,\n  HarnessMetadata,\n  HarnessPrompt,\n  HarnessRun,\n  JsonValue,\n  NormalizedMessage,\n  NormalizedSession,\n  TimingSummary,\n  ToolCallRecord,\n  UsageSummary,\n} from \"vitest-evals/harness\";\nimport {\n  attachHarnessRunToError,\n  isHarnessRun,\n  isNormalizedSession,\n  normalizeContent,\n  resolveHarnessRunErrors,\n  serializeError,\n  toJsonValue,\n} from \"vitest-evals/harness\";\nimport {\n  executeWithReplay,\n  getReplayMetadataFromError,\n  normalizeReplayMetadata,\n} from \"vitest-evals/replay\";\nimport type {\n  ReplayMode,\n  ToolRecording,\n  ToolReplayConfig,\n} from \"vitest-evals/replay\";\n\ntype MaybePromise<T> = T | Promise<T>;\ntype AnyPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;\ntype InferredPiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<\n  string,\n  PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>\n>;\n\ntype PiAgentToolLike<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  name: string;\n  execute: (toolCallId: string, args: Record<string, JsonValue>) => unknown;\n};\n\nconst ORIGINAL_NATIVE_EXECUTE = Symbol(\"vitest-evals.originalNativeExecute\");\n\ntype NativeToolExecute<\n  TInput,\n  TMetadata extends HarnessMetadata,\n> = PiAgentToolLike<TInput, TMetadata>[\"execute\"] & {\n  [ORIGINAL_NATIVE_EXECUTE]?: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n};\n\nexport type PiAiReplayMode = ReplayMode;\n\nexport interface PiAiEventSink {\n  message: (message: NormalizedMessage) => void;\n  system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  user: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;\n  tool: (\n    name: string,\n    content: JsonValue,\n    metadata?: Record<string, JsonValue>,\n  ) => void;\n}\n\nexport interface PiAiToolContext<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  input: TInput;\n  metadata: HarnessContext<TMetadata>[\"metadata\"];\n  signal?: AbortSignal;\n  setArtifact: HarnessContext<TMetadata>[\"setArtifact\"];\n}\n\nexport type PiAiToolRecording<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n> = ToolRecording<TArgs, TResult>;\n\nexport type PiAiToolReplayConfig<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;\n\nexport type PiAiToolReplayPolicy<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> =\n  | boolean\n  | PiAiToolReplayConfig<\n      Record<string, JsonValue>,\n      JsonValue,\n      TInput,\n      TMetadata\n    >;\n\nexport type PiAiToolReplayPolicies<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = Record<string, PiAiToolReplayPolicy<TInput, TMetadata>>;\n\nexport interface PiAiToolDefinition<\n  TArgs extends Record<string, JsonValue> = Record<string, JsonValue>,\n  TResult extends JsonValue = JsonValue,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  description?: string;\n  execute: (\n    args: TArgs,\n    context: PiAiToolContext<TInput, TMetadata>,\n  ) => MaybePromise<TResult>;\n}\n\nexport type PiAiToolset<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = AnyPiAiToolset<TInput, TMetadata>;\n\ntype ToolArgs<TTool> = TTool extends PiAiToolDefinition<\n  infer TArgs,\n  infer _TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TArgs\n  : never;\n\ntype ToolResult<TTool> = TTool extends PiAiToolDefinition<\n  infer _TArgs,\n  infer TResult,\n  infer _TInput,\n  infer _TMetadata\n>\n  ? TResult\n  : never;\n\nexport type PiAiRuntime<\n  TTools extends PiAiToolset<TInput, TMetadata>,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> = {\n  tools: {\n    [K in keyof TTools]: (\n      args: ToolArgs<TTools[K]>,\n    ) => Promise<ToolResult<TTools[K]>>;\n  };\n  events: PiAiEventSink;\n  signal?: AbortSignal;\n};\n\nexport interface PiAiHarnessRunArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> {\n  agent: TAgent;\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  runtime: PiAiRuntime<TTools, TInput, TMetadata>;\n}\n\nexport interface PiAiHarnessResultArgs<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {\n  result: TResult;\n}\n\nexport interface PiAiCreateAgentArgs<\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n> {\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n}\n\ninterface PiAiHarnessBaseOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  agent?: TAgent;\n  createAgent?: (\n    args: PiAiCreateAgentArgs<TInput, TMetadata>,\n  ) => MaybePromise<TAgent>;\n  toolReplay?: PiAiToolReplayPolicies<TInput, TMetadata>;\n  normalize?: PiAiHarnessNormalizeOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >;\n  prompt: HarnessPrompt;\n  name?: string;\n}\n\nexport interface PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> {\n  tools: TTools;\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport interface PiAiHarnessInferredToolsOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n> extends PiAiHarnessBaseOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    InferredPiAiToolset<TInput, TMetadata>\n  > {\n  tools?: undefined;\n  run?: (\n    args: PiAiHarnessRunArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      InferredPiAiToolset<TInput, TMetadata>\n    >,\n  ) => MaybePromise<TResult | HarnessRun>;\n}\n\nexport type PiAiHarnessOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> =\n  | PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>\n  | PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>;\n\ntype PiAiHarnessRunOptions<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> & {\n  run?: (\n    args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\ntype PiAiRunnableAgent<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n> = {\n  run: (\n    input: TInput,\n    runtime: PiAiRuntime<TTools, TInput, TMetadata>,\n  ) => MaybePromise<TResult | HarnessRun>;\n};\n\nexport interface PiAiHarnessNormalizeOptions<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n> {\n  session?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<NormalizedSession>;\n  output?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<JsonValue | undefined>;\n  usage?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<UsageSummary>;\n  timings?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<TimingSummary | undefined>;\n  errors?: (\n    args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>,\n  ) => MaybePromise<Array<Record<string, JsonValue>>>;\n}\n\ntype InferredToolSurfaces<TInput, TMetadata extends HarnessMetadata> = {\n  runtimeTools?: InferredPiAiToolset<TInput, TMetadata>;\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>;\n};\n\ntype PiToolExecutionState = {\n  activeNativeToolNames: Map<string, number>;\n};\n\n/** Adapts a Pi agent runtime into a normalized vitest-evals harness. */\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessWithToolsOptions<\n    TAgent,\n    TInput,\n    TMetadata,\n    TResult,\n    TTools\n  >,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n>(\n  options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>,\n): Harness<TInput, TMetadata>;\nexport function piAiHarness<\n  TAgent,\n  TInput = string,\n  TMetadata extends HarnessMetadata = HarnessMetadata,\n  TResult = unknown,\n  TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<\n    TInput,\n    TMetadata\n  >,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): Harness<TInput, TMetadata> {\n  return {\n    name: options.name ?? \"pi-ai\",\n    prompt: options.prompt,\n    run: async (input, context) => {\n      const agent = await resolveAgent(options, {\n        input,\n        context,\n      });\n      const messages: NormalizedMessage[] = [\n        {\n          role: \"user\",\n          content: normalizeContent(input),\n        },\n      ];\n      const inferredTools = resolveInferredToolSurfaces<TInput, TMetadata>(\n        agent,\n      );\n\n      if (hasExplicitToolset(options)) {\n        return executePiHarnessRun(\n          options,\n          agent,\n          input,\n          context,\n          messages,\n          options.tools,\n          inferredTools.nativeToolsets,\n        );\n      }\n\n      return executePiHarnessRun(\n        options,\n        agent,\n        input,\n        context,\n        messages,\n        inferredTools.runtimeTools,\n        inferredTools.nativeToolsets,\n      );\n    },\n  };\n}\n\nasync function executePiHarnessRun<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  agent: TAgent,\n  input: TInput,\n  context: HarnessContext<TMetadata>,\n  messages: NormalizedMessage[],\n  runtimeTools: TTools | undefined,\n  nativeToolsets?: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n): Promise<HarnessRun> {\n  const executionState = createPiToolExecutionState();\n  const runtime = createRuntime({\n    input,\n    context,\n    tools: runtimeTools,\n    toolReplay: options.toolReplay,\n    executionState,\n    messages,\n  });\n\n  try {\n    const result = await withInstrumentedAgentTools(\n      agent,\n      nativeToolsets,\n      {\n        input,\n        context,\n        messages,\n        toolCalls: runtime.toolCalls,\n        toolReplay: options.toolReplay,\n        executionState,\n      },\n      () =>\n        runAgent(options, {\n          agent,\n          input,\n          context,\n          runtime,\n        }),\n    );\n\n    if (isHarnessRun(result) && !hasResultOverrides(options)) {\n      if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {\n        result.artifacts = context.artifacts;\n      }\n      return result;\n    }\n\n    const normalizeResult = result as TResult;\n    const resultArgs = {\n      agent,\n      input,\n      context,\n      runtime,\n      result: normalizeResult,\n    } satisfies PiAiHarnessResultArgs<\n      TAgent,\n      TInput,\n      TMetadata,\n      TResult,\n      TTools\n    >;\n\n    const output = options.normalize?.output\n      ? await options.normalize.output(resultArgs)\n      : resolveOutput(normalizeResult);\n    const usage = options.normalize?.usage\n      ? await options.normalize.usage(resultArgs)\n      : resolveUsage(normalizeResult, runtime.toolCalls.length);\n    const session = options.normalize?.session\n      ? await options.normalize.session(resultArgs)\n      : resolveSession(normalizeResult, messages, output, usage);\n\n    return {\n      session,\n      output,\n      usage,\n      timings: options.normalize?.timings\n        ? await options.normalize.timings(resultArgs)\n        : undefined,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: options.normalize?.errors\n        ? await options.normalize.errors(resultArgs)\n        : resolveErrors(normalizeResult),\n    };\n  } catch (error) {\n    const usage = resolveUsage(undefined, runtime.toolCalls.length);\n    const run = {\n      session: resolveSession(undefined, messages, undefined, usage),\n      output: undefined,\n      usage,\n      artifacts:\n        Object.keys(context.artifacts).length > 0\n          ? context.artifacts\n          : undefined,\n      errors: [serializeError(error)],\n    } satisfies HarnessRun;\n\n    throw attachHarnessRunToError(error, run);\n  }\n}\n\nasync function resolveAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  args: PiAiCreateAgentArgs<TInput, TMetadata>,\n) {\n  if (options.agent !== undefined) {\n    return options.agent;\n  }\n\n  if (options.createAgent) {\n    return options.createAgent(args);\n  }\n\n  throw new Error(\n    \"piAiHarness requires either an agent instance or a createAgent() function.\",\n  );\n}\n\nfunction hasResultOverrides<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>) {\n  return Boolean(\n    options.normalize?.output ??\n      options.normalize?.session ??\n      options.normalize?.usage ??\n      options.normalize?.timings ??\n      options.normalize?.errors,\n  );\n}\n\nfunction resolveInferredToolSurfaces<TInput, TMetadata extends HarnessMetadata>(\n  agent: unknown,\n): InferredToolSurfaces<TInput, TMetadata> {\n  let runtimeTools: InferredPiAiToolset<TInput, TMetadata> | undefined;\n  const nativeToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nextRuntimeTools = getRuntimeToolset<TInput, TMetadata>(candidate);\n    if (runtimeTools === undefined && nextRuntimeTools !== undefined) {\n      runtimeTools = nextRuntimeTools;\n    }\n\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      nativeToolsets.push(nativeTools);\n    }\n  }\n\n  return {\n    ...(runtimeTools ? { runtimeTools } : {}),\n    ...(nativeToolsets.length > 0 ? { nativeToolsets } : {}),\n  };\n}\n\nfunction getAgentToolCandidates(agent: unknown): object[] {\n  const roots = getAgentRoots(agent);\n  const candidates: object[] = [];\n  const seen = new Set<object>();\n\n  for (const root of roots) {\n    addUniqueObject(candidates, seen, root);\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"state\"));\n    addUniqueObject(candidates, seen, getObjectProperty(root, \"initialState\"));\n  }\n\n  return candidates;\n}\n\nfunction getAgentRoots(agent: unknown): object[] {\n  return [asObject(agent)]\n    .concat(asObject(getObjectProperty(agent, \"agent\")))\n    .filter((value): value is object => value !== undefined);\n}\n\nfunction addUniqueObject(\n  candidates: object[],\n  seen: Set<object>,\n  value: unknown,\n) {\n  if (!value || typeof value !== \"object\" || seen.has(value)) {\n    return;\n  }\n\n  seen.add(value);\n  candidates.push(value);\n}\n\nfunction asObject(value: unknown): object | undefined {\n  return value && typeof value === \"object\" ? value : undefined;\n}\n\nfunction getObjectProperty(value: unknown, key: string): unknown {\n  return value && typeof value === \"object\"\n    ? (value as Record<string, unknown>)[key]\n    : undefined;\n}\n\nfunction getRuntimeToolset<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): InferredPiAiToolset<TInput, TMetadata> | undefined {\n  const candidate =\n    getObjectProperty(value, \"tools\") ?? getObjectProperty(value, \"toolset\");\n\n  return isPiAiToolset(candidate)\n    ? (candidate as InferredPiAiToolset<TInput, TMetadata>)\n    : undefined;\n}\n\nfunction getNativeToolArray<TInput, TMetadata extends HarnessMetadata>(\n  value: object,\n): PiAgentToolLike<TInput, TMetadata>[] | undefined {\n  const candidate = getObjectProperty(value, \"tools\");\n  if (isAgentToolArray(candidate)) {\n    return candidate as PiAgentToolLike<TInput, TMetadata>[];\n  }\n\n  return undefined;\n}\n\nasync function runAgent<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessRunOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n  args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>,\n): Promise<TResult | HarnessRun> {\n  if (options.run) {\n    return options.run(args);\n  }\n\n  if (hasPiAiRunMethod<TInput, TMetadata, TResult, TTools>(args.agent)) {\n    return args.agent.run(args.input, args.runtime);\n  }\n\n  throw new Error(\n    \"piAiHarness requires a run() function unless the provided agent exposes run(input, runtime).\",\n  );\n}\n\nfunction hasExplicitToolset<\n  TAgent,\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  options: PiAiHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>,\n): options is PiAiHarnessWithToolsOptions<\n  TAgent,\n  TInput,\n  TMetadata,\n  TResult,\n  TTools\n> {\n  return options.tools !== undefined;\n}\n\nfunction hasPiAiRunMethod<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TResult,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>(\n  agent: unknown,\n): agent is PiAiRunnableAgent<TInput, TMetadata, TResult, TTools> {\n  if (!agent || typeof agent !== \"object\") {\n    return false;\n  }\n\n  return (\n    \"run\" in agent && typeof (agent as { run?: unknown }).run === \"function\"\n  );\n}\n\nfunction isPiAiToolset(value: unknown): value is PiAiToolset {\n  if (!value || typeof value !== \"object\" || Array.isArray(value)) {\n    return false;\n  }\n\n  const tools = Object.values(value);\n  return (\n    tools.length > 0 &&\n    tools.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nfunction isAgentToolArray(value: unknown): value is PiAgentToolLike[] {\n  return (\n    Array.isArray(value) &&\n    value.every((tool) =>\n      Boolean(\n        tool &&\n          typeof tool === \"object\" &&\n          \"name\" in tool &&\n          typeof (tool as { name?: unknown }).name === \"string\" &&\n          \"execute\" in tool &&\n          typeof (tool as { execute?: unknown }).execute === \"function\",\n      ),\n    )\n  );\n}\n\nasync function withInstrumentedAgentTools<\n  TResult,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(\n  agent: unknown,\n  toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> | undefined,\n  args: {\n    input: TInput;\n    context: HarnessContext<TMetadata>;\n    messages: NormalizedMessage[];\n    toolCalls: ToolCallRecord[];\n    toolReplay: PiAiToolReplayPolicies<TInput, TMetadata> | undefined;\n    executionState: PiToolExecutionState;\n  },\n  callback: () => Promise<TResult>,\n) {\n  if (!toolsets || toolsets.length === 0) {\n    return callback();\n  }\n\n  const originalExecutions = new Map<\n    PiAgentToolLike<TInput, TMetadata>,\n    PiAgentToolLike<TInput, TMetadata>[\"execute\"]\n  >();\n  const originalResets = new Map<ResettableAgent, ResettableAgent[\"reset\"]>();\n\n  const patchTool = (tool: PiAgentToolLike<TInput, TMetadata>) => {\n    if (originalExecutions.has(tool)) {\n      return;\n    }\n\n    const originalExecute = getNativeToolExecuteOrigin(tool.execute);\n    originalExecutions.set(tool, originalExecute);\n    const instrumentedExecute: NativeToolExecute<TInput, TMetadata> = async (\n      toolCallId: string,\n      rawArgs: Record<string, JsonValue>,\n    ) => {\n      const startedAt = new Date();\n      const toolContext = {\n        input: args.input,\n        metadata: args.context.metadata,\n        signal: args.context.signal,\n        setArtifact: args.context.setArtifact,\n      } satisfies PiAiToolContext<TInput, TMetadata>;\n      const leaveNativeTool = enterNativeToolExecution(\n        args.executionState,\n        tool.name,\n      );\n\n      try {\n        const execution = await executeNativeToolWithReplay({\n          toolName: tool.name,\n          toolCallId,\n          execute: originalExecute,\n          replay: args.toolReplay?.[tool.name],\n          args: rawArgs,\n          context: toolContext,\n        });\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          result: execution.normalizedResult,\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(execution.replay),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        args.messages.push({\n          role: \"tool\",\n          content: execution.normalizedResult,\n          metadata: {\n            name: tool.name,\n          },\n        });\n        return execution.result;\n      } catch (error) {\n        const finishedAt = new Date();\n        const call = {\n          name: tool.name,\n          arguments: rawArgs,\n          error: serializeToolCallError(error),\n          startedAt: startedAt.toISOString(),\n          finishedAt: finishedAt.toISOString(),\n          durationMs: finishedAt.getTime() - startedAt.getTime(),\n          metadata: normalizeReplayMetadata(getReplayMetadataFromError(error)),\n        } satisfies ToolCallRecord;\n        args.toolCalls.push(call);\n        args.messages.push({\n          role: \"assistant\",\n          toolCalls: [call],\n        });\n        throw error;\n      } finally {\n        leaveNativeTool();\n      }\n    };\n    instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute;\n    tool.execute = instrumentedExecute;\n  };\n\n  const patchToolsets = (\n    nextToolsets: Array<PiAgentToolLike<TInput, TMetadata>[]>,\n  ) => {\n    for (const toolset of nextToolsets) {\n      for (const tool of toolset) {\n        patchTool(tool);\n      }\n    }\n  };\n\n  patchToolsets(toolsets);\n\n  for (const target of getAgentResetTargets(agent)) {\n    const originalReset = target.reset;\n    originalResets.set(target, originalReset);\n    target.reset = function patchedReset(this: ResettableAgent, ...resetArgs) {\n      const resetResult = originalReset.apply(this, resetArgs);\n\n      if (isPromiseLike(resetResult)) {\n        return resetResult.finally(() => {\n          patchToolsets(\n            resolveInferredNativeToolsets<TInput, TMetadata>(agent),\n          );\n        });\n      }\n\n      patchToolsets(resolveInferredNativeToolsets<TInput, TMetadata>(agent));\n      return resetResult;\n    };\n  }\n\n  try {\n    return await callback();\n  } finally {\n    for (const [target, originalReset] of originalResets) {\n      target.reset = originalReset;\n    }\n\n    for (const [tool, originalExecute] of originalExecutions) {\n      tool.execute = originalExecute;\n    }\n  }\n}\n\ntype ResettableAgent = {\n  reset: (...args: unknown[]) => unknown;\n};\n\nfunction getAgentResetTargets(agent: unknown): ResettableAgent[] {\n  return getAgentRoots(agent).filter(isResettableAgent);\n}\n\nfunction isResettableAgent(value: object): value is ResettableAgent {\n  return \"reset\" in value && typeof value.reset === \"function\";\n}\n\nfunction resolveInferredNativeToolsets<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>(agent: unknown): Array<PiAgentToolLike<TInput, TMetadata>[]> {\n  const toolsets: Array<PiAgentToolLike<TInput, TMetadata>[]> = [];\n  const seenToolsets = new Set<PiAgentToolLike<TInput, TMetadata>[]>();\n\n  for (const candidate of getAgentToolCandidates(agent)) {\n    const nativeTools = getNativeToolArray(candidate);\n    if (nativeTools && !seenToolsets.has(nativeTools)) {\n      seenToolsets.add(nativeTools);\n      toolsets.push(nativeTools);\n    }\n  }\n\n  return toolsets;\n}\n\nfunction isPromiseLike(value: unknown): value is Promise<unknown> {\n  return Boolean(\n    value && typeof (value as { then?: unknown }).then === \"function\",\n  );\n}\n\nfunction serializeToolCallError(\n  error: unknown,\n): NonNullable<ToolCallRecord[\"error\"]> {\n  const serialized = serializeError(error);\n  const { message, type, ...details } = serialized;\n\n  return {\n    ...details,\n    message: typeof message === \"string\" ? message : String(message),\n    ...(typeof type === \"string\" ? { type } : {}),\n  };\n}\n\nfunction getNativeToolExecuteOrigin<TInput, TMetadata extends HarnessMetadata>(\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"],\n) {\n  const nativeExecute = execute as NativeToolExecute<TInput, TMetadata>;\n  return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;\n}\n\nfunction createPiToolExecutionState(): PiToolExecutionState {\n  return {\n    activeNativeToolNames: new Map(),\n  };\n}\n\nfunction enterNativeToolExecution(\n  state: PiToolExecutionState,\n  toolName: string,\n) {\n  state.activeNativeToolNames.set(\n    toolName,\n    (state.activeNativeToolNames.get(toolName) ?? 0) + 1,\n  );\n\n  return () => {\n    const nextCount = (state.activeNativeToolNames.get(toolName) ?? 1) - 1;\n    if (nextCount <= 0) {\n      state.activeNativeToolNames.delete(toolName);\n      return;\n    }\n\n    state.activeNativeToolNames.set(toolName, nextCount);\n  };\n}\n\nfunction hasActiveNativeToolExecution(\n  state: PiToolExecutionState,\n  toolName: string,\n) {\n  return (state.activeNativeToolNames.get(toolName) ?? 0) > 0;\n}\n\nasync function executeNativeToolWithReplay<\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  toolCallId,\n  execute,\n  replay,\n  args,\n  context,\n}: {\n  toolName: string;\n  toolCallId: string;\n  execute: PiAgentToolLike<TInput, TMetadata>[\"execute\"];\n  replay: PiAiToolReplayPolicy<TInput, TMetadata> | undefined;\n  args: Record<string, JsonValue>;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  let didExecute = false;\n  let liveResult: unknown;\n\n  const execution = await executeWithReplay({\n    toolName: createNativeReplayToolName(toolName),\n    args,\n    context,\n    execute: async (toolArgs) => {\n      didExecute = true;\n      liveResult = await execute(toolCallId, toolArgs);\n      return createNativeToolReplayEnvelope(liveResult);\n    },\n    replay,\n  });\n\n  if (didExecute) {\n    return {\n      result: liveResult,\n      normalizedResult: normalizeReplayToolResult(liveResult),\n      replay: execution.replay,\n    };\n  }\n\n  return {\n    ...resolveNativeToolReplayResult(execution.result),\n    replay: execution.replay,\n  };\n}\n\nfunction createNativeReplayToolName(toolName: string) {\n  return `${toolName}.native`;\n}\n\nfunction createRuntime<\n  TInput,\n  TMetadata extends HarnessMetadata,\n  TTools extends PiAiToolset<TInput, TMetadata>,\n>({\n  input,\n  context,\n  tools,\n  toolReplay,\n  executionState,\n  messages,\n}: {\n  input: TInput;\n  context: HarnessContext<TMetadata>;\n  tools: TTools | undefined;\n  toolReplay: PiAiToolReplayPolicies<TInput, TMetadata> | undefined;\n  executionState: PiToolExecutionState;\n  messages: NormalizedMessage[];\n}): PiAiRuntime<TTools, TInput, TMetadata> & {\n  toolCalls: ToolCallRecord[];\n} {\n  const toolCalls: ToolCallRecord[] = [];\n  const eventSink: PiAiEventSink = {\n    message: (message) => {\n      messages.push(message);\n    },\n    system: (content, metadata) => {\n      messages.push({\n        role: \"system\",\n        content,\n        metadata,\n      });\n    },\n    user: (content, metadata) => {\n      messages.push({\n        role: \"user\",\n        content,\n        metadata,\n      });\n    },\n    assistant: (content, metadata) => {\n      messages.push({\n        role: \"assistant\",\n        content,\n        metadata,\n      });\n    },\n    tool: (name, content, metadata) => {\n      messages.push({\n        role: \"tool\",\n        content,\n        metadata: {\n          name,\n          ...(metadata ?? {}),\n        },\n      });\n    },\n  };\n\n  const runtimeTools = Object.fromEntries(\n    Object.entries(tools ?? {}).map(([toolName, tool]) => [\n      toolName,\n      async (args: Record<string, JsonValue>) => {\n        const startedAt = new Date();\n        const isNativeImplementationCall = hasActiveNativeToolExecution(\n          executionState,\n          toolName,\n        );\n        const toolContext = {\n          input,\n          metadata: context.metadata,\n          signal: context.signal,\n          setArtifact: context.setArtifact,\n        } satisfies PiAiToolContext<TInput, TMetadata>;\n\n        try {\n          const execution = await executeToolWithReplay({\n            toolName,\n            tool,\n            replay: isNativeImplementationCall\n              ? undefined\n              : toolReplay?.[toolName],\n            args,\n            context: toolContext,\n          });\n          const finishedAt = new Date();\n\n          if (isNativeImplementationCall) {\n            return execution.result;\n          }\n\n          const call = {\n            name: toolName,\n            arguments: args,\n            result: execution.result,\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(execution.replay),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          messages.push({\n            role: \"tool\",\n            content: execution.result,\n            metadata: {\n              name: toolName,\n            },\n          });\n          return execution.result;\n        } catch (error) {\n          const finishedAt = new Date();\n          if (isNativeImplementationCall) {\n            throw error;\n          }\n\n          const call = {\n            name: toolName,\n            arguments: args,\n            error: serializeToolCallError(error),\n            startedAt: startedAt.toISOString(),\n            finishedAt: finishedAt.toISOString(),\n            durationMs: finishedAt.getTime() - startedAt.getTime(),\n            metadata: normalizeReplayMetadata(\n              getReplayMetadataFromError(error),\n            ),\n          } satisfies ToolCallRecord;\n          toolCalls.push(call);\n          messages.push({\n            role: \"assistant\",\n            toolCalls: [call],\n          });\n          throw error;\n        }\n      },\n    ]),\n  ) as unknown as PiAiRuntime<TTools, TInput, TMetadata>[\"tools\"];\n\n  return {\n    tools: runtimeTools,\n    events: eventSink,\n    signal: context.signal,\n    toolCalls,\n  };\n}\n\nfunction resolveOutput(result: unknown): JsonValue | undefined {\n  if (!result || typeof result !== \"object\") {\n    return toJsonValue(result);\n  }\n\n  const candidates = [\n    \"output\",\n    \"decision\",\n    \"result\",\n    \"final\",\n  ] satisfies string[];\n\n  for (const key of candidates) {\n    const value = (result as Record<string, unknown>)[key];\n    const normalized = toJsonValue(value);\n    if (normalized !== undefined) {\n      return normalized;\n    }\n  }\n\n  return undefined;\n}\n\nfunction normalizeToolResult(result: unknown): JsonValue | undefined {\n  const details =\n    result && typeof result === \"object\"\n      ? toJsonValue((result as { details?: unknown }).details)\n      : undefined;\n  if (details !== undefined) {\n    return details;\n  }\n\n  return (\n    toJsonValue(result) ?? (result === undefined ? undefined : String(result))\n  );\n}\n\nfunction normalizeReplayToolResult(result: unknown): JsonValue {\n  return normalizeToolResult(result) ?? null;\n}\n\ntype NativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 2;\n  };\n  agentResult: JsonValue;\n  normalizedResult: JsonValue;\n};\n\ntype LegacyNativeToolReplayEnvelope = {\n  __vitestEvals: {\n    kind: \"pi-ai-native-tool-result\";\n    version: 1;\n  };\n  normalizedResult: JsonValue;\n  agentResult?: JsonValue;\n};\n\nfunction createNativeToolReplayEnvelope(\n  result: unknown,\n): NativeToolReplayEnvelope {\n  const normalizedResult = normalizeReplayToolResult(result);\n\n  return {\n    __vitestEvals: {\n      kind: \"pi-ai-native-tool-result\",\n      version: 2,\n    },\n    agentResult: toJsonValue(result) ?? normalizedResult,\n    normalizedResult,\n  };\n}\n\nfunction resolveNativeToolReplayResult(result: JsonValue) {\n  if (isNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  if (isLegacyNativeToolReplayEnvelope(result)) {\n    return {\n      result: result.agentResult ?? result.normalizedResult,\n      normalizedResult: result.normalizedResult,\n    };\n  }\n\n  return {\n    result,\n    normalizedResult: normalizeReplayToolResult(result),\n  };\n}\n\nfunction isNativeToolReplayEnvelope(\n  value: unknown,\n): value is NativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"agentResult\" in value &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isNativeToolReplayMarker(\n  value: unknown,\n): value is NativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 2,\n  );\n}\n\nfunction isLegacyNativeToolReplayEnvelope(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"__vitestEvals\" in value &&\n      isLegacyNativeToolReplayMarker(\n        (value as { __vitestEvals?: unknown }).__vitestEvals,\n      ) &&\n      \"normalizedResult\" in value,\n  );\n}\n\nfunction isLegacyNativeToolReplayMarker(\n  value: unknown,\n): value is LegacyNativeToolReplayEnvelope[\"__vitestEvals\"] {\n  return Boolean(\n    value &&\n      typeof value === \"object\" &&\n      \"kind\" in value &&\n      (value as { kind?: unknown }).kind === \"pi-ai-native-tool-result\" &&\n      \"version\" in value &&\n      (value as { version?: unknown }).version === 1,\n  );\n}\n\nfunction resolveUsage(result: unknown, toolCallCount: number): UsageSummary {\n  if (!result || typeof result !== \"object\") {\n    return toolCallCount > 0 ? { toolCalls: toolCallCount } : {};\n  }\n\n  const usageValue =\n    (result as Record<string, unknown>).usage ??\n    (result as Record<string, unknown>).metrics;\n\n  const usage =\n    usageValue && typeof usageValue === \"object\"\n      ? ({ ...(usageValue as Record<string, unknown>) } as UsageSummary)\n      : {};\n\n  if (usage.toolCalls === undefined && toolCallCount > 0) {\n    usage.toolCalls = toolCallCount;\n  }\n\n  return usage;\n}\n\nfunction resolveSession(\n  result: unknown,\n  messages: NormalizedMessage[],\n  output: JsonValue | undefined,\n  usage: UsageSummary,\n): NormalizedSession {\n  if (\n    isNormalizedSession(\n      (result as Record<string, unknown> | undefined)?.session,\n    )\n  ) {\n    return (result as { session: NormalizedSession }).session;\n  }\n\n  if (\n    isNormalizedSession((result as Record<string, unknown> | undefined)?.trace)\n  ) {\n    return (result as { trace: NormalizedSession }).trace;\n  }\n\n  const sessionMessages = [...messages];\n  if (\n    output !== undefined &&\n    !sessionMessages.some(\n      (message) =>\n        message.role === \"assistant\" && message.content !== undefined,\n    )\n  ) {\n    sessionMessages.push({\n      role: \"assistant\",\n      content: output,\n    });\n  }\n\n  return {\n    messages: sessionMessages,\n    outputText: typeof output === \"string\" ? output : undefined,\n    provider:\n      ((result as Record<string, unknown> | undefined)?.provider as\n        | string\n        | undefined) ?? usage.provider,\n    model:\n      ((result as Record<string, unknown> | undefined)?.model as\n        | string\n        | undefined) ?? usage.model,\n  };\n}\n\nfunction resolveErrors(result: unknown): Array<Record<string, JsonValue>> {\n  return resolveHarnessRunErrors(result);\n}\n\nasync function executeToolWithReplay<\n  TArgs extends Record<string, JsonValue>,\n  TResult extends JsonValue,\n  TInput,\n  TMetadata extends HarnessMetadata,\n>({\n  toolName,\n  tool,\n  replay,\n  args,\n  context,\n}: {\n  toolName: string;\n  tool: PiAiToolDefinition<TArgs, TResult, TInput, TMetadata>;\n  replay: PiAiToolReplayPolicy<TInput, TMetadata> | undefined;\n  args: TArgs;\n  context: PiAiToolContext<TInput, TMetadata>;\n}) {\n  return executeWithReplay<\n    Record<string, JsonValue>,\n    JsonValue,\n    PiAiToolContext<TInput, TMetadata>\n  >({\n    toolName,\n    args,\n    context,\n    execute: (toolArgs, toolContext) =>\n      tool.execute(toolArgs as TArgs, toolContext),\n    replay,\n  });\n}\n"],"mappings":";AAaA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AA4BP,IAAM,0BAA0B,OAAO,oCAAoC;AAmTpE,SAAS,YAUd,SAC4B;AAC5B,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,QAAQ,QAAQ;AAAA,IAChB,KAAK,OAAO,OAAO,YAAY;AAC7B,YAAM,QAAQ,MAAM,aAAa,SAAS;AAAA,QACxC;AAAA,QACA;AAAA,MACF,CAAC;AACD,YAAM,WAAgC;AAAA,QACpC;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,KAAK;AAAA,QACjC;AAAA,MACF;AACA,YAAM,gBAAgB;AAAA,QACpB;AAAA,MACF;AAEA,UAAI,mBAAmB,OAAO,GAAG;AAC/B,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,UACR,cAAc;AAAA,QAChB;AAAA,MACF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,cAAc;AAAA,QACd,cAAc;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,oBAOb,SACA,OACA,OACA,SACA,UACA,cACA,gBACqB;AACrB,QAAM,iBAAiB,2BAA2B;AAClD,QAAM,UAAU,cAAc;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,YAAY,QAAQ;AAAA,IACpB;AAAA,IACA;AAAA,EACF,CAAC;AAED,MAAI;AACF,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,WAAW,QAAQ;AAAA,QACnB,YAAY,QAAQ;AAAA,QACpB;AAAA,MACF;AAAA,MACA,MACE,SAAS,SAAS;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACL;AAEA,QAAI,aAAa,MAAM,KAAK,CAAC,mBAAmB,OAAO,GAAG;AACxD,UAAI,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,KAAK,CAAC,OAAO,WAAW;AAClE,eAAO,YAAY,QAAQ;AAAA,MAC7B;AACA,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB;AACxB,UAAM,aAAa;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IACV;AAQA,UAAM,SAAS,QAAQ,WAAW,SAC9B,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AACjC,UAAM,QAAQ,QAAQ,WAAW,QAC7B,MAAM,QAAQ,UAAU,MAAM,UAAU,IACxC,aAAa,iBAAiB,QAAQ,UAAU,MAAM;AAC1D,UAAM,UAAU,QAAQ,WAAW,UAC/B,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C,eAAe,iBAAiB,UAAU,QAAQ,KAAK;AAE3D,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS,QAAQ,WAAW,UACxB,MAAM,QAAQ,UAAU,QAAQ,UAAU,IAC1C;AAAA,MACJ,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,QAAQ,WAAW,SACvB,MAAM,QAAQ,UAAU,OAAO,UAAU,IACzC,cAAc,eAAe;AAAA,IACnC;AAAA,EACF,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,QAAW,QAAQ,UAAU,MAAM;AAC9D,UAAM,MAAM;AAAA,MACV,SAAS,eAAe,QAAW,UAAU,QAAW,KAAK;AAAA,MAC7D,QAAQ;AAAA,MACR;AAAA,MACA,WACE,OAAO,KAAK,QAAQ,SAAS,EAAE,SAAS,IACpC,QAAQ,YACR;AAAA,MACN,QAAQ,CAAC,eAAe,KAAK,CAAC;AAAA,IAChC;AAEA,UAAM,wBAAwB,OAAO,GAAG;AAAA,EAC1C;AACF;AAEA,eAAe,aAOb,SACA,MACA;AACA,MAAI,QAAQ,UAAU,QAAW;AAC/B,WAAO,QAAQ;AAAA,EACjB;AAEA,MAAI,QAAQ,aAAa;AACvB,WAAO,QAAQ,YAAY,IAAI;AAAA,EACjC;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAMP,SAA4E;AAC5E,SAAO;AAAA,IACL,QAAQ,WAAW,UACjB,QAAQ,WAAW,WACnB,QAAQ,WAAW,SACnB,QAAQ,WAAW,WACnB,QAAQ,WAAW;AAAA,EACvB;AACF;AAEA,SAAS,4BACP,OACyC;AACzC,MAAI;AACJ,QAAM,iBAA8D,CAAC;AACrE,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,mBAAmB,kBAAqC,SAAS;AACvE,QAAI,iBAAiB,UAAa,qBAAqB,QAAW;AAChE,qBAAe;AAAA,IACjB;AAEA,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,qBAAe,KAAK,WAAW;AAAA,IACjC;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAI,eAAe,EAAE,aAAa,IAAI,CAAC;AAAA,IACvC,GAAI,eAAe,SAAS,IAAI,EAAE,eAAe,IAAI,CAAC;AAAA,EACxD;AACF;AAEA,SAAS,uBAAuB,OAA0B;AACxD,QAAM,QAAQ,cAAc,KAAK;AACjC,QAAM,aAAuB,CAAC;AAC9B,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,QAAQ,OAAO;AACxB,oBAAgB,YAAY,MAAM,IAAI;AACtC,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,OAAO,CAAC;AAClE,oBAAgB,YAAY,MAAM,kBAAkB,MAAM,cAAc,CAAC;AAAA,EAC3E;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA0B;AAC/C,SAAO,CAAC,SAAS,KAAK,CAAC,EACpB,OAAO,SAAS,kBAAkB,OAAO,OAAO,CAAC,CAAC,EAClD,OAAO,CAAC,UAA2B,UAAU,MAAS;AAC3D;AAEA,SAAS,gBACP,YACA,MACA,OACA;AACA,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,KAAK,IAAI,KAAK,GAAG;AAC1D;AAAA,EACF;AAEA,OAAK,IAAI,KAAK;AACd,aAAW,KAAK,KAAK;AACvB;AAEA,SAAS,SAAS,OAAoC;AACpD,SAAO,SAAS,OAAO,UAAU,WAAW,QAAQ;AACtD;AAEA,SAAS,kBAAkB,OAAgB,KAAsB;AAC/D,SAAO,SAAS,OAAO,UAAU,WAC5B,MAAkC,GAAG,IACtC;AACN;AAEA,SAAS,kBACP,OACoD;AACpD,QAAM,YACJ,kBAAkB,OAAO,OAAO,KAAK,kBAAkB,OAAO,SAAS;AAEzE,SAAO,cAAc,SAAS,IACzB,YACD;AACN;AAEA,SAAS,mBACP,OACkD;AAClD,QAAM,YAAY,kBAAkB,OAAO,OAAO;AAClD,MAAI,iBAAiB,SAAS,GAAG;AAC/B,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEA,eAAe,SAOb,SACA,MAC+B;AAC/B,MAAI,QAAQ,KAAK;AACf,WAAO,QAAQ,IAAI,IAAI;AAAA,EACzB;AAEA,MAAI,iBAAqD,KAAK,KAAK,GAAG;AACpE,WAAO,KAAK,MAAM,IAAI,KAAK,OAAO,KAAK,OAAO;AAAA,EAChD;AAEA,QAAM,IAAI;AAAA,IACR;AAAA,EACF;AACF;AAEA,SAAS,mBAOP,SAOA;AACA,SAAO,QAAQ,UAAU;AAC3B;AAEA,SAAS,iBAMP,OACgE;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO;AAAA,EACT;AAEA,SACE,SAAS,SAAS,OAAQ,MAA4B,QAAQ;AAElE;AAEA,SAAS,cAAc,OAAsC;AAC3D,MAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAAG;AAC/D,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ,OAAO,OAAO,KAAK;AACjC,SACE,MAAM,SAAS,KACf,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,SAAS,iBAAiB,OAA4C;AACpE,SACE,MAAM,QAAQ,KAAK,KACnB,MAAM;AAAA,IAAM,CAAC,SACX;AAAA,MACE,QACE,OAAO,SAAS,YAChB,UAAU,QACV,OAAQ,KAA4B,SAAS,YAC7C,aAAa,QACb,OAAQ,KAA+B,YAAY;AAAA,IACvD;AAAA,EACF;AAEJ;AAEA,eAAe,2BAKb,OACA,UACA,MAQA,UACA;AACA,MAAI,CAAC,YAAY,SAAS,WAAW,GAAG;AACtC,WAAO,SAAS;AAAA,EAClB;AAEA,QAAM,qBAAqB,oBAAI,IAG7B;AACF,QAAM,iBAAiB,oBAAI,IAA+C;AAE1E,QAAM,YAAY,CAAC,SAA6C;AAC9D,QAAI,mBAAmB,IAAI,IAAI,GAAG;AAChC;AAAA,IACF;AAEA,UAAM,kBAAkB,2BAA2B,KAAK,OAAO;AAC/D,uBAAmB,IAAI,MAAM,eAAe;AAC5C,UAAM,sBAA4D,OAChE,YACA,YACG;AACH,YAAM,YAAY,oBAAI,KAAK;AAC3B,YAAM,cAAc;AAAA,QAClB,OAAO,KAAK;AAAA,QACZ,UAAU,KAAK,QAAQ;AAAA,QACvB,QAAQ,KAAK,QAAQ;AAAA,QACrB,aAAa,KAAK,QAAQ;AAAA,MAC5B;AACA,YAAM,kBAAkB;AAAA,QACtB,KAAK;AAAA,QACL,KAAK;AAAA,MACP;AAEA,UAAI;AACF,cAAM,YAAY,MAAM,4BAA4B;AAAA,UAClD,UAAU,KAAK;AAAA,UACf;AAAA,UACA,SAAS;AAAA,UACT,QAAQ,KAAK,aAAa,KAAK,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AACD,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,QAAQ,UAAU;AAAA,UAClB,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,UAAU,wBAAwB,UAAU,MAAM;AAAA,QACpD;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,SAAS,UAAU;AAAA,UACnB,UAAU;AAAA,YACR,MAAM,KAAK;AAAA,UACb;AAAA,QACF,CAAC;AACD,eAAO,UAAU;AAAA,MACnB,SAAS,OAAO;AACd,cAAM,aAAa,oBAAI,KAAK;AAC5B,cAAM,OAAO;AAAA,UACX,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,UACX,OAAO,uBAAuB,KAAK;AAAA,UACnC,WAAW,UAAU,YAAY;AAAA,UACjC,YAAY,WAAW,YAAY;AAAA,UACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,UACrD,UAAU,wBAAwB,2BAA2B,KAAK,CAAC;AAAA,QACrE;AACA,aAAK,UAAU,KAAK,IAAI;AACxB,aAAK,SAAS,KAAK;AAAA,UACjB,MAAM;AAAA,UACN,WAAW,CAAC,IAAI;AAAA,QAClB,CAAC;AACD,cAAM;AAAA,MACR,UAAE;AACA,wBAAgB;AAAA,MAClB;AAAA,IACF;AACA,wBAAoB,uBAAuB,IAAI;AAC/C,SAAK,UAAU;AAAA,EACjB;AAEA,QAAM,gBAAgB,CACpB,iBACG;AACH,eAAW,WAAW,cAAc;AAClC,iBAAW,QAAQ,SAAS;AAC1B,kBAAU,IAAI;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,gBAAc,QAAQ;AAEtB,aAAW,UAAU,qBAAqB,KAAK,GAAG;AAChD,UAAM,gBAAgB,OAAO;AAC7B,mBAAe,IAAI,QAAQ,aAAa;AACxC,WAAO,QAAQ,SAAS,gBAAuC,WAAW;AACxE,YAAM,cAAc,cAAc,MAAM,MAAM,SAAS;AAEvD,UAAI,cAAc,WAAW,GAAG;AAC9B,eAAO,YAAY,QAAQ,MAAM;AAC/B;AAAA,YACE,8BAAiD,KAAK;AAAA,UACxD;AAAA,QACF,CAAC;AAAA,MACH;AAEA,oBAAc,8BAAiD,KAAK,CAAC;AACrE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI;AACF,WAAO,MAAM,SAAS;AAAA,EACxB,UAAE;AACA,eAAW,CAAC,QAAQ,aAAa,KAAK,gBAAgB;AACpD,aAAO,QAAQ;AAAA,IACjB;AAEA,eAAW,CAAC,MAAM,eAAe,KAAK,oBAAoB;AACxD,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AACF;AAMA,SAAS,qBAAqB,OAAmC;AAC/D,SAAO,cAAc,KAAK,EAAE,OAAO,iBAAiB;AACtD;AAEA,SAAS,kBAAkB,OAAyC;AAClE,SAAO,WAAW,SAAS,OAAO,MAAM,UAAU;AACpD;AAEA,SAAS,8BAGP,OAA6D;AAC7D,QAAM,WAAwD,CAAC;AAC/D,QAAM,eAAe,oBAAI,IAA0C;AAEnE,aAAW,aAAa,uBAAuB,KAAK,GAAG;AACrD,UAAM,cAAc,mBAAmB,SAAS;AAChD,QAAI,eAAe,CAAC,aAAa,IAAI,WAAW,GAAG;AACjD,mBAAa,IAAI,WAAW;AAC5B,eAAS,KAAK,WAAW;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,OAA2C;AAChE,SAAO;AAAA,IACL,SAAS,OAAQ,MAA6B,SAAS;AAAA,EACzD;AACF;AAEA,SAAS,uBACP,OACsC;AACtC,QAAM,aAAa,eAAe,KAAK;AACvC,QAAM,EAAE,SAAS,MAAM,GAAG,QAAQ,IAAI;AAEtC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,SAAS,OAAO,YAAY,WAAW,UAAU,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,WAAW,EAAE,KAAK,IAAI,CAAC;AAAA,EAC7C;AACF;AAEA,SAAS,2BACP,SACA;AACA,QAAM,gBAAgB;AACtB,SAAO,cAAc,uBAAuB,KAAK;AACnD;AAEA,SAAS,6BAAmD;AAC1D,SAAO;AAAA,IACL,uBAAuB,oBAAI,IAAI;AAAA,EACjC;AACF;AAEA,SAAS,yBACP,OACA,UACA;AACA,QAAM,sBAAsB;AAAA,IAC1B;AAAA,KACC,MAAM,sBAAsB,IAAI,QAAQ,KAAK,KAAK;AAAA,EACrD;AAEA,SAAO,MAAM;AACX,UAAM,aAAa,MAAM,sBAAsB,IAAI,QAAQ,KAAK,KAAK;AACrE,QAAI,aAAa,GAAG;AAClB,YAAM,sBAAsB,OAAO,QAAQ;AAC3C;AAAA,IACF;AAEA,UAAM,sBAAsB,IAAI,UAAU,SAAS;AAAA,EACrD;AACF;AAEA,SAAS,6BACP,OACA,UACA;AACA,UAAQ,MAAM,sBAAsB,IAAI,QAAQ,KAAK,KAAK;AAC5D;AAEA,eAAe,4BAGb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAOG;AACD,MAAI,aAAa;AACjB,MAAI;AAEJ,QAAM,YAAY,MAAM,kBAAkB;AAAA,IACxC,UAAU,2BAA2B,QAAQ;AAAA,IAC7C;AAAA,IACA;AAAA,IACA,SAAS,OAAO,aAAa;AAC3B,mBAAa;AACb,mBAAa,MAAM,QAAQ,YAAY,QAAQ;AAC/C,aAAO,+BAA+B,UAAU;AAAA,IAClD;AAAA,IACA;AAAA,EACF,CAAC;AAED,MAAI,YAAY;AACd,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,kBAAkB,0BAA0B,UAAU;AAAA,MACtD,QAAQ,UAAU;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG,8BAA8B,UAAU,MAAM;AAAA,IACjD,QAAQ,UAAU;AAAA,EACpB;AACF;AAEA,SAAS,2BAA2B,UAAkB;AACpD,SAAO,GAAG,QAAQ;AACpB;AAEA,SAAS,cAIP;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GASE;AACA,QAAM,YAA8B,CAAC;AACrC,QAAM,YAA2B;AAAA,IAC/B,SAAS,CAAC,YAAY;AACpB,eAAS,KAAK,OAAO;AAAA,IACvB;AAAA,IACA,QAAQ,CAAC,SAAS,aAAa;AAC7B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,SAAS,aAAa;AAC3B,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,WAAW,CAAC,SAAS,aAAa;AAChC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA,MAAM,CAAC,MAAM,SAAS,aAAa;AACjC,eAAS,KAAK;AAAA,QACZ,MAAM;AAAA,QACN;AAAA,QACA,UAAU;AAAA,UACR;AAAA,UACA,GAAI,YAAY,CAAC;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,eAAe,OAAO;AAAA,IAC1B,OAAO,QAAQ,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM;AAAA,MACpD;AAAA,MACA,OAAO,SAAoC;AACzC,cAAM,YAAY,oBAAI,KAAK;AAC3B,cAAM,6BAA6B;AAAA,UACjC;AAAA,UACA;AAAA,QACF;AACA,cAAM,cAAc;AAAA,UAClB;AAAA,UACA,UAAU,QAAQ;AAAA,UAClB,QAAQ,QAAQ;AAAA,UAChB,aAAa,QAAQ;AAAA,QACvB;AAEA,YAAI;AACF,gBAAM,YAAY,MAAM,sBAAsB;AAAA,YAC5C;AAAA,YACA;AAAA,YACA,QAAQ,6BACJ,SACA,aAAa,QAAQ;AAAA,YACzB;AAAA,YACA,SAAS;AAAA,UACX,CAAC;AACD,gBAAM,aAAa,oBAAI,KAAK;AAE5B,cAAI,4BAA4B;AAC9B,mBAAO,UAAU;AAAA,UACnB;AAEA,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,QAAQ,UAAU;AAAA,YAClB,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,UAAU,wBAAwB,UAAU,MAAM;AAAA,UACpD;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,SAAS,UAAU;AAAA,YACnB,UAAU;AAAA,cACR,MAAM;AAAA,YACR;AAAA,UACF,CAAC;AACD,iBAAO,UAAU;AAAA,QACnB,SAAS,OAAO;AACd,gBAAM,aAAa,oBAAI,KAAK;AAC5B,cAAI,4BAA4B;AAC9B,kBAAM;AAAA,UACR;AAEA,gBAAM,OAAO;AAAA,YACX,MAAM;AAAA,YACN,WAAW;AAAA,YACX,OAAO,uBAAuB,KAAK;AAAA,YACnC,WAAW,UAAU,YAAY;AAAA,YACjC,YAAY,WAAW,YAAY;AAAA,YACnC,YAAY,WAAW,QAAQ,IAAI,UAAU,QAAQ;AAAA,YACrD,UAAU;AAAA,cACR,2BAA2B,KAAK;AAAA,YAClC;AAAA,UACF;AACA,oBAAU,KAAK,IAAI;AACnB,mBAAS,KAAK;AAAA,YACZ,MAAM;AAAA,YACN,WAAW,CAAC,IAAI;AAAA,UAClB,CAAC;AACD,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,QAAQ,QAAQ;AAAA,IAChB;AAAA,EACF;AACF;AAEA,SAAS,cAAc,QAAwC;AAC7D,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,WAAO,YAAY,MAAM;AAAA,EAC3B;AAEA,QAAM,aAAa;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,aAAW,OAAO,YAAY;AAC5B,UAAM,QAAS,OAAmC,GAAG;AACrD,UAAM,aAAa,YAAY,KAAK;AACpC,QAAI,eAAe,QAAW;AAC5B,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,oBAAoB,QAAwC;AACnE,QAAM,UACJ,UAAU,OAAO,WAAW,WACxB,YAAa,OAAiC,OAAO,IACrD;AACN,MAAI,YAAY,QAAW;AACzB,WAAO;AAAA,EACT;AAEA,SACE,YAAY,MAAM,MAAM,WAAW,SAAY,SAAY,OAAO,MAAM;AAE5E;AAEA,SAAS,0BAA0B,QAA4B;AAC7D,SAAO,oBAAoB,MAAM,KAAK;AACxC;AAoBA,SAAS,+BACP,QAC0B;AAC1B,QAAM,mBAAmB,0BAA0B,MAAM;AAEzD,SAAO;AAAA,IACL,eAAe;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,IACX;AAAA,IACA,aAAa,YAAY,MAAM,KAAK;AAAA,IACpC;AAAA,EACF;AACF;AAEA,SAAS,8BAA8B,QAAmB;AACxD,MAAI,2BAA2B,MAAM,GAAG;AACtC,WAAO;AAAA,MACL,QAAQ,OAAO;AAAA,MACf,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,iCAAiC,MAAM,GAAG;AAC5C,WAAO;AAAA,MACL,QAAQ,OAAO,eAAe,OAAO;AAAA,MACrC,kBAAkB,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,kBAAkB,0BAA0B,MAAM;AAAA,EACpD;AACF;AAEA,SAAS,2BACP,OACmC;AACnC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,iBAAiB,SACjB,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,yBACP,OACoD;AACpD,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,iCACP,OACyC;AACzC,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,mBAAmB,SACnB;AAAA,MACG,MAAsC;AAAA,IACzC,KACA,sBAAsB;AAAA,EAC1B;AACF;AAEA,SAAS,+BACP,OAC0D;AAC1D,SAAO;AAAA,IACL,SACE,OAAO,UAAU,YACjB,UAAU,SACT,MAA6B,SAAS,8BACvC,aAAa,SACZ,MAAgC,YAAY;AAAA,EACjD;AACF;AAEA,SAAS,aAAa,QAAiB,eAAqC;AAC1E,MAAI,CAAC,UAAU,OAAO,WAAW,UAAU;AACzC,WAAO,gBAAgB,IAAI,EAAE,WAAW,cAAc,IAAI,CAAC;AAAA,EAC7D;AAEA,QAAM,aACH,OAAmC,SACnC,OAAmC;AAEtC,QAAM,QACJ,cAAc,OAAO,eAAe,WAC/B,EAAE,GAAI,WAAuC,IAC9C,CAAC;AAEP,MAAI,MAAM,cAAc,UAAa,gBAAgB,GAAG;AACtD,UAAM,YAAY;AAAA,EACpB;AAEA,SAAO;AACT;AAEA,SAAS,eACP,QACA,UACA,QACA,OACmB;AACnB,MACE;AAAA,IACG,QAAgD;AAAA,EACnD,GACA;AACA,WAAQ,OAA0C;AAAA,EACpD;AAEA,MACE,oBAAqB,QAAgD,KAAK,GAC1E;AACA,WAAQ,OAAwC;AAAA,EAClD;AAEA,QAAM,kBAAkB,CAAC,GAAG,QAAQ;AACpC,MACE,WAAW,UACX,CAAC,gBAAgB;AAAA,IACf,CAAC,YACC,QAAQ,SAAS,eAAe,QAAQ,YAAY;AAAA,EACxD,GACA;AACA,oBAAgB,KAAK;AAAA,MACnB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,UAAU;AAAA,IACV,YAAY,OAAO,WAAW,WAAW,SAAS;AAAA,IAClD,UACI,QAAgD,YAEhC,MAAM;AAAA,IAC1B,OACI,QAAgD,SAEhC,MAAM;AAAA,EAC5B;AACF;AAEA,SAAS,cAAc,QAAmD;AACxE,SAAO,wBAAwB,MAAM;AACvC;AAEA,eAAe,sBAKb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAMG;AACD,SAAO,kBAIL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,CAAC,UAAU,gBAClB,KAAK,QAAQ,UAAmB,WAAW;AAAA,IAC7C;AAAA,EACF,CAAC;AACH;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vitest-evals/harness-pi-ai",
-  "version": "0.9.0-beta.1",
+  "version": "0.9.0-beta.3",
   "sideEffects": false,
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -27,7 +27,7 @@
   "devDependencies": {
     "@mariozechner/pi-agent-core": "^0.67.68",
     "@mariozechner/pi-ai": "^0.67.68",
-    "vitest-evals": "0.9.0-beta.1"
+    "vitest-evals": "0.9.0-beta.3"
   },
   "scripts": {
     "build": "tsup --config ./tsup.config.ts"