@vitest-evals/harness-pi-ai 0.9.0-beta.2 → 0.9.0-beta.4

package/README.md

@@ -13,14 +13,18 @@ npm install -D vitest-evals @vitest-evals/harness-pi-ai
 ```ts
 import { expect } from "vitest";
 import { piAiHarness } from "@vitest-evals/harness-pi-ai";
-import { describeEval, toolCalls } from "vitest-evals";
+import {
+  createJudge,
+  describeEval,
+  toolCalls,
+  type JudgeContext,
+} from "vitest-evals";
 
 const harness = piAiHarness({
-  createAgent: () => createRefundAgent(),
+  agent: () => createRefundAgent(),
   toolReplay: {
     lookupInvoice: true,
   },
-  prompt: sharedJudgePrompt,
 });
 
 describeEval("refund agent", { harness }, (it) => {
@@ -38,8 +42,25 @@ describeEval("refund agent", { harness }, (it) => {
 });
 ```
 
-`prompt` gives rubric or factuality judges the same provider/model setup
-through `JudgeContext.harness.prompt`.
+`run` executes the Pi agent under test. Judges are created separately; keep
+judge prompts and model calls in the judge instead of putting a judge model
+call on the app harness.
+
+```ts
+const RefundRubricJudge = createJudge(
+  "RefundRubricJudge",
+  async (ctx: JudgeContext<string, RefundDecision>) => {
+    const verdict = await queryRefundJudgeModel({
+      prompt: formatJudgePrompt({
+        input: ctx.input,
+        output: ctx.output,
+      }),
+    });
+
+    return parseJudgeVerdict(verdict);
+  },
+);
+```
 
 If the agent already exposes its own tools, the adapter will infer them from
 the agent by default. If your existing Pi Mono agent already exposes its own
@@ -48,8 +69,7 @@ seam:
 
 ```ts
 const harness = piAiHarness({
-  createAgent: () => createRefundAgent(),
-  prompt: sharedJudgePrompt,
+  agent: () => createRefundAgent(),
   run: ({ agent, input, runtime }) => agent.execute(input, runtime),
 });
 ```
@@ -57,42 +77,59 @@ const harness = piAiHarness({
 If the agent already implements `run(input, runtime)`, you can omit `run` and
 the harness will call that method automatically.
 
+`agent` can be an object or a per-run factory. The factory receives the per-run
+input and harness context before the adapter infers and instruments native
+agent tools. Use that for scenario-specific instructions, tool closures, or
+metadata without leaving the default replay path:
+
+```ts
+const harness = piAiHarness({
+  agent: ({ input, context }) =>
+    createRefundAgent({
+      instructions: buildInstructions(input),
+      metadata: context.metadata,
+    }),
+  toolReplay: {
+    lookupInvoice: true,
+  },
+});
+```
+
 You should not need to configure output/session/usage basics for the normal Pi
 path. Pass your agent and let the adapter infer both the toolset and the
-normalized result from common Pi-style return values such as `decision` or
-`output`.
+normalized result from the agent transcript. When your app returns a structured
+domain value, return it as `output`.
 
 If the agent completely hides its tools, `tools` still exists as a low-level
 override:
 
 ```ts
 const harness = piAiHarness({
-  createAgent: () => createRefundAgent(),
-  prompt: sharedJudgePrompt,
+  agent: () => createRefundAgent(),
   tools: hiddenAgentTools,
 });
 ```
 
-If you do have an unusual wrapper around the agent result, low-level
-normalization hooks still exist under `normalize`:
+If you do have an unusual wrapper around the agent result, make the app-facing
+value explicit in the `run()` return:
 
 ```ts
 const harness = piAiHarness({
-  createAgent: () => createWrappedRefundAgent(),
-  prompt: sharedJudgePrompt,
-  run: ({ agent, input, runtime }) => agent.run(input, runtime),
-  normalize: {
-    output: ({ result }) => result.customDecision,
-  },
+  agent: () => createWrappedRefundAgent(),
+  run: async ({ agent, input, runtime }) => ({
+    output: await agent.run(input, runtime),
+  }),
 });
 ```
 
 The adapter provides:
 
 - a runtime/tool injection seam for an existing agent
-- a required prompt seam for LLM-backed judges
 - normalized session capture from emitted events and wrapped tool calls
-- usage/output inference for common `pi-ai`-style result objects
+- usage capture plus typed app output from `run()` results that return `output`
+- native app output is accepted only when it is already JSON-safe; arbitrary
+  fields, primitive raw results, and non-JSON values require an explicit
+  `output` selector
 - opt-in tool replay/recording from harness-level `toolReplay`
 
 See the workspace demo in `apps/demo-pi`.
@@ -118,11 +155,10 @@ Then opt individual tools into recording/replay from the harness:
 
 ```ts
 const harness = piAiHarness({
-  createAgent: () => createRefundAgent(),
+  agent: () => createRefundAgent(),
   toolReplay: {
     lookupInvoice: true,
   },
-  prompt: sharedJudgePrompt,
 });
 ```
 

package/dist/index.d.mts

@@ -1,10 +1,19 @@
-import { NormalizedMessage, JsonValue, HarnessMetadata, HarnessContext, NormalizedSession, UsageSummary, TimingSummary, HarnessPrompt, HarnessRun, Harness } from 'vitest-evals/harness';
+import { NormalizedMessage, JsonValue, HarnessMetadata, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
 
 type MaybePromise<T> = T | Promise<T>;
+type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
+type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
+type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: PiAiCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
 type AnyPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;
 type InferredPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>>;
+type PiAiResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : undefined;
+type PiAiAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata>> = TAgent extends {
+    run: (input: TInput, runtime: PiAiRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
+} ? Awaited<TResult> : unknown;
+/** Replay mode alias used by the Pi AI harness package. */
 type PiAiReplayMode = ReplayMode;
+/** Event sink for recording Pi AI transcript messages. */
 interface PiAiEventSink {
     message: (message: NormalizedMessage) => void;
     system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
@@ -12,23 +21,31 @@ interface PiAiEventSink {
     assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
     tool: (name: string, content: JsonValue, metadata?: Record<string, JsonValue>) => void;
 }
+/** Context passed to instrumented Pi AI tool executions. */
 interface PiAiToolContext<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
     input: TInput;
     metadata: HarnessContext<TMetadata>["metadata"];
     signal?: AbortSignal;
     setArtifact: HarnessContext<TMetadata>["setArtifact"];
 }
+/** Tool replay recording shape for Pi AI tools. */
 type PiAiToolRecording<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
+/** Replay configuration for a Pi AI tool. */
 type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;
+/** Replay policy for one Pi AI tool. */
 type PiAiToolReplayPolicy<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = boolean | PiAiToolReplayConfig<Record<string, JsonValue>, JsonValue, TInput, TMetadata>;
+/** Replay policy map keyed by Pi AI tool name. */
 type PiAiToolReplayPolicies<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolReplayPolicy<TInput, TMetadata>>;
+/** Tool definition accepted by the Pi AI harness runtime. */
 interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
     description?: string;
     execute: (args: TArgs, context: PiAiToolContext<TInput, TMetadata>) => MaybePromise<TResult>;
 }
+/** Toolset shape accepted by the Pi AI harness. */
 type PiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = AnyPiAiToolset<TInput, TMetadata>;
 type ToolArgs<TTool> = TTool extends PiAiToolDefinition<infer TArgs, infer _TResult, infer _TInput, infer _TMetadata> ? TArgs : never;
 type ToolResult<TTool> = TTool extends PiAiToolDefinition<infer _TArgs, infer TResult, infer _TInput, infer _TMetadata> ? TResult : never;
+/** Runtime object passed into Pi AI agent entrypoints. */
 type PiAiRuntime<TTools extends PiAiToolset<TInput, TMetadata>, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = {
     tools: {
         [K in keyof TTools]: (args: ToolArgs<TTools[K]>) => Promise<ToolResult<TTools[K]>>;
@@ -36,41 +53,69 @@ type PiAiRuntime<TTools extends PiAiToolset<TInput, TMetadata>, TInput = string,
     events: PiAiEventSink;
     signal?: AbortSignal;
 };
+/** Arguments passed to custom Pi AI harness run callbacks. */
 interface PiAiHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata>> {
     agent: TAgent;
     input: TInput;
     context: HarnessContext<TMetadata>;
     runtime: PiAiRuntime<TTools, TInput, TMetadata>;
 }
+/** Arguments passed to Pi AI harness output selectors. */
 interface PiAiHarnessResultArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends PiAiToolset<TInput, TMetadata>> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {
     result: TResult;
 }
-interface PiAiHarnessBaseOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> {
-    agent?: TAgent;
-    createAgent?: () => MaybePromise<TAgent>;
+/** Arguments passed to per-run Pi AI agent factories. */
+interface PiAiCreateAgentArgs<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
+    input: TInput;
+    context: HarnessContext<TMetadata>;
+}
+interface PiAiHarnessBaseOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> {
+    agent: AgentSource<TAgent, TInput, TMetadata>;
     toolReplay?: PiAiToolReplayPolicies<TInput, TMetadata>;
-    normalize?: PiAiHarnessNormalizeOptions<TAgent, TInput, TMetadata, TResult, TTools>;
-    prompt: HarnessPrompt;
+    output?: PiAiHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
     name?: string;
 }
-interface PiAiHarnessWithToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> {
+interface PiAiHarnessWithToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> {
     tools: TTools;
-    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun>;
+    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun<TOutput>>;
 }
-interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>> {
+interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>, TOutput> {
     tools?: undefined;
-    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>) => MaybePromise<TResult | HarnessRun>;
-}
-type PiAiHarnessOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> = PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools> | PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>;
-interface PiAiHarnessNormalizeOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> {
-    session?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<NormalizedSession>;
-    output?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<JsonValue | undefined>;
-    usage?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<UsageSummary>;
-    timings?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TimingSummary | undefined>;
-    errors?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<Array<Record<string, JsonValue>>>;
+    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>) => MaybePromise<TResult | HarnessRun<TOutput>>;
 }
+type PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> & {
+    output: PiAiHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
+};
+type PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult, TOutput> & {
+    output: PiAiHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>, TOutput>;
+};
+type PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, JsonValue | undefined> & {
+    tools: TTools;
+    run: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult>;
+    output?: never;
+};
+type PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, unknown, TTools, JsonValue | undefined> & {
+    tools: TTools;
+    run?: never;
+    output?: never;
+};
+type PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>, JsonValue | undefined> & {
+    tools?: undefined;
+    run: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>) => MaybePromise<TResult>;
+    output?: never;
+};
+type PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, unknown, InferredPiAiToolset<TInput, TMetadata>, JsonValue | undefined> & {
+    tools?: undefined;
+    run?: never;
+    output?: never;
+};
+type PiAiHarnessOutputSelector<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TOutput>;
 /** Adapts a Pi agent runtime into a normalized vitest-evals harness. */
-declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, TMetadata>;
-declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>): Harness<TInput, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput, TMetadata, TResult, TTools, TOutput>): Harness<TInput, TOutput, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput, TMetadata, TResult, TOutput>): Harness<TInput, TOutput, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput, TMetadata, TTools>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TMetadata, TTools>>, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata>(options: PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput, TMetadata>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>>, TMetadata>;
 
-export { type PiAiEventSink, type PiAiHarnessInferredToolsOptions, type PiAiHarnessNormalizeOptions, type PiAiHarnessOptions, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiHarnessWithToolsOptions, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };
+export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };

package/dist/index.d.ts

@@ -1,10 +1,19 @@
-import { NormalizedMessage, JsonValue, HarnessMetadata, HarnessContext, NormalizedSession, UsageSummary, TimingSummary, HarnessPrompt, HarnessRun, Harness } from 'vitest-evals/harness';
+import { NormalizedMessage, JsonValue, HarnessMetadata, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
 
 type MaybePromise<T> = T | Promise<T>;
+type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
+type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
+type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: PiAiCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
 type AnyPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;
 type InferredPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>>;
+type PiAiResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : undefined;
+type PiAiAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata>> = TAgent extends {
+    run: (input: TInput, runtime: PiAiRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
+} ? Awaited<TResult> : unknown;
+/** Replay mode alias used by the Pi AI harness package. */
 type PiAiReplayMode = ReplayMode;
+/** Event sink for recording Pi AI transcript messages. */
 interface PiAiEventSink {
     message: (message: NormalizedMessage) => void;
     system: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
@@ -12,23 +21,31 @@ interface PiAiEventSink {
     assistant: (content: JsonValue, metadata?: Record<string, JsonValue>) => void;
     tool: (name: string, content: JsonValue, metadata?: Record<string, JsonValue>) => void;
 }
+/** Context passed to instrumented Pi AI tool executions. */
 interface PiAiToolContext<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
     input: TInput;
     metadata: HarnessContext<TMetadata>["metadata"];
     signal?: AbortSignal;
     setArtifact: HarnessContext<TMetadata>["setArtifact"];
 }
+/** Tool replay recording shape for Pi AI tools. */
 type PiAiToolRecording<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
+/** Replay configuration for a Pi AI tool. */
 type PiAiToolReplayConfig<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = ToolReplayConfig<TArgs, TResult, PiAiToolContext<TInput, TMetadata>>;
+/** Replay policy for one Pi AI tool. */
 type PiAiToolReplayPolicy<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = boolean | PiAiToolReplayConfig<Record<string, JsonValue>, JsonValue, TInput, TMetadata>;
+/** Replay policy map keyed by Pi AI tool name. */
 type PiAiToolReplayPolicies<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolReplayPolicy<TInput, TMetadata>>;
+/** Tool definition accepted by the Pi AI harness runtime. */
 interface PiAiToolDefinition<TArgs extends Record<string, JsonValue> = Record<string, JsonValue>, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
     description?: string;
     execute: (args: TArgs, context: PiAiToolContext<TInput, TMetadata>) => MaybePromise<TResult>;
 }
+/** Toolset shape accepted by the Pi AI harness. */
 type PiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = AnyPiAiToolset<TInput, TMetadata>;
 type ToolArgs<TTool> = TTool extends PiAiToolDefinition<infer TArgs, infer _TResult, infer _TInput, infer _TMetadata> ? TArgs : never;
 type ToolResult<TTool> = TTool extends PiAiToolDefinition<infer _TArgs, infer TResult, infer _TInput, infer _TMetadata> ? TResult : never;
+/** Runtime object passed into Pi AI agent entrypoints. */
 type PiAiRuntime<TTools extends PiAiToolset<TInput, TMetadata>, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = {
     tools: {
         [K in keyof TTools]: (args: ToolArgs<TTools[K]>) => Promise<ToolResult<TTools[K]>>;
@@ -36,41 +53,69 @@ type PiAiRuntime<TTools extends PiAiToolset<TInput, TMetadata>, TInput = string,
     events: PiAiEventSink;
     signal?: AbortSignal;
 };
+/** Arguments passed to custom Pi AI harness run callbacks. */
 interface PiAiHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata>> {
     agent: TAgent;
     input: TInput;
     context: HarnessContext<TMetadata>;
     runtime: PiAiRuntime<TTools, TInput, TMetadata>;
 }
+/** Arguments passed to Pi AI harness output selectors. */
 interface PiAiHarnessResultArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends PiAiToolset<TInput, TMetadata>> extends PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {
     result: TResult;
 }
-interface PiAiHarnessBaseOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> {
-    agent?: TAgent;
-    createAgent?: () => MaybePromise<TAgent>;
+/** Arguments passed to per-run Pi AI agent factories. */
+interface PiAiCreateAgentArgs<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
+    input: TInput;
+    context: HarnessContext<TMetadata>;
+}
+interface PiAiHarnessBaseOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> {
+    agent: AgentSource<TAgent, TInput, TMetadata>;
     toolReplay?: PiAiToolReplayPolicies<TInput, TMetadata>;
-    normalize?: PiAiHarnessNormalizeOptions<TAgent, TInput, TMetadata, TResult, TTools>;
-    prompt: HarnessPrompt;
+    output?: PiAiHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
     name?: string;
 }
-interface PiAiHarnessWithToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> {
+interface PiAiHarnessWithToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> {
     tools: TTools;
-    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun>;
+    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun<TOutput>>;
 }
-interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>> {
+interface PiAiHarnessInferredToolsOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> extends PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>, TOutput> {
     tools?: undefined;
-    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>) => MaybePromise<TResult | HarnessRun>;
-}
-type PiAiHarnessOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> = PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools> | PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>;
-interface PiAiHarnessNormalizeOptions<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> {
-    session?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<NormalizedSession>;
-    output?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<JsonValue | undefined>;
-    usage?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<UsageSummary>;
-    timings?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TimingSummary | undefined>;
-    errors?: (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<Array<Record<string, JsonValue>>>;
+    run?: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>) => MaybePromise<TResult | HarnessRun<TOutput>>;
 }
+type PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> & {
+    output: PiAiHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
+};
+type PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined> = PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult, TOutput> & {
+    output: PiAiHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>, TOutput>;
+};
+type PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, JsonValue | undefined> & {
+    tools: TTools;
+    run: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult>;
+    output?: never;
+};
+type PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, unknown, TTools, JsonValue | undefined> & {
+    tools: TTools;
+    run?: never;
+    output?: never;
+};
+type PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, InferredPiAiToolset<TInput, TMetadata>, JsonValue | undefined> & {
+    tools?: undefined;
+    run: (args: PiAiHarnessRunArgs<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>) => MaybePromise<TResult>;
+    output?: never;
+};
+type PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = PiAiHarnessBaseOptions<TAgent, TInput, TMetadata, unknown, InferredPiAiToolset<TInput, TMetadata>, JsonValue | undefined> & {
+    tools?: undefined;
+    run?: never;
+    output?: never;
+};
+type PiAiHarnessOutputSelector<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = (args: PiAiHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TOutput>;
 /** Adapts a Pi agent runtime into a normalized vitest-evals harness. */
-declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsOptions<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, TMetadata>;
-declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsOptions<TAgent, TInput, TMetadata, TResult>): Harness<TInput, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessWithToolsOptionsWithOutput<TAgent, TInput, TMetadata, TResult, TTools, TOutput>): Harness<TInput, TOutput, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: PiAiHarnessInferredToolsOptionsWithOutput<TAgent, TInput, TMetadata, TResult, TOutput>): Harness<TInput, TOutput, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata> = PiAiToolset<TInput, TMetadata>>(options: PiAiHarnessWithToolsAgentOptionsWithoutOutput<TAgent, TInput, TMetadata, TTools>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TMetadata, TTools>>, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>, TMetadata>;
+declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata>(options: PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput, TMetadata>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>>, TMetadata>;
 
-export { type PiAiEventSink, type PiAiHarnessInferredToolsOptions, type PiAiHarnessNormalizeOptions, type PiAiHarnessOptions, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiHarnessWithToolsOptions, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };
+export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };