@vitest-evals/harness-pi-ai 0.9.0 → 0.11.0

package/README.md

@@ -43,23 +43,24 @@ describeEval("refund agent", { harness }, (it) => {
 ```
 
 `run` executes the Pi agent under test. Judges are created separately; keep
-judge prompts and model calls in the judge instead of putting a judge model
-call on the app harness.
+judge prompts on the judge and model calls on a judge harness instead of
+putting a judge model call on the app harness.
 
 ```ts
-const RefundRubricJudge = createJudge(
-  "RefundRubricJudge",
-  async (ctx: JudgeContext<string, RefundDecision>) => {
-    const verdict = await queryRefundJudgeModel({
-      prompt: formatJudgePrompt({
-        input: ctx.input,
-        output: ctx.output,
-      }),
-    });
+import { getModel } from "@mariozechner/pi-ai";
+import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
+import { describeEval, FactualityJudge } from "vitest-evals";
 
-    return parseJudgeVerdict(verdict);
-  },
-);
+const judgeHarness = piAiJudgeHarness({
+  model: getModel("anthropic", "claude-sonnet-4-5"),
+  temperature: 0,
+});
+const factualityJudge = FactualityJudge({ judgeHarness });
+
+describeEval("refund agent", {
+  harness,
+  judges: [factualityJudge],
+});
 ```
 
 If the agent already exposes its own tools, the adapter will infer them from
@@ -170,7 +171,7 @@ overwrite the native recording.
 
 Supported modes:
 
+- `auto` (default): replay when present, otherwise call live and write a recording
+- `record`: always call live and overwrite the recording
 - `off`: never read or write recordings
-- `auto`: replay when present, otherwise call live and write a recording
 - `strict`: require an existing recording and fail if it is missing
-- `record`: always call live and overwrite the recording

package/dist/index.d.mts

@@ -1,11 +1,61 @@
+import * as vitest_evals_judges from 'vitest-evals/judges';
 import { NormalizedMessage, JsonValue, HarnessMetadata, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
+import { Api, Model, SimpleStreamOptions } from '@mariozechner/pi-ai';
 
 type MaybePromise<T> = T | Promise<T>;
 type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
 type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
 type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: PiAiCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
 type AnyPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;
+/**
+ * Configuration for adapting a Pi AI model into a judge harness.
+ *
+ * @example
+ * ```ts
+ * import { getModel } from "@mariozechner/pi-ai";
+ * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
+ *
+ * const judgeHarness = piAiJudgeHarness({
+ *   model: getModel("anthropic", "claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ * ```
+ */
+interface PiAiJudgeHarnessConfig<TApi extends Api = Api> {
+    /** Pi AI model used for judge prompts. */
+    model: Model<TApi>;
+    /** Optional display name for diagnostics. */
+    name?: string;
+    /** Optional judge-model temperature. */
+    temperature?: number;
+    /** Optional judge-model token cap. */
+    maxOutputTokens?: number;
+    /** Additional Pi AI stream options forwarded to `completeSimple(...)`. */
+    options?: Omit<SimpleStreamOptions, "signal" | "temperature" | "maxTokens">;
+}
+/**
+ * Adapts a Pi AI model into the provider-neutral judge harness API.
+ *
+ * @example
+ * ```ts
+ * import { getModel } from "@mariozechner/pi-ai";
+ * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
+ * import { describeEval, FactualityJudge } from "vitest-evals";
+ *
+ * const judgeHarness = piAiJudgeHarness({
+ *   model: getModel("anthropic", "claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ *
+ * describeEval("qa agent", {
+ *   harness: qaHarness,
+ *   judgeHarness,
+ *   judges: [FactualityJudge()],
+ * }, (it) => {});
+ * ```
+ */
+declare function piAiJudgeHarness<TApi extends Api>(config: PiAiJudgeHarnessConfig<TApi>): vitest_evals_judges.JudgeHarness;
 type InferredPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>>;
 type PiAiResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : undefined;
 type PiAiAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata>> = TAgent extends {
@@ -118,4 +168,4 @@ declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessM
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>, TMetadata>;
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata>(options: PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput, TMetadata>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>>, TMetadata>;
 
-export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };
+export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiJudgeHarnessConfig, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness, piAiJudgeHarness };

package/dist/index.d.ts

@@ -1,11 +1,61 @@
+import * as vitest_evals_judges from 'vitest-evals/judges';
 import { NormalizedMessage, JsonValue, HarnessMetadata, HarnessContext, HarnessRun, Harness } from 'vitest-evals/harness';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
+import { Api, Model, SimpleStreamOptions } from '@mariozechner/pi-ai';
 
 type MaybePromise<T> = T | Promise<T>;
 type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
 type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
 type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: PiAiCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
 type AnyPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<any, any, TInput, TMetadata>>;
+/**
+ * Configuration for adapting a Pi AI model into a judge harness.
+ *
+ * @example
+ * ```ts
+ * import { getModel } from "@mariozechner/pi-ai";
+ * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
+ *
+ * const judgeHarness = piAiJudgeHarness({
+ *   model: getModel("anthropic", "claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ * ```
+ */
+interface PiAiJudgeHarnessConfig<TApi extends Api = Api> {
+    /** Pi AI model used for judge prompts. */
+    model: Model<TApi>;
+    /** Optional display name for diagnostics. */
+    name?: string;
+    /** Optional judge-model temperature. */
+    temperature?: number;
+    /** Optional judge-model token cap. */
+    maxOutputTokens?: number;
+    /** Additional Pi AI stream options forwarded to `completeSimple(...)`. */
+    options?: Omit<SimpleStreamOptions, "signal" | "temperature" | "maxTokens">;
+}
+/**
+ * Adapts a Pi AI model into the provider-neutral judge harness API.
+ *
+ * @example
+ * ```ts
+ * import { getModel } from "@mariozechner/pi-ai";
+ * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
+ * import { describeEval, FactualityJudge } from "vitest-evals";
+ *
+ * const judgeHarness = piAiJudgeHarness({
+ *   model: getModel("anthropic", "claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ *
+ * describeEval("qa agent", {
+ *   harness: qaHarness,
+ *   judgeHarness,
+ *   judges: [FactualityJudge()],
+ * }, (it) => {});
+ * ```
+ */
+declare function piAiJudgeHarness<TApi extends Api>(config: PiAiJudgeHarnessConfig<TApi>): vitest_evals_judges.JudgeHarness;
 type InferredPiAiToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, PiAiToolDefinition<Record<string, JsonValue>, JsonValue, TInput, TMetadata>>;
 type PiAiResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : undefined;
 type PiAiAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends PiAiToolset<TInput, TMetadata>> = TAgent extends {
@@ -118,4 +168,4 @@ declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessM
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown>(options: PiAiHarnessInferredToolsRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult>): Harness<TInput, PiAiResultOutput<Awaited<TResult>>, TMetadata>;
 declare function piAiHarness<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata>(options: PiAiHarnessInferredToolsAgentOptionsWithoutOutput<TAgent, TInput, TMetadata>): Harness<TInput, PiAiResultOutput<PiAiAgentResult<TAgent, TInput, TMetadata, InferredPiAiToolset<TInput, TMetadata>>>, TMetadata>;
 
-export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness };
+export { type PiAiCreateAgentArgs, type PiAiEventSink, type PiAiHarnessResultArgs, type PiAiHarnessRunArgs, type PiAiJudgeHarnessConfig, type PiAiReplayMode, type PiAiRuntime, type PiAiToolContext, type PiAiToolDefinition, type PiAiToolRecording, type PiAiToolReplayConfig, type PiAiToolReplayPolicies, type PiAiToolReplayPolicy, type PiAiToolset, piAiHarness, piAiJudgeHarness };

package/dist/index.js

@@ -20,54 +20,129 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
-  piAiHarness: () => piAiHarness
+  piAiHarness: () => piAiHarness,
+  piAiJudgeHarness: () => piAiJudgeHarness
 });
 module.exports = __toCommonJS(index_exports);
 var import_harness = require("vitest-evals/harness");
 var import_replay = require("vitest-evals/replay");
+var import_judges = require("vitest-evals/judges");
+var import_pi_ai = require("@mariozechner/pi-ai");
+var nextTraceId = 0;
+function piAiJudgeHarness(config) {
+  return (0, import_judges.createJudgeHarness)({
+    name: config.name ?? "pi-ai",
+    run: async ({ responseFormat, system, prompt }, { signal }) => {
+      const message = await (0, import_pi_ai.completeSimple)(
+        config.model,
+        {
+          systemPrompt: formatPiAiJudgeSystemPrompt(system, responseFormat),
+          messages: [
+            {
+              role: "user",
+              content: prompt,
+              timestamp: Date.now()
+            }
+          ]
+        },
+        {
+          ...config.options ?? {},
+          ...config.temperature !== void 0 ? { temperature: config.temperature } : {},
+          ...config.maxOutputTokens !== void 0 ? { maxTokens: config.maxOutputTokens } : {},
+          ...signal ? { signal } : {}
+        }
+      );
+      return resolvePiAiJudgeText(message);
+    }
+  });
+}
+function formatPiAiJudgeSystemPrompt(system, responseFormat) {
+  if (responseFormat?.type !== "json") {
+    return system;
+  }
+  const jsonInstruction = [
+    "Return only valid JSON. Do not include markdown fences or explanatory prose.",
+    responseFormat.schema ? `JSON Schema:
+${JSON.stringify(responseFormat.schema, null, 2)}` : void 0
+  ].filter(Boolean).join("\n\n");
+  return system ? `${system}
+
+${jsonInstruction}` : jsonInstruction;
+}
+function resolvePiAiJudgeText(message) {
+  if (message.stopReason === "error" || message.stopReason === "aborted") {
+    throw new Error(message.errorMessage ?? "Pi AI judge harness failed.");
+  }
+  return message.content.filter((part) => part.type === "text").map((part) => part.text).join("").trim();
+}
 var ORIGINAL_NATIVE_EXECUTE = Symbol("vitest-evals.originalNativeExecute");
 function piAiHarness(options) {
   validateOptions(options);
+  const harnessName = options.name ?? "pi-ai";
   const harness = {
-    name: options.name ?? "pi-ai",
+    name: harnessName,
     run: async (input, context) => {
-      const agent = await resolveAgent(options, {
-        input,
-        context
-      });
-      const messages = [
-        {
-          role: "user",
-          content: (0, import_harness.normalizeContent)(input)
+      const startedAt = /* @__PURE__ */ new Date();
+      try {
+        const agent = await resolveAgent(options, {
+          input,
+          context
+        });
+        const messages = [
+          {
+            role: "user",
+            content: (0, import_harness.normalizeContent)(input)
+          }
+        ];
+        const inferredTools = resolveInferredToolSurfaces(
+          agent
+        );
+        if (hasExplicitToolset(options)) {
+          return await executePiHarnessRun(
+            options,
+            agent,
+            input,
+            context,
+            messages,
+            options.tools,
+            inferredTools.nativeToolsets
+          );
         }
-      ];
-      const inferredTools = resolveInferredToolSurfaces(
-        agent
-      );
-      if (hasExplicitToolset(options)) {
-        return executePiHarnessRun(
+        return await executePiHarnessRun(
           options,
           agent,
           input,
           context,
           messages,
-          options.tools,
+          inferredTools.runtimeTools,
           inferredTools.nativeToolsets
         );
+      } catch (error) {
+        if ((0, import_harness.getHarnessRunFromError)(error)) {
+          throw error;
+        }
+        throw (0, import_harness.attachHarnessRunToError)(
+          error,
+          createFailedPiAiRun(input, context, error, harnessName, startedAt)
+        );
       }
-      return executePiHarnessRun(
-        options,
-        agent,
-        input,
-        context,
-        messages,
-        inferredTools.runtimeTools,
-        inferredTools.nativeToolsets
-      );
     }
   };
   return harness;
 }
+function createFailedPiAiRun(input, context, error, harnessName, startedAt) {
+  const run = (0, import_harness.createFailedHarnessRun)(input, error, {
+    artifacts: context.artifacts
+  });
+  (0, import_harness.ensureRunTrace)(run, {
+    name: harnessName,
+    startedAt,
+    finishedAt: /* @__PURE__ */ new Date(),
+    operationName: "invoke_agent",
+    source: "harness-pi-ai"
+  });
+  return run;
+}
 function validateOptions(options) {
   if (options.agent === void 0) {
     throw new Error(
@@ -76,6 +151,7 @@ function validateOptions(options) {
   }
 }
 async function executePiHarnessRun(options, agent, input, context, messages, runtimeTools, nativeToolsets) {
+  const trace = createTraceRecorder(options.name ?? "pi-ai");
   const executionState = createPiToolExecutionState();
   const runtime = createRuntime({
     input,
@@ -108,6 +184,7 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
       if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {
         result.artifacts = context.artifacts;
       }
+      attachPiAiTrace(result, trace, /* @__PURE__ */ new Date());
       return result;
     }
     const normalizeResult = result;
@@ -121,21 +198,42 @@ async function executePiHarnessRun(options, agent, input, context, messages, run
     const output = options.output ? await options.output(resultArgs) : resolveOutput(normalizeResult);
     const usage = resolveUsage(normalizeResult, runtime.toolCalls.length);
     const session = resolveSession(normalizeResult, messages, output, usage);
+    const errors = resolveErrors(normalizeResult);
+    const finishedAt = /* @__PURE__ */ new Date();
     return {
       session,
       output,
       usage,
       artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
-      errors: resolveErrors(normalizeResult)
+      errors,
+      traces: [
+        finishPiAiTrace(trace, {
+          session,
+          usage,
+          errors,
+          finishedAt
+        })
+      ]
     };
   } catch (error) {
     const usage = resolveUsage(void 0, runtime.toolCalls.length);
+    const session = resolveSession(void 0, messages, void 0, usage);
+    const finishedAt = /* @__PURE__ */ new Date();
+    const serializedError = (0, import_harness.serializeError)(error);
     const run = {
-      session: resolveSession(void 0, messages, void 0, usage),
+      session,
       output: void 0,
       usage,
       artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
-      errors: [(0, import_harness.serializeError)(error)]
+      errors: [serializedError],
+      traces: [
+        finishPiAiTrace(trace, {
+          session,
+          usage,
+          errors: [serializedError],
+          finishedAt
+        })
+      ]
     };
     throw (0, import_harness.attachHarnessRunToError)(error, run);
   }
@@ -155,6 +253,83 @@ function isAgentFactory(agent) {
 function hasOutputSelector(options) {
   return Boolean(options.output);
 }
+function createTraceRecorder(name) {
+  const id = `pi_ai_trace_${++nextTraceId}`;
+  return {
+    id,
+    rootSpanId: `${id}:run`,
+    name,
+    startedAt: /* @__PURE__ */ new Date()
+  };
+}
+function attachPiAiTrace(run, trace, finishedAt) {
+  const nativeTrace = finishPiAiTrace(trace, {
+    session: run.session,
+    usage: run.usage,
+    errors: run.errors ?? [],
+    finishedAt
+  });
+  run.traces = [...run.traces ?? [], nativeTrace];
+}
+function finishPiAiTrace(trace, options) {
+  const finishedAt = options.finishedAt;
+  const durationMs = finishedAt.getTime() - trace.startedAt.getTime();
+  const rootError = options.errors?.[0] ? (0, import_harness.normalizeSpanError)(options.errors[0]) : void 0;
+  const rootSpan = {
+    id: trace.rootSpanId,
+    traceId: trace.id,
+    name: trace.name,
+    kind: "run",
+    startedAt: trace.startedAt.toISOString(),
+    finishedAt: finishedAt.toISOString(),
+    durationMs,
+    status: rootError ? "error" : "ok",
+    ...rootError ? { error: rootError } : {},
+    attributes: (0, import_harness.normalizeSpanAttributes)({
+      "gen_ai.operation.name": "invoke_agent",
+      "gen_ai.workflow.name": trace.name,
+      ...(0, import_harness.createGenAiUsageAttributes)(options.usage)
+    })
+  };
+  const modelSpan = createUsageModelSpan(trace, options.usage);
+  const spans = [
+    rootSpan,
+    ...modelSpan ? [modelSpan] : [],
+    ...(0, import_harness.createToolCallSpans)((0, import_harness.toolCalls)(options.session), {
+      traceId: trace.id,
+      parentId: trace.rootSpanId,
+      spanIdPrefix: `${trace.id}:tool`
+    })
+  ];
+  return {
+    id: trace.id,
+    name: trace.name,
+    startedAt: trace.startedAt.toISOString(),
+    finishedAt: finishedAt.toISOString(),
+    durationMs,
+    metadata: {
+      source: "harness-pi-ai"
+    },
+    spans
+  };
+}
+function createUsageModelSpan(trace, usage) {
+  if (!usage?.provider && !usage?.model) {
+    return void 0;
+  }
+  return {
+    id: `${trace.id}:model:1`,
+    traceId: trace.id,
+    parentId: trace.rootSpanId,
+    name: usage.model ? `pi-ai ${usage.model}` : "pi-ai model",
+    kind: "model",
+    status: "ok",
+    attributes: (0, import_harness.normalizeSpanAttributes)({
+      "gen_ai.operation.name": "chat",
+      ...(0, import_harness.createGenAiUsageAttributes)(usage)
+    })
+  };
+}
 function resolveInferredToolSurfaces(agent) {
   let runtimeTools;
   const nativeToolsets = [];
@@ -286,6 +461,7 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
         });
         const finishedAt = /* @__PURE__ */ new Date();
         const call = {
+          id: toolCallId,
           name: tool.name,
           arguments: rawArgs,
           result: execution.normalizedResult,
@@ -310,6 +486,7 @@ async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
       } catch (error) {
         const finishedAt = /* @__PURE__ */ new Date();
         const call = {
+          id: toolCallId,
           name: tool.name,
           arguments: rawArgs,
           error: serializeToolCallError(error),
@@ -802,6 +979,7 @@ async function executeToolWithReplay({
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  piAiHarness
+  piAiHarness,
+  piAiJudgeHarness
 });
 //# sourceMappingURL=index.js.map