@danypops/pi-eval-harness 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Daniel Popsuev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,53 @@
+# @danypops/pi-eval-harness
+
+Scores a real agent run's own tool-call behavior over Pi's real `AgentSessionEvent` stream --
+AND/OR tool-call matching, graduated checker composition, and turn/tool-call/token-usage
+rollups. Ported from Alef's own `packages/core/eval` evaluation framework, adapted to operate
+directly on the real event union any `@danypops/pi-process-harness` (or `pi --mode rpc`) run
+already produces, instead of a bespoke OTel span format.
+
+## Usage
+
+```ts
+import { deriveTurns, expectsAll, extractToolExecutions, summarizeRunUsage } from "@danypops/pi-eval-harness";
+
+const executions = extractToolExecutions(sessionEvents);
+// [{ toolCallId, toolName, args, result, isError }, ...] in completion order
+
+const checker = expectsAll([{ tool: "search_code", target: { pattern: "TODO" } }]);
+const result = await checker.check({ executions });
+// { pass, score, errors }
+
+const turns = deriveTurns(sessionEvents);
+const usage = summarizeRunUsage(turns);
+// { turns, tokensIn, tokensOut, cacheReadTokens, costUsd, toolCalls, toolNames }
+```
+
+## Scope
+
+- `extractToolExecutions` -- pairs `tool_execution_start`/`tool_execution_end` by `toolCallId`
+  into one real completed call per pair, in completion order.
+- `matchesToolCall`/`describeToolCall` -- whether one completed execution satisfies a `ToolCall`
+  expectation (tool name, target args, produced output), and a human-readable description of it.
+- `expectsAll`/`expectsAny`/`all` -- `Checker`s with AND/OR/composed semantics and graduated
+  `CheckerResult` scoring (0.0 hard fail, 1.0 full pass).
+- `deriveTurns`/`summarizeRunUsage` -- one `Turn` per real `turn_end` event (model, token usage,
+  cost, ordered tool-call names), rolled up into whole-run totals.
+
+## Testing your own Checkers: fixture self-test discipline
+
+Ported from Alef's own `Evaluation.fixture`/`FixtureSet` discipline: a `Checker` must be proven
+correct against a small, hand-authored, known-good `ToolExecution[]` fixture -- with **zero**
+live process spawns or LLM calls -- before it is ever trusted against a real run. This package's
+own test suite follows exactly that pattern (see `test/checker.test.ts` and `test/tool-call.test.ts`):
+build a `ToolExecution` fixture by hand, call `checker.check({ executions })` directly, assert the
+exact `{ pass, score, errors }` you expect. No helper wraps this -- the whole point is that
+`Checker.check()` is already a pure, synchronous-or-trivially-awaitable function; wrapping it
+would only hide the assertion, not simplify it.
+
+A `Checker` that only ever gets exercised against a real, expensive `pi-process-harness` run has
+no fast, deterministic proof it is correct in isolation -- write the fixture test first.
+
+## License
+
+MIT

package/dist/checker.d.ts

@@ -0,0 +1,33 @@
+import { type ToolCall } from "./tool-call.js";
+import type { ToolExecution } from "./tool-executions.js";
+/** Outcome of a Checker.check() call with graduated scoring, ported from Alef's own eval framework. */
+export interface CheckerResult {
+    readonly pass: boolean;
+    /** Graduated score 0-1: 0.0 hard fail, 0.5 partial, 1.0 full pass. */
+    readonly score: number;
+    readonly errors: readonly string[];
+}
+/** Runtime context passed to a Checker -- the real completed tool executions from one run. */
+export interface CheckerContext {
+    readonly executions: readonly ToolExecution[];
+}
+/** A pure, deterministic verifier over one run's real tool executions. */
+export interface Checker {
+    check(context: CheckerContext): CheckerResult | Promise<CheckerResult>;
+}
+/**
+ * A Checker requiring every expectation to be satisfied by at least one execution (AND
+ * semantics) -- ported from Alef's `Evaluation.expects`.
+ */
+export declare function expectsAll(expectations: readonly ToolCall[]): Checker;
+/**
+ * A Checker requiring at least one expectation to be satisfied by at least one execution (OR
+ * semantics) -- ported from Alef's `Evaluation.expectsAny`.
+ */
+export declare function expectsAny(expectations: readonly ToolCall[]): Checker;
+/**
+ * Composes several checkers with AND semantics: the combined score is the minimum of every
+ * checker's own score, and every checker's errors are concatenated -- ported from Alef's
+ * `checker.ts` `all()`.
+ */
+export declare function all(...checkers: readonly Checker[]): Checker;

package/dist/checker.js

@@ -0,0 +1,47 @@
+import { describeToolCall, matchesToolCall } from "./tool-call.js";
+/**
+ * A Checker requiring every expectation to be satisfied by at least one execution (AND
+ * semantics) -- ported from Alef's `Evaluation.expects`.
+ */
+export function expectsAll(expectations) {
+    return {
+        check({ executions }) {
+            const errors = expectations
+                .filter((expectation) => !executions.some((execution) => matchesToolCall(execution, expectation)))
+                .map((expectation) => `Expected ${describeToolCall(expectation)}`);
+            return { pass: errors.length === 0, score: errors.length === 0 ? 1 : 0, errors };
+        },
+    };
+}
+/**
+ * A Checker requiring at least one expectation to be satisfied by at least one execution (OR
+ * semantics) -- ported from Alef's `Evaluation.expectsAny`.
+ */
+export function expectsAny(expectations) {
+    return {
+        check({ executions }) {
+            if (expectations.length === 0)
+                return { pass: true, score: 1, errors: [] };
+            const satisfied = expectations.some((expectation) => executions.some((execution) => matchesToolCall(execution, expectation)));
+            if (satisfied)
+                return { pass: true, score: 1, errors: [] };
+            const description = expectations.map((expectation) => describeToolCall(expectation)).join(" OR ");
+            return { pass: false, score: 0, errors: [`Expected at least one: ${description}`] };
+        },
+    };
+}
+/**
+ * Composes several checkers with AND semantics: the combined score is the minimum of every
+ * checker's own score, and every checker's errors are concatenated -- ported from Alef's
+ * `checker.ts` `all()`.
+ */
+export function all(...checkers) {
+    return {
+        async check(context) {
+            const results = await Promise.all(checkers.map((checker) => checker.check(context)));
+            const errors = results.flatMap((result) => result.errors);
+            const score = results.length === 0 ? 1 : Math.min(...results.map((result) => result.score));
+            return { pass: errors.length === 0, score, errors };
+        },
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { all, type Checker, type CheckerContext, type CheckerResult, expectsAll, expectsAny } from "./checker.js";
+export { describeToolCall, matchesToolCall, type ToolCall } from "./tool-call.js";
+export { extractToolExecutions, type ToolExecution } from "./tool-executions.js";
+export { deriveTurns, type RunUsageSummary, summarizeRunUsage, type Turn } from "./turns.js";

package/dist/index.js

@@ -0,0 +1,4 @@
+export { all, expectsAll, expectsAny } from "./checker.js";
+export { describeToolCall, matchesToolCall } from "./tool-call.js";
+export { extractToolExecutions } from "./tool-executions.js";
+export { deriveTurns, summarizeRunUsage } from "./turns.js";

package/dist/tool-call.d.ts

@@ -0,0 +1,36 @@
+import type { ToolExecution } from "./tool-executions.js";
+/**
+ * One expected tool interaction, ported from Alef's own `packages/core/eval` evaluation
+ * framework (`ToolCall`/`matchesToolCall`/`expects`/`expectsAny`) -- adapted from Alef's OTel
+ * `SpanRecord[]` onto this package's own `ToolExecution[]` (see `tool-executions.ts`), the real
+ * projection of a Pi `AgentSessionEvent` stream's completed tool calls.
+ *
+ * tool    -- which tool was invoked (OR semantics across the array).
+ * target  -- what the tool was called on, matched against its real input args.
+ * produces -- what the tool's output must contain.
+ *
+ * All non-undefined fields must match for the expectation to be satisfied.
+ */
+export interface ToolCall {
+    /** Acceptable tool name(s) -- any one satisfies the call dimension. */
+    readonly tool: string | readonly string[];
+    /** Fields that must appear in the tool's input args. */
+    readonly target?: {
+        /** File path the tool operated on (substring or regex). */
+        readonly path?: string | RegExp;
+        /** Search pattern used (substring or regex). */
+        readonly pattern?: string | RegExp;
+        /** Symbol name targeted (substring or regex). */
+        readonly symbol?: string | RegExp;
+        /** URL fetched (substring or regex). */
+        readonly url?: string | RegExp;
+        /** Arbitrary payload field -- key/value or regex. */
+        readonly [key: string]: string | RegExp | undefined;
+    };
+    /** What the tool's output must contain (substring or regex). */
+    readonly produces?: string | RegExp;
+}
+/** Whether one real completed tool execution satisfies a tool-call expectation. */
+export declare function matchesToolCall(execution: ToolExecution, expectation: ToolCall): boolean;
+/** Formats a tool-call expectation as a human-readable description for a checker error. */
+export declare function describeToolCall(expectation: ToolCall): string;

package/dist/tool-call.js

@@ -0,0 +1,53 @@
+function matchValue(actual, expected) {
+    return expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);
+}
+/** Stringifies a real tool result (which may be any JSON value, not only text) for `produces` matching. */
+function stringifyResult(result) {
+    if (typeof result === "string")
+        return result;
+    try {
+        return JSON.stringify(result) ?? "";
+    }
+    catch {
+        return String(result);
+    }
+}
+function argValue(args, key) {
+    if (typeof args !== "object" || args === null)
+        return "";
+    const value = args[key];
+    return typeof value === "string" ? value : "";
+}
+/** Whether one real completed tool execution satisfies a tool-call expectation. */
+export function matchesToolCall(execution, expectation) {
+    const tools = Array.isArray(expectation.tool) ? expectation.tool : [expectation.tool];
+    if (!tools.some((name) => execution.toolName === name))
+        return false;
+    if (expectation.target) {
+        for (const [key, pattern] of Object.entries(expectation.target)) {
+            if (pattern === undefined)
+                continue;
+            if (!matchValue(argValue(execution.args, key), pattern))
+                return false;
+        }
+    }
+    if (expectation.produces !== undefined) {
+        if (!matchValue(stringifyResult(execution.result), expectation.produces))
+            return false;
+    }
+    return true;
+}
+/** Formats a tool-call expectation as a human-readable description for a checker error. */
+export function describeToolCall(expectation) {
+    const tools = Array.isArray(expectation.tool) ? expectation.tool.join("|") : expectation.tool;
+    const target = expectation.target
+        ? ` on ${Object.entries(expectation.target)
+            .filter(([, value]) => value !== undefined)
+            .map(([key, value]) => `${key}=${value instanceof RegExp ? value.source : String(value)}`)
+            .join(", ")}`
+        : "";
+    const produces = expectation.produces !== undefined
+        ? ` → ${expectation.produces instanceof RegExp ? expectation.produces.source : expectation.produces}`
+        : "";
+    return `${tools}${target}${produces}`;
+}

package/dist/tool-executions.d.ts

@@ -0,0 +1,16 @@
+import type { AgentSessionEvent } from "@earendil-works/pi-coding-agent";
+/** One completed tool call, paired from its real tool_execution_start/tool_execution_end events. */
+export interface ToolExecution {
+    readonly toolCallId: string;
+    readonly toolName: string;
+    readonly args: unknown;
+    readonly result: unknown;
+    readonly isError: boolean;
+}
+/**
+ * Pairs each `tool_execution_start` with its matching `tool_execution_end` by `toolCallId`, in
+ * completion order. A start with no matching end (a call still running when the stream ends, or
+ * aborted mid-flight) is dropped -- only a fully finished call is real trace data a checker can
+ * score.
+ */
+export declare function extractToolExecutions(events: readonly AgentSessionEvent[]): ToolExecution[];

package/dist/tool-executions.js

@@ -0,0 +1,29 @@
+/**
+ * Pairs each `tool_execution_start` with its matching `tool_execution_end` by `toolCallId`, in
+ * completion order. A start with no matching end (a call still running when the stream ends, or
+ * aborted mid-flight) is dropped -- only a fully finished call is real trace data a checker can
+ * score.
+ */
+export function extractToolExecutions(events) {
+    const starts = new Map();
+    const executions = [];
+    for (const event of events) {
+        if (event.type === "tool_execution_start") {
+            starts.set(event.toolCallId, { toolName: event.toolName, args: event.args });
+            continue;
+        }
+        if (event.type === "tool_execution_end") {
+            const start = starts.get(event.toolCallId);
+            if (start === undefined)
+                continue;
+            executions.push({
+                toolCallId: event.toolCallId,
+                toolName: start.toolName,
+                args: start.args,
+                result: event.result,
+                isError: event.isError,
+            });
+        }
+    }
+    return executions;
+}

package/dist/turns.d.ts

@@ -0,0 +1,38 @@
+import type { AgentSessionEvent } from "@earendil-works/pi-coding-agent";
+/**
+ * One real LLM turn, derived directly from a `turn_end` event -- unlike Alef's own
+ * `deriveturns` (which had to reconstruct turn boundaries by scanning for OTel `chat ` spans
+ * and attributing trailing command spans to them), Pi's real `AgentEvent` union already emits
+ * an explicit `turn_end` per turn, carrying the assistant's own message (with real `Usage`) and
+ * that turn's own `toolResults` directly -- no reconstruction needed.
+ */
+export interface Turn {
+    /** 1-based turn index within the run. */
+    readonly turn: number;
+    /** Model id from the assistant message, empty when the turn's message isn't an assistant reply. */
+    readonly model: string;
+    readonly tokensIn: number;
+    readonly tokensOut: number;
+    readonly cacheReadTokens: number;
+    /** Real cost in USD from Usage.cost.total, when the provider reports pricing. */
+    readonly costUsd: number;
+    /** Number of tool calls dispatched from this turn. */
+    readonly toolCalls: number;
+    /** Names of tools called in this turn, in dispatch order. */
+    readonly toolNames: readonly string[];
+}
+/** Derives one Turn per real `turn_end` event, in the stream's own order. */
+export declare function deriveTurns(events: readonly AgentSessionEvent[]): Turn[];
+/** Aggregate token/cost/turn/tool-call totals across a whole run's turns. */
+export interface RunUsageSummary {
+    readonly turns: number;
+    readonly tokensIn: number;
+    readonly tokensOut: number;
+    readonly cacheReadTokens: number;
+    readonly costUsd: number;
+    readonly toolCalls: number;
+    /** Every tool name called across the whole run, in dispatch order. */
+    readonly toolNames: readonly string[];
+}
+/** Rolls up per-turn Turn records into whole-run usage totals. */
+export declare function summarizeRunUsage(turns: readonly Turn[]): RunUsageSummary;

package/dist/turns.js

@@ -0,0 +1,36 @@
+/** Derives one Turn per real `turn_end` event, in the stream's own order. */
+export function deriveTurns(events) {
+    const turns = [];
+    let turnIndex = 0;
+    for (const event of events) {
+        if (event.type !== "turn_end")
+            continue;
+        turnIndex++;
+        const message = event.message;
+        const isAssistant = message.role === "assistant";
+        const usage = isAssistant ? message.usage : undefined;
+        turns.push({
+            turn: turnIndex,
+            model: isAssistant ? message.model : "",
+            tokensIn: usage?.input ?? 0,
+            tokensOut: usage?.output ?? 0,
+            cacheReadTokens: usage?.cacheRead ?? 0,
+            costUsd: usage?.cost.total ?? 0,
+            toolCalls: event.toolResults.length,
+            toolNames: event.toolResults.map((result) => result.toolName),
+        });
+    }
+    return turns;
+}
+/** Rolls up per-turn Turn records into whole-run usage totals. */
+export function summarizeRunUsage(turns) {
+    return {
+        turns: turns.length,
+        tokensIn: turns.reduce((sum, turn) => sum + turn.tokensIn, 0),
+        tokensOut: turns.reduce((sum, turn) => sum + turn.tokensOut, 0),
+        cacheReadTokens: turns.reduce((sum, turn) => sum + turn.cacheReadTokens, 0),
+        costUsd: turns.reduce((sum, turn) => sum + turn.costUsd, 0),
+        toolCalls: turns.reduce((sum, turn) => sum + turn.toolCalls, 0),
+        toolNames: turns.flatMap((turn) => turn.toolNames),
+    };
+}

package/package.json

@@ -0,0 +1,47 @@
+{
+	"name": "@danypops/pi-eval-harness",
+	"version": "0.1.0",
+	"description": "Scores a real agent run's own tool-call behavior -- AND/OR tool-call matching, graduated checker composition, and turn/tool-call/token-usage rollups -- over Pi's own real AgentSessionEvent stream (e.g. from @danypops/pi-process-harness).",
+	"license": "MIT",
+	"type": "module",
+	"main": "./dist/index.js",
+	"sideEffects": false,
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
+		"./package.json": "./package.json"
+	},
+	"scripts": {
+		"build": "rm -rf dist && tsc -p tsconfig.build.json",
+		"test": "bun run build && bun test test",
+		"typecheck": "tsc --noEmit",
+		"format": "biome format --write ."
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/DanyPops/pi-integral.git",
+		"directory": "packages/pi-eval-harness"
+	},
+	"homepage": "https://github.com/DanyPops/pi-integral/tree/main/packages/pi-eval-harness#readme",
+	"bugs": {
+		"url": "https://github.com/DanyPops/pi-integral/issues"
+	},
+	"peerDependencies": {
+		"@earendil-works/pi-coding-agent": "*"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "^2.5.6",
+		"@danypops/pi-process-harness": "workspace:*",
+		"@earendil-works/pi-ai": "*",
+		"@earendil-works/pi-coding-agent": "*",
+		"@types/node": "^22.0.0",
+		"bun-types": "latest",
+		"typebox": "*",
+		"typescript": "^5.9.2"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"files": ["src", "dist", "README.md", "LICENSE"],
+	"keywords": ["pi", "pi-extension", "testing", "eval", "agent-evaluation", "tool-call"]
+}

package/src/checker.ts

@@ -0,0 +1,67 @@
+import { describeToolCall, matchesToolCall, type ToolCall } from "./tool-call.js";
+import type { ToolExecution } from "./tool-executions.js";
+
+/** Outcome of a Checker.check() call with graduated scoring, ported from Alef's own eval framework. */
+export interface CheckerResult {
+	readonly pass: boolean;
+	/** Graduated score 0-1: 0.0 hard fail, 0.5 partial, 1.0 full pass. */
+	readonly score: number;
+	readonly errors: readonly string[];
+}
+
+/** Runtime context passed to a Checker -- the real completed tool executions from one run. */
+export interface CheckerContext {
+	readonly executions: readonly ToolExecution[];
+}
+
+/** A pure, deterministic verifier over one run's real tool executions. */
+export interface Checker {
+	check(context: CheckerContext): CheckerResult | Promise<CheckerResult>;
+}
+
+/**
+ * A Checker requiring every expectation to be satisfied by at least one execution (AND
+ * semantics) -- ported from Alef's `Evaluation.expects`.
+ */
+export function expectsAll(expectations: readonly ToolCall[]): Checker {
+	return {
+		check({ executions }: CheckerContext): CheckerResult {
+			const errors = expectations
+				.filter((expectation) => !executions.some((execution) => matchesToolCall(execution, expectation)))
+				.map((expectation) => `Expected ${describeToolCall(expectation)}`);
+			return { pass: errors.length === 0, score: errors.length === 0 ? 1 : 0, errors };
+		},
+	};
+}
+
+/**
+ * A Checker requiring at least one expectation to be satisfied by at least one execution (OR
+ * semantics) -- ported from Alef's `Evaluation.expectsAny`.
+ */
+export function expectsAny(expectations: readonly ToolCall[]): Checker {
+	return {
+		check({ executions }: CheckerContext): CheckerResult {
+			if (expectations.length === 0) return { pass: true, score: 1, errors: [] };
+			const satisfied = expectations.some((expectation) => executions.some((execution) => matchesToolCall(execution, expectation)));
+			if (satisfied) return { pass: true, score: 1, errors: [] };
+			const description = expectations.map((expectation) => describeToolCall(expectation)).join(" OR ");
+			return { pass: false, score: 0, errors: [`Expected at least one: ${description}`] };
+		},
+	};
+}
+
+/**
+ * Composes several checkers with AND semantics: the combined score is the minimum of every
+ * checker's own score, and every checker's errors are concatenated -- ported from Alef's
+ * `checker.ts` `all()`.
+ */
+export function all(...checkers: readonly Checker[]): Checker {
+	return {
+		async check(context: CheckerContext): Promise<CheckerResult> {
+			const results = await Promise.all(checkers.map((checker) => checker.check(context)));
+			const errors = results.flatMap((result) => result.errors);
+			const score = results.length === 0 ? 1 : Math.min(...results.map((result) => result.score));
+			return { pass: errors.length === 0, score, errors };
+		},
+	};
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export { all, type Checker, type CheckerContext, type CheckerResult, expectsAll, expectsAny } from "./checker.js";
+export { describeToolCall, matchesToolCall, type ToolCall } from "./tool-call.js";
+export { extractToolExecutions, type ToolExecution } from "./tool-executions.js";
+export { deriveTurns, type RunUsageSummary, summarizeRunUsage, type Turn } from "./turns.js";

package/src/tool-call.ts

@@ -0,0 +1,88 @@
+import type { ToolExecution } from "./tool-executions.js";
+
+/**
+ * One expected tool interaction, ported from Alef's own `packages/core/eval` evaluation
+ * framework (`ToolCall`/`matchesToolCall`/`expects`/`expectsAny`) -- adapted from Alef's OTel
+ * `SpanRecord[]` onto this package's own `ToolExecution[]` (see `tool-executions.ts`), the real
+ * projection of a Pi `AgentSessionEvent` stream's completed tool calls.
+ *
+ * tool    -- which tool was invoked (OR semantics across the array).
+ * target  -- what the tool was called on, matched against its real input args.
+ * produces -- what the tool's output must contain.
+ *
+ * All non-undefined fields must match for the expectation to be satisfied.
+ */
+export interface ToolCall {
+	/** Acceptable tool name(s) -- any one satisfies the call dimension. */
+	readonly tool: string | readonly string[];
+	/** Fields that must appear in the tool's input args. */
+	readonly target?: {
+		/** File path the tool operated on (substring or regex). */
+		readonly path?: string | RegExp;
+		/** Search pattern used (substring or regex). */
+		readonly pattern?: string | RegExp;
+		/** Symbol name targeted (substring or regex). */
+		readonly symbol?: string | RegExp;
+		/** URL fetched (substring or regex). */
+		readonly url?: string | RegExp;
+		/** Arbitrary payload field -- key/value or regex. */
+		readonly [key: string]: string | RegExp | undefined;
+	};
+	/** What the tool's output must contain (substring or regex). */
+	readonly produces?: string | RegExp;
+}
+
+function matchValue(actual: string, expected: string | RegExp): boolean {
+	return expected instanceof RegExp ? expected.test(actual) : actual.includes(expected);
+}
+
+/** Stringifies a real tool result (which may be any JSON value, not only text) for `produces` matching. */
+function stringifyResult(result: unknown): string {
+	if (typeof result === "string") return result;
+	try {
+		return JSON.stringify(result) ?? "";
+	} catch {
+		return String(result);
+	}
+}
+
+function argValue(args: unknown, key: string): string {
+	if (typeof args !== "object" || args === null) return "";
+	const value = (args as Record<string, unknown>)[key];
+	return typeof value === "string" ? value : "";
+}
+
+/** Whether one real completed tool execution satisfies a tool-call expectation. */
+export function matchesToolCall(execution: ToolExecution, expectation: ToolCall): boolean {
+	const tools = Array.isArray(expectation.tool) ? expectation.tool : [expectation.tool];
+	if (!tools.some((name) => execution.toolName === name)) return false;
+
+	if (expectation.target) {
+		for (const [key, pattern] of Object.entries(expectation.target)) {
+			if (pattern === undefined) continue;
+			if (!matchValue(argValue(execution.args, key), pattern)) return false;
+		}
+	}
+
+	if (expectation.produces !== undefined) {
+		if (!matchValue(stringifyResult(execution.result), expectation.produces)) return false;
+	}
+
+	return true;
+}
+
+/** Formats a tool-call expectation as a human-readable description for a checker error. */
+export function describeToolCall(expectation: ToolCall): string {
+	const tools = Array.isArray(expectation.tool) ? expectation.tool.join("|") : expectation.tool;
+	const target = expectation.target
+		? ` on ${Object.entries(expectation.target)
+				.filter(([, value]) => value !== undefined)
+				.map(([key, value]) => `${key}=${value instanceof RegExp ? value.source : String(value)}`)
+				.join(", ")}`
+		: "";
+	const produces =
+		expectation.produces !== undefined
+			? ` → ${expectation.produces instanceof RegExp ? expectation.produces.source : expectation.produces}`
+			: "";
+	return `${tools}${target}${produces}`;
+}

package/src/tool-executions.ts

@@ -0,0 +1,39 @@
+import type { AgentSessionEvent } from "@earendil-works/pi-coding-agent";
+
+/** One completed tool call, paired from its real tool_execution_start/tool_execution_end events. */
+export interface ToolExecution {
+	readonly toolCallId: string;
+	readonly toolName: string;
+	readonly args: unknown;
+	readonly result: unknown;
+	readonly isError: boolean;
+}
+
+/**
+ * Pairs each `tool_execution_start` with its matching `tool_execution_end` by `toolCallId`, in
+ * completion order. A start with no matching end (a call still running when the stream ends, or
+ * aborted mid-flight) is dropped -- only a fully finished call is real trace data a checker can
+ * score.
+ */
+export function extractToolExecutions(events: readonly AgentSessionEvent[]): ToolExecution[] {
+	const starts = new Map<string, { toolName: string; args: unknown }>();
+	const executions: ToolExecution[] = [];
+	for (const event of events) {
+		if (event.type === "tool_execution_start") {
+			starts.set(event.toolCallId, { toolName: event.toolName, args: event.args });
+			continue;
+		}
+		if (event.type === "tool_execution_end") {
+			const start = starts.get(event.toolCallId);
+			if (start === undefined) continue;
+			executions.push({
+				toolCallId: event.toolCallId,
+				toolName: start.toolName,
+				args: start.args,
+				result: event.result,
+				isError: event.isError,
+			});
+		}
+	}
+	return executions;
+}

package/src/turns.ts

@@ -0,0 +1,73 @@
+import type { AgentSessionEvent } from "@earendil-works/pi-coding-agent";
+
+/**
+ * One real LLM turn, derived directly from a `turn_end` event -- unlike Alef's own
+ * `deriveturns` (which had to reconstruct turn boundaries by scanning for OTel `chat ` spans
+ * and attributing trailing command spans to them), Pi's real `AgentEvent` union already emits
+ * an explicit `turn_end` per turn, carrying the assistant's own message (with real `Usage`) and
+ * that turn's own `toolResults` directly -- no reconstruction needed.
+ */
+export interface Turn {
+	/** 1-based turn index within the run. */
+	readonly turn: number;
+	/** Model id from the assistant message, empty when the turn's message isn't an assistant reply. */
+	readonly model: string;
+	readonly tokensIn: number;
+	readonly tokensOut: number;
+	readonly cacheReadTokens: number;
+	/** Real cost in USD from Usage.cost.total, when the provider reports pricing. */
+	readonly costUsd: number;
+	/** Number of tool calls dispatched from this turn. */
+	readonly toolCalls: number;
+	/** Names of tools called in this turn, in dispatch order. */
+	readonly toolNames: readonly string[];
+}
+
+/** Derives one Turn per real `turn_end` event, in the stream's own order. */
+export function deriveTurns(events: readonly AgentSessionEvent[]): Turn[] {
+	const turns: Turn[] = [];
+	let turnIndex = 0;
+	for (const event of events) {
+		if (event.type !== "turn_end") continue;
+		turnIndex++;
+		const message = event.message;
+		const isAssistant = message.role === "assistant";
+		const usage = isAssistant ? message.usage : undefined;
+		turns.push({
+			turn: turnIndex,
+			model: isAssistant ? message.model : "",
+			tokensIn: usage?.input ?? 0,
+			tokensOut: usage?.output ?? 0,
+			cacheReadTokens: usage?.cacheRead ?? 0,
+			costUsd: usage?.cost.total ?? 0,
+			toolCalls: event.toolResults.length,
+			toolNames: event.toolResults.map((result) => result.toolName),
+		});
+	}
+	return turns;
+}
+
+/** Aggregate token/cost/turn/tool-call totals across a whole run's turns. */
+export interface RunUsageSummary {
+	readonly turns: number;
+	readonly tokensIn: number;
+	readonly tokensOut: number;
+	readonly cacheReadTokens: number;
+	readonly costUsd: number;
+	readonly toolCalls: number;
+	/** Every tool name called across the whole run, in dispatch order. */
+	readonly toolNames: readonly string[];
+}
+
+/** Rolls up per-turn Turn records into whole-run usage totals. */
+export function summarizeRunUsage(turns: readonly Turn[]): RunUsageSummary {
+	return {
+		turns: turns.length,
+		tokensIn: turns.reduce((sum, turn) => sum + turn.tokensIn, 0),
+		tokensOut: turns.reduce((sum, turn) => sum + turn.tokensOut, 0),
+		cacheReadTokens: turns.reduce((sum, turn) => sum + turn.cacheReadTokens, 0),
+		costUsd: turns.reduce((sum, turn) => sum + turn.costUsd, 0),
+		toolCalls: turns.reduce((sum, turn) => sum + turn.toolCalls, 0),
+		toolNames: turns.flatMap((turn) => turn.toolNames),
+	};
+}