@danypops/pi-eval-harness 0.1.0 → 0.2.0

package/dist/ablation.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Runs one scenario under several named tool-availability configs and diffs the results --
+ * ported from djinn's own `testkit/eval_ablation.go` (`AblationConfig`/`AblationResult`/
+ * `AblationDelta`/`Ablate()`) and `eval_report.go`'s `FormatAblation()`. The first config is the
+ * baseline; every later config's own TrialMetrics get a delta computed against it.
+ */
+import { type RunTrialsOptions, type TrialMetrics, type TrialResult } from "./trials.js";
+/** One named variant of the scenario -- e.g. "baseline" (Pi's own built-in tools) vs "with-lector" (the same run, Lector's tools also registered). */
+export interface AblationConfig {
+    readonly name: string;
+    /** Runs one real trial under this config. Whatever varies between arms (which extensions load, which tools are registered) lives inside this closure. */
+    readonly runOne: () => Promise<TrialResult>;
+}
+/** The performance difference of one config's TrialMetrics vs the baseline (first) config's. */
+export interface AblationDelta {
+    readonly passRateDelta: number;
+    readonly meanScoreDelta: number;
+    readonly meanDurationMsDelta: number;
+    readonly meanTokensInDelta: number;
+}
+/** One config's own trial metrics, plus its delta vs baseline. `delta` is undefined for the baseline itself. */
+export interface AblationResult {
+    readonly config: AblationConfig;
+    readonly metrics: TrialMetrics;
+    readonly delta?: AblationDelta;
+}
+/**
+ * Runs `n` trials of every config (sequentially, config by config -- each config's own trials
+ * still run at `options.concurrency` internally) and returns side-by-side results. Configs are
+ * independent scenarios in the caller's own domain (e.g. real spawned `pi` processes with
+ * different extension sets); this function only orchestrates trial counts and diffs.
+ */
+export declare function ablate(configs: readonly AblationConfig[], n: number, options?: RunTrialsOptions): Promise<AblationResult[]>;
+/** Renders a human-readable ablation comparison table -- ported from djinn's `FormatAblation()`. */
+export declare function formatAblation(label: string, results: readonly AblationResult[]): string;

package/dist/ablation.js

@@ -0,0 +1,55 @@
+/**
+ * Runs one scenario under several named tool-availability configs and diffs the results --
+ * ported from djinn's own `testkit/eval_ablation.go` (`AblationConfig`/`AblationResult`/
+ * `AblationDelta`/`Ablate()`) and `eval_report.go`'s `FormatAblation()`. The first config is the
+ * baseline; every later config's own TrialMetrics get a delta computed against it.
+ */
+import { runTrials } from "./trials.js";
+/**
+ * Runs `n` trials of every config (sequentially, config by config -- each config's own trials
+ * still run at `options.concurrency` internally) and returns side-by-side results. Configs are
+ * independent scenarios in the caller's own domain (e.g. real spawned `pi` processes with
+ * different extension sets); this function only orchestrates trial counts and diffs.
+ */
+export async function ablate(configs, n, options = {}) {
+    if (configs.length === 0)
+        return [];
+    const results = [];
+    for (const config of configs) {
+        results.push({ config, metrics: await runTrials(config.runOne, n, options) });
+    }
+    const baseline = results[0]?.metrics;
+    if (baseline === undefined)
+        return results;
+    return results.map((result, index) => {
+        if (index === 0)
+            return result;
+        const metrics = result.metrics;
+        return {
+            ...result,
+            delta: {
+                passRateDelta: metrics.passRate - baseline.passRate,
+                meanScoreDelta: metrics.meanScore - baseline.meanScore,
+                meanDurationMsDelta: metrics.meanDurationMs - baseline.meanDurationMs,
+                meanTokensInDelta: metrics.meanTokensIn - baseline.meanTokensIn,
+            },
+        };
+    });
+}
+function signed(value, digits) {
+    const rounded = value.toFixed(digits);
+    return value >= 0 ? `+${rounded}` : rounded;
+}
+/** Renders a human-readable ablation comparison table -- ported from djinn's `FormatAblation()`. */
+export function formatAblation(label, results) {
+    const lines = [`=== Ablation: ${label} ===`];
+    for (const result of results) {
+        const m = result.metrics;
+        lines.push(`  ${result.config.name.padEnd(14)}  pass_rate=${m.passRate.toFixed(2)}  mean_score=${m.meanScore.toFixed(2)}  mean_duration_ms=${m.meanDurationMs.toFixed(0)}  tokens_in=${m.meanTokensIn.toFixed(0)}`);
+        if (result.delta) {
+            const d = result.delta;
+            lines.push(`  ${"(vs baseline)".padEnd(14)}  Δpass_rate=${signed(d.passRateDelta, 2)}  Δscore=${signed(d.meanScoreDelta, 2)}  Δduration_ms=${signed(d.meanDurationMsDelta, 0)}  Δtokens_in=${signed(d.meanTokensInDelta, 0)}`);
+        }
+    }
+    return lines.join("\n");
+}

package/dist/checker.d.ts

@@ -7,9 +7,16 @@ export interface CheckerResult {
     readonly score: number;
     readonly errors: readonly string[];
 }
-/** Runtime context passed to a Checker -- the real completed tool executions from one run. */
+/**
+ * Runtime context passed to a Checker -- the real completed tool executions from one run, plus
+ * (when the run actually had a real workspace directory) its absolute path, for a checker that
+ * verifies real post-run workspace state rather than which tools were called. Optional: a
+ * checker that only inspects `executions` (matching/rollup checks) never needs it, and every
+ * existing construction of this context predates the field.
+ */
 export interface CheckerContext {
     readonly executions: readonly ToolExecution[];
+    readonly workspace?: string;
 }
 /** A pure, deterministic verifier over one run's real tool executions. */
 export interface Checker {

package/dist/index.d.ts

@@ -1,4 +1,8 @@
+export { type AblationConfig, type AblationDelta, type AblationResult, ablate, formatAblation, } from "./ablation.js";
 export { all, type Checker, type CheckerContext, type CheckerResult, expectsAll, expectsAny } from "./checker.js";
+export { detectLoop, type LoopDetectionResult, type MistakeSummary, summarizeMistakes } from "./mistakes.js";
 export { describeToolCall, matchesToolCall, type ToolCall } from "./tool-call.js";
 export { extractToolExecutions, type ToolExecution } from "./tool-executions.js";
+export { aggregateTrials, MaxErrorRateExceeded, type RunTrialsOptions, runTrials, type TrialMetrics, type TrialResult, } from "./trials.js";
 export { deriveTurns, type RunUsageSummary, summarizeRunUsage, type Turn } from "./turns.js";
+export { any, fileContains, fileExists, lintPasses } from "./workspace-checkers.js";

package/dist/index.js

@@ -1,4 +1,8 @@
+export { ablate, formatAblation, } from "./ablation.js";
 export { all, expectsAll, expectsAny } from "./checker.js";
+export { detectLoop, summarizeMistakes } from "./mistakes.js";
 export { describeToolCall, matchesToolCall } from "./tool-call.js";
 export { extractToolExecutions } from "./tool-executions.js";
+export { aggregateTrials, MaxErrorRateExceeded, runTrials, } from "./trials.js";
 export { deriveTurns, summarizeRunUsage } from "./turns.js";
+export { any, fileContains, fileExists, lintPasses } from "./workspace-checkers.js";

package/dist/mistakes.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Mistake-rate and loop-detection rollups over one run's real tool executions -- ported from
+ * djinn's own `LoopThreshold`/`LoopDetected`/`LoopTool` (`testkit/eval_evaluation.go`/
+ * `eval_runner.go`): the exact edit-build-fail-retry failure mode the JetBrains Rider blog
+ * post's own worked example showed in detail (11 edit/build cycles fighting the compiler).
+ */
+import type { ToolExecution } from "./tool-executions.js";
+/** Real, per-execution error signal rolled up across a whole run. */
+export interface MistakeSummary {
+    readonly totalExecutions: number;
+    /** Count of tool_execution_end events with isError: true. */
+    readonly errorCount: number;
+    /** errorCount / totalExecutions; 0 for a run with no executions at all. */
+    readonly errorRate: number;
+}
+/** Rolls up isError across every completed execution in a run. */
+export declare function summarizeMistakes(executions: readonly ToolExecution[]): MistakeSummary;
+export interface LoopDetectionResult {
+    readonly loopDetected: boolean;
+    /** The tool name that triggered detection. Present only when loopDetected is true. */
+    readonly loopTool?: string;
+    /** The real highest same-tool call count observed, regardless of whether it crossed the threshold. */
+    readonly maxToolCallCount: number;
+}
+/**
+ * Detects a real repeated-tool-call pattern: the same tool name called more than `threshold`
+ * times in one run (default 10, matching djinn's own default). A simple total-count threshold
+ * per tool name, not a strict-consecutive-run check -- matching djinn's own definition.
+ */
+export declare function detectLoop(executions: readonly ToolExecution[], threshold?: number): LoopDetectionResult;

package/dist/mistakes.js

@@ -0,0 +1,32 @@
+/** Rolls up isError across every completed execution in a run. */
+export function summarizeMistakes(executions) {
+    const errorCount = executions.filter((execution) => execution.isError).length;
+    return {
+        totalExecutions: executions.length,
+        errorCount,
+        errorRate: executions.length === 0 ? 0 : errorCount / executions.length,
+    };
+}
+/**
+ * Detects a real repeated-tool-call pattern: the same tool name called more than `threshold`
+ * times in one run (default 10, matching djinn's own default). A simple total-count threshold
+ * per tool name, not a strict-consecutive-run check -- matching djinn's own definition.
+ */
+export function detectLoop(executions, threshold = 10) {
+    const counts = new Map();
+    for (const execution of executions) {
+        counts.set(execution.toolName, (counts.get(execution.toolName) ?? 0) + 1);
+    }
+    let maxToolCallCount = 0;
+    let loopTool;
+    for (const [toolName, count] of counts) {
+        if (count > maxToolCallCount) {
+            maxToolCallCount = count;
+            loopTool = toolName;
+        }
+    }
+    if (maxToolCallCount > threshold && loopTool !== undefined) {
+        return { loopDetected: true, loopTool, maxToolCallCount };
+    }
+    return { loopDetected: false, maxToolCallCount };
+}

package/dist/trials.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Multi-trial aggregation over N real, non-deterministic runs of the same scenario -- ported
+ * from Alef's own `EvaluationRunner.runN` (pass@k, variance, min/max score, a bounded
+ * concurrency cap) and djinn's `TrialMetrics` (pass rate, mean score/latency/tokens) shape.
+ * A single real live-LLM trial's score is noise, not signal -- this is what turns it into one.
+ */
+/** One real trial's own outcome. `error` set means the trial itself failed to run to completion -- a real infrastructure/runtime failure, distinct from a checker legitimately scoring it 0. */
+export interface TrialResult {
+    readonly pass: boolean;
+    readonly score: number;
+    readonly durationMs: number;
+    readonly tokensIn: number;
+    readonly tokensOut: number;
+    readonly costUsd: number;
+    readonly error?: string;
+}
+/** Aggregate statistics across N trials of one scenario. */
+export interface TrialMetrics {
+    readonly trials: number;
+    readonly passes: number;
+    readonly passRate: number;
+    readonly meanScore: number;
+    readonly variance: number;
+    readonly minScore: number;
+    readonly maxScore: number;
+    readonly meanDurationMs: number;
+    readonly meanTokensIn: number;
+    readonly meanTokensOut: number;
+    readonly meanCostUsd: number;
+}
+/**
+ * Thrown when too many trials errored outright to trust the aggregate as a real measurement --
+ * ported from Alef's own `[MaxErrorRate]` guard. Without this, a scenario that errors on every
+ * trial would silently aggregate to "0% pass", indistinguishable from a real, measured failure.
+ */
+export declare class MaxErrorRateExceeded extends Error {
+    readonly errorRate: number;
+    readonly maxErrorRate: number;
+    readonly errorCount: number;
+    readonly totalTrials: number;
+    constructor(errorRate: number, maxErrorRate: number, errorCount: number, totalTrials: number, firstError: string | undefined);
+}
+/**
+ * Aggregates already-collected trial results into TrialMetrics. Throws MaxErrorRateExceeded if
+ * `maxErrorRate` (0-1, default 0 = disabled) is exceeded by the fraction of errored trials.
+ */
+export declare function aggregateTrials(results: readonly TrialResult[], options?: {
+    readonly maxErrorRate?: number;
+}): TrialMetrics;
+export interface RunTrialsOptions {
+    /** Trials run concurrently at once. Default 3, matching Alef's own ALEF_EVAL_CONCURRENCY default -- real LLM providers rate-limit. */
+    readonly concurrency?: number;
+    readonly maxErrorRate?: number;
+}
+/**
+ * Runs `runOne` (one real trial of a scenario) `n` times at a bounded concurrency, then
+ * aggregates. A rejected `runOne` call becomes a TrialResult with `error` set (score 0, pass
+ * false) rather than aborting the whole batch -- one crashed trial should not lose every other
+ * trial's own real data.
+ */
+export declare function runTrials(runOne: () => Promise<TrialResult>, n: number, options?: RunTrialsOptions): Promise<TrialMetrics>;

package/dist/trials.js

@@ -0,0 +1,91 @@
+/**
+ * Multi-trial aggregation over N real, non-deterministic runs of the same scenario -- ported
+ * from Alef's own `EvaluationRunner.runN` (pass@k, variance, min/max score, a bounded
+ * concurrency cap) and djinn's `TrialMetrics` (pass rate, mean score/latency/tokens) shape.
+ * A single real live-LLM trial's score is noise, not signal -- this is what turns it into one.
+ */
+/**
+ * Thrown when too many trials errored outright to trust the aggregate as a real measurement --
+ * ported from Alef's own `[MaxErrorRate]` guard. Without this, a scenario that errors on every
+ * trial would silently aggregate to "0% pass", indistinguishable from a real, measured failure.
+ */
+export class MaxErrorRateExceeded extends Error {
+    errorRate;
+    maxErrorRate;
+    errorCount;
+    totalTrials;
+    constructor(errorRate, maxErrorRate, errorCount, totalTrials, firstError) {
+        super(`${(errorRate * 100).toFixed(0)}% of trials errored (${errorCount}/${totalTrials}), threshold ${(maxErrorRate * 100).toFixed(0)}%. First error: ${firstError ?? "unknown"}`);
+        this.errorRate = errorRate;
+        this.maxErrorRate = maxErrorRate;
+        this.errorCount = errorCount;
+        this.totalTrials = totalTrials;
+        this.name = "MaxErrorRateExceeded";
+    }
+}
+function mean(values) {
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+/**
+ * Aggregates already-collected trial results into TrialMetrics. Throws MaxErrorRateExceeded if
+ * `maxErrorRate` (0-1, default 0 = disabled) is exceeded by the fraction of errored trials.
+ */
+export function aggregateTrials(results, options = {}) {
+    if (results.length === 0)
+        throw new Error("aggregateTrials: at least one trial result is required");
+    const maxErrorRate = options.maxErrorRate ?? 0;
+    if (maxErrorRate > 0) {
+        const errored = results.filter((result) => result.error !== undefined);
+        const errorRate = errored.length / results.length;
+        if (errorRate > maxErrorRate) {
+            throw new MaxErrorRateExceeded(errorRate, maxErrorRate, errored.length, results.length, errored[0]?.error);
+        }
+    }
+    const scores = results.map((result) => result.score);
+    const passes = results.filter((result) => result.pass).length;
+    const meanScore = mean(scores);
+    return {
+        trials: results.length,
+        passes,
+        passRate: passes / results.length,
+        meanScore,
+        variance: mean(scores.map((score) => (score - meanScore) ** 2)),
+        minScore: Math.min(...scores),
+        maxScore: Math.max(...scores),
+        meanDurationMs: mean(results.map((result) => result.durationMs)),
+        meanTokensIn: mean(results.map((result) => result.tokensIn)),
+        meanTokensOut: mean(results.map((result) => result.tokensOut)),
+        meanCostUsd: mean(results.map((result) => result.costUsd)),
+    };
+}
+/**
+ * Runs `runOne` (one real trial of a scenario) `n` times at a bounded concurrency, then
+ * aggregates. A rejected `runOne` call becomes a TrialResult with `error` set (score 0, pass
+ * false) rather than aborting the whole batch -- one crashed trial should not lose every other
+ * trial's own real data.
+ */
+export async function runTrials(runOne, n, options = {}) {
+    const concurrency = options.concurrency ?? 3;
+    const results = [];
+    for (let i = 0; i < n; i += concurrency) {
+        const batchSize = Math.min(concurrency, n - i);
+        const batch = await Promise.all(Array.from({ length: batchSize }, async () => {
+            try {
+                return await runOne();
+            }
+            catch (error) {
+                return {
+                    pass: false,
+                    score: 0,
+                    durationMs: 0,
+                    tokensIn: 0,
+                    tokensOut: 0,
+                    costUsd: 0,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        }));
+        results.push(...batch);
+    }
+    return aggregateTrials(results, options.maxErrorRate !== undefined ? { maxErrorRate: options.maxErrorRate } : {});
+}

package/dist/workspace-checkers.d.ts

@@ -0,0 +1,16 @@
+import type { Checker } from "./checker.js";
+/** A Checker verifying a file exists in the run's workspace. Existence alone scores 0.5 -- content is unchecked. */
+export declare function fileExists(relativePath: string): Checker;
+/**
+ * A Checker verifying a file contains every required substring. Graduated: 1.0 all present,
+ * 0.5 some present, 0.0 none present or the file is missing.
+ */
+export declare function fileContains(relativePath: string, ...required: readonly string[]): Checker;
+/**
+ * A Checker running a real command in the run's workspace and asserting exit code 0 -- an
+ * outcome checker: verifies what the agent DID (does it still build/lint/pass tests), not what
+ * it said.
+ */
+export declare function lintPasses(cmd: string, args?: readonly string[]): Checker;
+/** Composes several checkers, returning the maximum score (lenient OR) -- complements `all()`'s min-score AND. */
+export declare function any(...checkers: readonly Checker[]): Checker;

package/dist/workspace-checkers.js

@@ -0,0 +1,92 @@
+/**
+ * Outcome checkers verifying real post-run workspace state -- what the agent actually DID, not
+ * which tools it called. Ported from Alef's own `packages/core/eval/src/checker.ts`
+ * (`fileExists`/`fileContains`/`lintPasses`/`any`), unchanged in scoring semantics.
+ */
+import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+function missingWorkspace(label) {
+    return { pass: false, score: 0, errors: [`${label}: no workspace in CheckerContext`] };
+}
+/** A Checker verifying a file exists in the run's workspace. Existence alone scores 0.5 -- content is unchecked. */
+export function fileExists(relativePath) {
+    return {
+        async check({ workspace }) {
+            if (workspace === undefined)
+                return missingWorkspace("fileExists");
+            try {
+                await readFile(join(workspace, relativePath), "utf-8");
+                return { pass: true, score: 0.5, errors: [] };
+            }
+            catch {
+                return { pass: false, score: 0, errors: [`File not found: ${relativePath}`] };
+            }
+        },
+    };
+}
+/**
+ * A Checker verifying a file contains every required substring. Graduated: 1.0 all present,
+ * 0.5 some present, 0.0 none present or the file is missing.
+ */
+export function fileContains(relativePath, ...required) {
+    return {
+        async check({ workspace }) {
+            if (workspace === undefined)
+                return missingWorkspace("fileContains");
+            let content;
+            try {
+                content = await readFile(join(workspace, relativePath), "utf-8");
+            }
+            catch {
+                return { pass: false, score: 0, errors: [`File not found: ${relativePath}`] };
+            }
+            const missing = required.filter((value) => !content.includes(value));
+            if (missing.length === 0)
+                return { pass: true, score: 1, errors: [] };
+            const found = required.length - missing.length;
+            const score = found > 0 ? 0.5 : 0;
+            return { pass: false, score, errors: missing.map((value) => `'${value}' not found in ${relativePath}`) };
+        },
+    };
+}
+/**
+ * A Checker running a real command in the run's workspace and asserting exit code 0 -- an
+ * outcome checker: verifies what the agent DID (does it still build/lint/pass tests), not what
+ * it said.
+ */
+export function lintPasses(cmd, args = []) {
+    return {
+        check({ workspace }) {
+            if (workspace === undefined)
+                return Promise.resolve(missingWorkspace("lintPasses"));
+            return new Promise((resolve) => {
+                const child = spawn(cmd, [...args], { cwd: workspace, stdio: "pipe" });
+                const stderr = [];
+                child.stderr?.on("data", (chunk) => stderr.push(chunk.toString()));
+                child.on("close", (code) => {
+                    if (code === 0) {
+                        resolve({ pass: true, score: 1, errors: [] });
+                    }
+                    else {
+                        resolve({ pass: false, score: 0, errors: [`${cmd} exited ${code}:\n${stderr.join("").trim()}`] });
+                    }
+                });
+                child.on("error", (error) => {
+                    resolve({ pass: false, score: 0, errors: [`Failed to run ${cmd}: ${error.message}`] });
+                });
+            });
+        },
+    };
+}
+/** Composes several checkers, returning the maximum score (lenient OR) -- complements `all()`'s min-score AND. */
+export function any(...checkers) {
+    return {
+        async check(context) {
+            const results = await Promise.all(checkers.map((checker) => checker.check(context)));
+            if (results.length === 0)
+                return { pass: true, score: 1, errors: [] };
+            return results.reduce((best, current) => (current.score >= best.score ? current : best));
+        },
+    };
+}

package/package.json

@@ -1,47 +1,62 @@
 {
-	"name": "@danypops/pi-eval-harness",
-	"version": "0.1.0",
-	"description": "Scores a real agent run's own tool-call behavior -- AND/OR tool-call matching, graduated checker composition, and turn/tool-call/token-usage rollups -- over Pi's own real AgentSessionEvent stream (e.g. from @danypops/pi-process-harness).",
-	"license": "MIT",
-	"type": "module",
-	"main": "./dist/index.js",
-	"sideEffects": false,
-	"types": "./dist/index.d.ts",
-	"exports": {
-		".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
-		"./package.json": "./package.json"
-	},
-	"scripts": {
-		"build": "rm -rf dist && tsc -p tsconfig.build.json",
-		"test": "bun run build && bun test test",
-		"typecheck": "tsc --noEmit",
-		"format": "biome format --write ."
-	},
-	"repository": {
-		"type": "git",
-		"url": "git+https://github.com/DanyPops/pi-integral.git",
-		"directory": "packages/pi-eval-harness"
-	},
-	"homepage": "https://github.com/DanyPops/pi-integral/tree/main/packages/pi-eval-harness#readme",
-	"bugs": {
-		"url": "https://github.com/DanyPops/pi-integral/issues"
-	},
-	"peerDependencies": {
-		"@earendil-works/pi-coding-agent": "*"
-	},
-	"devDependencies": {
-		"@biomejs/biome": "^2.5.6",
-		"@danypops/pi-process-harness": "workspace:*",
-		"@earendil-works/pi-ai": "*",
-		"@earendil-works/pi-coding-agent": "*",
-		"@types/node": "^22.0.0",
-		"bun-types": "latest",
-		"typebox": "*",
-		"typescript": "^5.9.2"
-	},
-	"publishConfig": {
-		"access": "public"
-	},
-	"files": ["src", "dist", "README.md", "LICENSE"],
-	"keywords": ["pi", "pi-extension", "testing", "eval", "agent-evaluation", "tool-call"]
+  "name": "@danypops/pi-eval-harness",
+  "version": "0.2.0",
+  "description": "Scores a real agent run's own tool-call behavior -- AND/OR tool-call matching, graduated checker composition, and turn/tool-call/token-usage rollups -- over Pi's own real AgentSessionEvent stream (e.g. from @danypops/pi-process-harness).",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "sideEffects": false,
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.build.json",
+    "test": "bun run build && bun test test",
+    "typecheck": "tsc --noEmit",
+    "format": "biome format --write ."
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/DanyPops/pi-integral.git",
+    "directory": "packages/pi-eval-harness"
+  },
+  "homepage": "https://github.com/DanyPops/pi-integral/tree/main/packages/pi-eval-harness#readme",
+  "bugs": {
+    "url": "https://github.com/DanyPops/pi-integral/issues"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.5.6",
+    "@danypops/pi-process-harness": "0.2.0",
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@types/node": "^22.0.0",
+    "bun-types": "latest",
+    "typebox": "*",
+    "typescript": "^5.9.2"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "src",
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "pi",
+    "pi-extension",
+    "testing",
+    "eval",
+    "agent-evaluation",
+    "tool-call"
+  ]
 }

package/src/ablation.ts

@@ -0,0 +1,84 @@
+/**
+ * Runs one scenario under several named tool-availability configs and diffs the results --
+ * ported from djinn's own `testkit/eval_ablation.go` (`AblationConfig`/`AblationResult`/
+ * `AblationDelta`/`Ablate()`) and `eval_report.go`'s `FormatAblation()`. The first config is the
+ * baseline; every later config's own TrialMetrics get a delta computed against it.
+ */
+import { type RunTrialsOptions, runTrials, type TrialMetrics, type TrialResult } from "./trials.js";
+
+/** One named variant of the scenario -- e.g. "baseline" (Pi's own built-in tools) vs "with-lector" (the same run, Lector's tools also registered). */
+export interface AblationConfig {
+	readonly name: string;
+	/** Runs one real trial under this config. Whatever varies between arms (which extensions load, which tools are registered) lives inside this closure. */
+	readonly runOne: () => Promise<TrialResult>;
+}
+
+/** The performance difference of one config's TrialMetrics vs the baseline (first) config's. */
+export interface AblationDelta {
+	readonly passRateDelta: number;
+	readonly meanScoreDelta: number;
+	readonly meanDurationMsDelta: number;
+	readonly meanTokensInDelta: number;
+}
+
+/** One config's own trial metrics, plus its delta vs baseline. `delta` is undefined for the baseline itself. */
+export interface AblationResult {
+	readonly config: AblationConfig;
+	readonly metrics: TrialMetrics;
+	readonly delta?: AblationDelta;
+}
+
+/**
+ * Runs `n` trials of every config (sequentially, config by config -- each config's own trials
+ * still run at `options.concurrency` internally) and returns side-by-side results. Configs are
+ * independent scenarios in the caller's own domain (e.g. real spawned `pi` processes with
+ * different extension sets); this function only orchestrates trial counts and diffs.
+ */
+export async function ablate(configs: readonly AblationConfig[], n: number, options: RunTrialsOptions = {}): Promise<AblationResult[]> {
+	if (configs.length === 0) return [];
+
+	const results: AblationResult[] = [];
+	for (const config of configs) {
+		results.push({ config, metrics: await runTrials(config.runOne, n, options) });
+	}
+
+	const baseline = results[0]?.metrics;
+	if (baseline === undefined) return results;
+
+	return results.map((result, index) => {
+		if (index === 0) return result;
+		const metrics = result.metrics;
+		return {
+			...result,
+			delta: {
+				passRateDelta: metrics.passRate - baseline.passRate,
+				meanScoreDelta: metrics.meanScore - baseline.meanScore,
+				meanDurationMsDelta: metrics.meanDurationMs - baseline.meanDurationMs,
+				meanTokensInDelta: metrics.meanTokensIn - baseline.meanTokensIn,
+			},
+		};
+	});
+}
+
+function signed(value: number, digits: number): string {
+	const rounded = value.toFixed(digits);
+	return value >= 0 ? `+${rounded}` : rounded;
+}
+
+/** Renders a human-readable ablation comparison table -- ported from djinn's `FormatAblation()`. */
+export function formatAblation(label: string, results: readonly AblationResult[]): string {
+	const lines: string[] = [`=== Ablation: ${label} ===`];
+	for (const result of results) {
+		const m = result.metrics;
+		lines.push(
+			`  ${result.config.name.padEnd(14)}  pass_rate=${m.passRate.toFixed(2)}  mean_score=${m.meanScore.toFixed(2)}  mean_duration_ms=${m.meanDurationMs.toFixed(0)}  tokens_in=${m.meanTokensIn.toFixed(0)}`,
+		);
+		if (result.delta) {
+			const d = result.delta;
+			lines.push(
+				`  ${"(vs baseline)".padEnd(14)}  Δpass_rate=${signed(d.passRateDelta, 2)}  Δscore=${signed(d.meanScoreDelta, 2)}  Δduration_ms=${signed(d.meanDurationMsDelta, 0)}  Δtokens_in=${signed(d.meanTokensInDelta, 0)}`,
+			);
+		}
+	}
+	return lines.join("\n");
+}

package/src/checker.ts

@@ -9,9 +9,16 @@ export interface CheckerResult {
 	readonly errors: readonly string[];
 }
 
-/** Runtime context passed to a Checker -- the real completed tool executions from one run. */
+/**
+ * Runtime context passed to a Checker -- the real completed tool executions from one run, plus
+ * (when the run actually had a real workspace directory) its absolute path, for a checker that
+ * verifies real post-run workspace state rather than which tools were called. Optional: a
+ * checker that only inspects `executions` (matching/rollup checks) never needs it, and every
+ * existing construction of this context predates the field.
+ */
 export interface CheckerContext {
 	readonly executions: readonly ToolExecution[];
+	readonly workspace?: string;
 }
 
 /** A pure, deterministic verifier over one run's real tool executions. */

package/src/index.ts

@@ -1,4 +1,21 @@
+export {
+	type AblationConfig,
+	type AblationDelta,
+	type AblationResult,
+	ablate,
+	formatAblation,
+} from "./ablation.js";
 export { all, type Checker, type CheckerContext, type CheckerResult, expectsAll, expectsAny } from "./checker.js";
+export { detectLoop, type LoopDetectionResult, type MistakeSummary, summarizeMistakes } from "./mistakes.js";
 export { describeToolCall, matchesToolCall, type ToolCall } from "./tool-call.js";
 export { extractToolExecutions, type ToolExecution } from "./tool-executions.js";
+export {
+	aggregateTrials,
+	MaxErrorRateExceeded,
+	type RunTrialsOptions,
+	runTrials,
+	type TrialMetrics,
+	type TrialResult,
+} from "./trials.js";
 export { deriveTurns, type RunUsageSummary, summarizeRunUsage, type Turn } from "./turns.js";
+export { any, fileContains, fileExists, lintPasses } from "./workspace-checkers.js";

package/src/mistakes.ts

@@ -0,0 +1,60 @@
+/**
+ * Mistake-rate and loop-detection rollups over one run's real tool executions -- ported from
+ * djinn's own `LoopThreshold`/`LoopDetected`/`LoopTool` (`testkit/eval_evaluation.go`/
+ * `eval_runner.go`): the exact edit-build-fail-retry failure mode the JetBrains Rider blog
+ * post's own worked example showed in detail (11 edit/build cycles fighting the compiler).
+ */
+import type { ToolExecution } from "./tool-executions.js";
+
+/** Real, per-execution error signal rolled up across a whole run. */
+export interface MistakeSummary {
+	readonly totalExecutions: number;
+	/** Count of tool_execution_end events with isError: true. */
+	readonly errorCount: number;
+	/** errorCount / totalExecutions; 0 for a run with no executions at all. */
+	readonly errorRate: number;
+}
+
+/** Rolls up isError across every completed execution in a run. */
+export function summarizeMistakes(executions: readonly ToolExecution[]): MistakeSummary {
+	const errorCount = executions.filter((execution) => execution.isError).length;
+	return {
+		totalExecutions: executions.length,
+		errorCount,
+		errorRate: executions.length === 0 ? 0 : errorCount / executions.length,
+	};
+}
+
+export interface LoopDetectionResult {
+	readonly loopDetected: boolean;
+	/** The tool name that triggered detection. Present only when loopDetected is true. */
+	readonly loopTool?: string;
+	/** The real highest same-tool call count observed, regardless of whether it crossed the threshold. */
+	readonly maxToolCallCount: number;
+}
+
+/**
+ * Detects a real repeated-tool-call pattern: the same tool name called more than `threshold`
+ * times in one run (default 10, matching djinn's own default). A simple total-count threshold
+ * per tool name, not a strict-consecutive-run check -- matching djinn's own definition.
+ */
+export function detectLoop(executions: readonly ToolExecution[], threshold = 10): LoopDetectionResult {
+	const counts = new Map<string, number>();
+	for (const execution of executions) {
+		counts.set(execution.toolName, (counts.get(execution.toolName) ?? 0) + 1);
+	}
+
+	let maxToolCallCount = 0;
+	let loopTool: string | undefined;
+	for (const [toolName, count] of counts) {
+		if (count > maxToolCallCount) {
+			maxToolCallCount = count;
+			loopTool = toolName;
+		}
+	}
+
+	if (maxToolCallCount > threshold && loopTool !== undefined) {
+		return { loopDetected: true, loopTool, maxToolCallCount };
+	}
+	return { loopDetected: false, maxToolCallCount };
+}

package/src/trials.ts

@@ -0,0 +1,132 @@
+/**
+ * Multi-trial aggregation over N real, non-deterministic runs of the same scenario -- ported
+ * from Alef's own `EvaluationRunner.runN` (pass@k, variance, min/max score, a bounded
+ * concurrency cap) and djinn's `TrialMetrics` (pass rate, mean score/latency/tokens) shape.
+ * A single real live-LLM trial's score is noise, not signal -- this is what turns it into one.
+ */
+
+/** One real trial's own outcome. `error` set means the trial itself failed to run to completion -- a real infrastructure/runtime failure, distinct from a checker legitimately scoring it 0. */
+export interface TrialResult {
+	readonly pass: boolean;
+	readonly score: number;
+	readonly durationMs: number;
+	readonly tokensIn: number;
+	readonly tokensOut: number;
+	readonly costUsd: number;
+	readonly error?: string;
+}
+
+/** Aggregate statistics across N trials of one scenario. */
+export interface TrialMetrics {
+	readonly trials: number;
+	readonly passes: number;
+	readonly passRate: number;
+	readonly meanScore: number;
+	readonly variance: number;
+	readonly minScore: number;
+	readonly maxScore: number;
+	readonly meanDurationMs: number;
+	readonly meanTokensIn: number;
+	readonly meanTokensOut: number;
+	readonly meanCostUsd: number;
+}
+
+/**
+ * Thrown when too many trials errored outright to trust the aggregate as a real measurement --
+ * ported from Alef's own `[MaxErrorRate]` guard. Without this, a scenario that errors on every
+ * trial would silently aggregate to "0% pass", indistinguishable from a real, measured failure.
+ */
+export class MaxErrorRateExceeded extends Error {
+	constructor(
+		readonly errorRate: number,
+		readonly maxErrorRate: number,
+		readonly errorCount: number,
+		readonly totalTrials: number,
+		firstError: string | undefined,
+	) {
+		super(
+			`${(errorRate * 100).toFixed(0)}% of trials errored (${errorCount}/${totalTrials}), threshold ${(maxErrorRate * 100).toFixed(0)}%. First error: ${firstError ?? "unknown"}`,
+		);
+		this.name = "MaxErrorRateExceeded";
+	}
+}
+
+function mean(values: readonly number[]): number {
+	return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+/**
+ * Aggregates already-collected trial results into TrialMetrics. Throws MaxErrorRateExceeded if
+ * `maxErrorRate` (0-1, default 0 = disabled) is exceeded by the fraction of errored trials.
+ */
+export function aggregateTrials(results: readonly TrialResult[], options: { readonly maxErrorRate?: number } = {}): TrialMetrics {
+	if (results.length === 0) throw new Error("aggregateTrials: at least one trial result is required");
+
+	const maxErrorRate = options.maxErrorRate ?? 0;
+	if (maxErrorRate > 0) {
+		const errored = results.filter((result) => result.error !== undefined);
+		const errorRate = errored.length / results.length;
+		if (errorRate > maxErrorRate) {
+			throw new MaxErrorRateExceeded(errorRate, maxErrorRate, errored.length, results.length, errored[0]?.error);
+		}
+	}
+
+	const scores = results.map((result) => result.score);
+	const passes = results.filter((result) => result.pass).length;
+	const meanScore = mean(scores);
+
+	return {
+		trials: results.length,
+		passes,
+		passRate: passes / results.length,
+		meanScore,
+		variance: mean(scores.map((score) => (score - meanScore) ** 2)),
+		minScore: Math.min(...scores),
+		maxScore: Math.max(...scores),
+		meanDurationMs: mean(results.map((result) => result.durationMs)),
+		meanTokensIn: mean(results.map((result) => result.tokensIn)),
+		meanTokensOut: mean(results.map((result) => result.tokensOut)),
+		meanCostUsd: mean(results.map((result) => result.costUsd)),
+	};
+}
+
+export interface RunTrialsOptions {
+	/** Trials run concurrently at once. Default 3, matching Alef's own ALEF_EVAL_CONCURRENCY default -- real LLM providers rate-limit. */
+	readonly concurrency?: number;
+	readonly maxErrorRate?: number;
+}
+
+/**
+ * Runs `runOne` (one real trial of a scenario) `n` times at a bounded concurrency, then
+ * aggregates. A rejected `runOne` call becomes a TrialResult with `error` set (score 0, pass
+ * false) rather than aborting the whole batch -- one crashed trial should not lose every other
+ * trial's own real data.
+ */
+export async function runTrials(runOne: () => Promise<TrialResult>, n: number, options: RunTrialsOptions = {}): Promise<TrialMetrics> {
+	const concurrency = options.concurrency ?? 3;
+	const results: TrialResult[] = [];
+
+	for (let i = 0; i < n; i += concurrency) {
+		const batchSize = Math.min(concurrency, n - i);
+		const batch = await Promise.all(
+			Array.from({ length: batchSize }, async (): Promise<TrialResult> => {
+				try {
+					return await runOne();
+				} catch (error) {
+					return {
+						pass: false,
+						score: 0,
+						durationMs: 0,
+						tokensIn: 0,
+						tokensOut: 0,
+						costUsd: 0,
+						error: error instanceof Error ? error.message : String(error),
+					};
+				}
+			}),
+		);
+		results.push(...batch);
+	}
+
+	return aggregateTrials(results, options.maxErrorRate !== undefined ? { maxErrorRate: options.maxErrorRate } : {});
+}

package/src/workspace-checkers.ts

@@ -0,0 +1,92 @@
+/**
+ * Outcome checkers verifying real post-run workspace state -- what the agent actually DID, not
+ * which tools it called. Ported from Alef's own `packages/core/eval/src/checker.ts`
+ * (`fileExists`/`fileContains`/`lintPasses`/`any`), unchanged in scoring semantics.
+ */
+import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { Checker, CheckerContext, CheckerResult } from "./checker.js";
+
+function missingWorkspace(label: string): CheckerResult {
+	return { pass: false, score: 0, errors: [`${label}: no workspace in CheckerContext`] };
+}
+
+/** A Checker verifying a file exists in the run's workspace. Existence alone scores 0.5 -- content is unchecked. */
+export function fileExists(relativePath: string): Checker {
+	return {
+		async check({ workspace }: CheckerContext): Promise<CheckerResult> {
+			if (workspace === undefined) return missingWorkspace("fileExists");
+			try {
+				await readFile(join(workspace, relativePath), "utf-8");
+				return { pass: true, score: 0.5, errors: [] };
+			} catch {
+				return { pass: false, score: 0, errors: [`File not found: ${relativePath}`] };
+			}
+		},
+	};
+}
+
+/**
+ * A Checker verifying a file contains every required substring. Graduated: 1.0 all present,
+ * 0.5 some present, 0.0 none present or the file is missing.
+ */
+export function fileContains(relativePath: string, ...required: readonly string[]): Checker {
+	return {
+		async check({ workspace }: CheckerContext): Promise<CheckerResult> {
+			if (workspace === undefined) return missingWorkspace("fileContains");
+			let content: string;
+			try {
+				content = await readFile(join(workspace, relativePath), "utf-8");
+			} catch {
+				return { pass: false, score: 0, errors: [`File not found: ${relativePath}`] };
+			}
+
+			const missing = required.filter((value) => !content.includes(value));
+			if (missing.length === 0) return { pass: true, score: 1, errors: [] };
+
+			const found = required.length - missing.length;
+			const score = found > 0 ? 0.5 : 0;
+			return { pass: false, score, errors: missing.map((value) => `'${value}' not found in ${relativePath}`) };
+		},
+	};
+}
+
+/**
+ * A Checker running a real command in the run's workspace and asserting exit code 0 -- an
+ * outcome checker: verifies what the agent DID (does it still build/lint/pass tests), not what
+ * it said.
+ */
+export function lintPasses(cmd: string, args: readonly string[] = []): Checker {
+	return {
+		check({ workspace }: CheckerContext): Promise<CheckerResult> {
+			if (workspace === undefined) return Promise.resolve(missingWorkspace("lintPasses"));
+			return new Promise((resolve) => {
+				const child = spawn(cmd, [...args], { cwd: workspace, stdio: "pipe" });
+				const stderr: string[] = [];
+				child.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk.toString()));
+				child.on("close", (code) => {
+					if (code === 0) {
+						resolve({ pass: true, score: 1, errors: [] });
+					} else {
+						resolve({ pass: false, score: 0, errors: [`${cmd} exited ${code}:\n${stderr.join("").trim()}`] });
+					}
+				});
+				child.on("error", (error) => {
+					resolve({ pass: false, score: 0, errors: [`Failed to run ${cmd}: ${error.message}`] });
+				});
+			});
+		},
+	};
+}
+
+/** Composes several checkers, returning the maximum score (lenient OR) -- complements `all()`'s min-score AND. */
+export function any(...checkers: readonly Checker[]): Checker {
+	return {
+		async check(context: CheckerContext): Promise<CheckerResult> {
+			const results = await Promise.all(checkers.map((checker) => checker.check(context)));
+			if (results.length === 0) return { pass: true, score: 1, errors: [] };
+			return results.reduce((best, current) => (current.score >= best.score ? current : best));
+		},
+	};
+}