@sebastiantuyu/agest 0.3.3-next.10 → 0.3.3-next.12

package/dist/cli.d.ts

@@ -1,8 +1,26 @@
 #!/usr/bin/env node
+import type { CheckpointRecord } from "./types.js";
+/**
+ * One record per `agent()` run, appended by the child process (see
+ * AgentContext.execute → AGEST_SUMMARY_FILE). The parent reads them all back to
+ * print a vitest-style footer across files.
+ */
+interface RunSummaryRecord {
+    file: string;
+    name?: string;
+    total: number;
+    passed: number;
+    failed: number;
+    duration: number;
+    costUsd: number | null;
+    /** Full checkpoint payload the parent appends to the canonical run log. */
+    checkpoint?: CheckpointRecord;
+}
 export interface ParsedRunArgs {
     pattern?: string;
     targets: string[];
     full: boolean;
+    record: boolean;
 }
 /**
  * Extract the args that follow the command word from a full `process.argv`.
@@ -13,4 +31,25 @@ export interface ParsedRunArgs {
  */
 export declare function getCommandArgs(argv: string[]): string[];
 export declare function parseRunArgs(args: string[]): ParsedRunArgs;
+export interface RunSummary {
+    /** Whether the footer should print — false for a single scene in one file. */
+    show: boolean;
+    discoveredFiles: number;
+    filesPassed: number;
+    filesFailed: number;
+    totalCases: number;
+    casesPassed: number;
+    casesFailed: number;
+    duration: number;
+    cost: number;
+}
+/**
+ * Aggregate every child's records into the footer totals. The footer only
+ * shows when more than one case ran (multiple files, or one file with multiple
+ * scenes) — a single scene already prints its own one-line summary. A file
+ * counts as failed if any of its agent() runs had a failing case, or if it
+ * never wrote a record (crashed before reporting).
+ */
+export declare function aggregateRunSummary(records: RunSummaryRecord[], discoveredFiles: number): RunSummary;
 export declare function main(argv: string[]): Promise<void>;
+export {};

package/dist/cli.js

@@ -1,10 +1,14 @@
 #!/usr/bin/env node
 import { spawn } from "child_process";
 import { fileURLToPath } from "node:url";
-import { realpathSync } from "node:fs";
+import { realpathSync, mkdtempSync, mkdirSync, readFileSync, appendFileSync, rmSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
 import { main as stats } from "./stats.js";
 import { main as preview } from "./preview.js";
 import { DEFAULT_PATTERN, discoverTestFiles } from "./discover.js";
+import { c } from "./logger.js";
 /**
  * Extract the args that follow the command word from a full `process.argv`.
  * `argv = [execPath, scriptPath, command, ...commandArgs]`, so the command's
@@ -19,6 +23,7 @@ export function parseRunArgs(args) {
     const targets = [];
     let pattern;
     let full = false;
+    let record = false;
     for (let i = 0; i < args.length; i++) {
         const a = args[i];
         if (a === "--pattern" || a === "-p") {
@@ -34,34 +39,146 @@ export function parseRunArgs(args) {
         else if (a === "--full") {
             full = true;
         }
+        else if (a === "--record") {
+            record = true;
+        }
         else {
             targets.push(a);
         }
     }
-    return { pattern, targets, full };
+    return { pattern, targets, full, record };
 }
 async function run(args) {
-    const { pattern, targets, full } = parseRunArgs(args);
+    const { pattern, targets, full, record } = parseRunArgs(args);
     const files = await discoverTestFiles(targets, { pattern });
     if (files.length === 0) {
         const effective = pattern ?? DEFAULT_PATTERN;
         console.error(`  No test files found (pattern: ${effective})`);
         process.exit(1);
     }
+    // Each child appends a summary record here; the parent reads them back for
+    // the aggregate footer. A unique dir keeps concurrent `agest run`s isolated.
+    const summaryFile = join(mkdtempSync(join(tmpdir(), "agest-")), "summary.jsonl");
+    // One sweepId per invocation groups every checkpoint row from this run.
+    const sweepId = randomUUID();
+    const childEnv = {
+        ...process.env,
+        AGEST_SUMMARY_FILE: summaryFile,
+        AGEST_SWEEP_ID: sweepId,
+        // The test file renders its own output in a child process; propagate
+        // --full so it emits the waterfall + full report rather than lean results.
+        ...(full ? { AGEST_FULL: "1" } : {}),
+        // Opt-in: persist a full per-scene YAML snapshot per agent() execution.
+        ...(record ? { AGEST_RECORD: "1" } : {}),
+    };
+    let anyChildCrashed = false;
+    // Run every file (vitest-style) instead of bailing on the first failure, so
+    // the footer reflects the whole run. Exit non-zero at the end if any failed.
     for (const file of files) {
         const child = spawn("npx", ["tsx", file], {
             stdio: "inherit",
             shell: true,
-            // The test file renders its own output in a child process; propagate the
-            // --full flag through the environment so it knows to emit the waterfall
-            // and full report rather than just per-scene results.
-            env: full ? { ...process.env, AGEST_FULL: "1" } : process.env,
+            env: childEnv,
         });
         const code = await new Promise((resolve) => child.on("close", (c) => resolve(c ?? 1)));
+        // A non-zero code means the file itself threw/crashed. Failing scenes do
+        // NOT surface here — the child resolves cleanly — so failure is read back
+        // from the summary records below.
         if (code !== 0)
-            process.exit(code);
+            anyChildCrashed = true;
+    }
+    const records = readSummary(summaryFile);
+    writeCheckpoints(records);
+    printRunSummary(records, files.length);
+    try {
+        rmSync(dirname(summaryFile), { recursive: true, force: true });
+    }
+    catch {
+        /* best-effort cleanup */
+    }
+    const casesFailed = records.reduce((sum, r) => sum + r.failed, 0);
+    if (anyChildCrashed || casesFailed > 0)
+        process.exit(1);
+}
+function readSummary(summaryFile) {
+    try {
+        return readFileSync(summaryFile, "utf8")
+            .split("\n")
+            .filter(Boolean)
+            .map((line) => JSON.parse(line));
+    }
+    catch {
+        return []; // no children wrote results (older lib, or all crashed early)
     }
 }
+/**
+ * The parent is the single writer of the canonical run log: append every
+ * child's checkpoint record to `.reports/checkpoints.jsonl` in one buffer
+ * (race-free across the spawned children). Best-effort — never break a run.
+ */
+function writeCheckpoints(records) {
+    const checkpoints = records
+        .map((r) => r.checkpoint)
+        .filter((c) => c != null);
+    if (checkpoints.length === 0)
+        return;
+    try {
+        const dir = join(process.cwd(), ".reports");
+        mkdirSync(dir, { recursive: true });
+        const lines = checkpoints.map((c) => JSON.stringify(c)).join("\n") + "\n";
+        appendFileSync(join(dir, "checkpoints.jsonl"), lines, "utf8");
+    }
+    catch {
+        /* ignore */
+    }
+}
+/**
+ * Aggregate every child's records into the footer totals. The footer only
+ * shows when more than one case ran (multiple files, or one file with multiple
+ * scenes) — a single scene already prints its own one-line summary. A file
+ * counts as failed if any of its agent() runs had a failing case, or if it
+ * never wrote a record (crashed before reporting).
+ */
+export function aggregateRunSummary(records, discoveredFiles) {
+    const totalCases = records.reduce((sum, r) => sum + r.total, 0);
+    const failsByFile = new Map();
+    for (const r of records) {
+        failsByFile.set(r.file, (failsByFile.get(r.file) ?? 0) + r.failed);
+    }
+    const missing = Math.max(0, discoveredFiles - failsByFile.size);
+    const filesFailed = [...failsByFile.values()].filter((f) => f > 0).length + missing;
+    const casesPassed = records.reduce((sum, r) => sum + r.passed, 0);
+    return {
+        show: records.length > 0 && (discoveredFiles > 1 || totalCases > 1),
+        discoveredFiles,
+        filesPassed: discoveredFiles - filesFailed,
+        filesFailed,
+        totalCases,
+        casesPassed,
+        casesFailed: totalCases - casesPassed,
+        duration: records.reduce((sum, r) => sum + (r.duration || 0), 0),
+        cost: records.reduce((sum, r) => sum + (r.costUsd ?? 0), 0),
+    };
+}
+/**
+ * Print the vitest-style footer. Delegates the math to aggregateRunSummary and
+ * only renders when that says so.
+ */
+function printRunSummary(records, discoveredFiles) {
+    const s = aggregateRunSummary(records, discoveredFiles);
+    if (!s.show)
+        return;
+    const tally = (failed, passed, total) => failed > 0
+        ? `${c.red(`${failed} failed`)} ${c.dim("|")} ${c.green(`${passed} passed`)} ${c.dim(`(${total})`)}`
+        : `${c.green(`${passed} passed`)} ${c.dim(`(${total})`)}`;
+    const line = (label, value) => console.log(`${c.dim(label.padStart(11))}  ${value}`);
+    console.log("");
+    line("Test Files", tally(s.filesFailed, s.filesPassed, s.discoveredFiles));
+    line("Tests", tally(s.casesFailed, s.casesPassed, s.totalCases));
+    line("Duration", `${s.duration}ms`);
+    if (s.cost > 0)
+        line("Cost", c.green(`$${Number(s.cost.toFixed(4))}`));
+}
 function printUsage() {
     console.log(`
   Usage: agest <command>
@@ -72,7 +189,10 @@ function printUsage() {
                agest run src/agest --pattern "**/*.test.ts"
                agest run "tests/**/*.agest.ts" path/to/file.agest.ts
                agest run tests/ --full                # also print waterfall + full report
+               agest run tests/ --record              # also save a full per-scene snapshot
     stats      Show aggregated test statistics
+               agest stats --suite <suiteHash>        # filter to one suite's history
+               agest stats --export-csv [path]        # flatten the run log to CSV
     preview    Generate an HTML report preview
 `);
 }

package/dist/context.d.ts

@@ -53,5 +53,14 @@ export declare class AgentContext<T = string> {
     execute(): Promise<AgentReport<T>>;
 }
 export declare function hashPromptOnly(prompt: string): string;
+/**
+ * Identity hash of the test suite: prompts, suite names, assertion field names +
+ * bodies (`fn.toString()`), and schema presence. Makes the suite a first-class
+ * dimension so a comparison can never silently span a changed set of scenes.
+ * `fn.toString()` is formatting/closure-sensitive — it over-segments (a cosmetic
+ * edit yields a new hash) but never silently merges two different suites, which
+ * is the safe direction.
+ */
+export declare function computeSuiteHash(definitions: SceneDefinition[]): string;
 export declare function setContext(ctx: AgentContext<any> | null): void;
 export declare function getContext(): AgentContext<any>;

package/dist/context.js

@@ -1,7 +1,10 @@
-import { createHash } from "crypto";
+import { createHash, randomUUID } from "crypto";
+import { appendFileSync } from "node:fs";
+import { relative } from "node:path";
 import { executeScene } from "./runner";
 import { resolveText } from "./resolve";
-import { formatReport, writeReport, writeDiffEntry } from "./reporter";
+import { formatReport, writeSnapshot, appendCheckpoint, writeDiffEntry } from "./reporter";
+import { wilsonInterval } from "./reports";
 import { logger, c } from "./logger";
 import { loadConfig } from "./config";
 import { setPricingOverrides } from "./pricing";
@@ -239,6 +242,9 @@ export class AgentContext {
             : undefined;
         const firstMeta = results.find((r) => r.response.metadata)?.response
             .metadata;
+        // Config identity. suiteHash + judge + runs complete the dimension set so
+        // comparisons never silently span a changed suite or sampling config.
+        // `temperature` is read opportunistically (open metadata map) when present.
         const dimensions = {};
         if (firstMeta?.model)
             dimensions.model = firstMeta.model;
@@ -248,6 +254,28 @@ export class AgentContext {
             dimensions.tools = [...firstMeta.tools].sort().join(",");
         else
             dimensions.tools = "none";
+        dimensions.suiteHash = computeSuiteHash(definitions);
+        dimensions.judge = config.judge?.model ?? "none";
+        dimensions.runs = String(config.runs ?? 1);
+        const temperature = firstMeta?.temperature;
+        if (temperature != null)
+            dimensions.temperature = String(temperature);
+        // Report-level statistical honesty. The trial basis is Σ runs across every
+        // scene (a multi-run scene contributes N trials), not just case count.
+        const casesPassed = results.filter((r) => r.passed).length;
+        let trials = 0;
+        let trialPasses = 0;
+        for (const r of results) {
+            if (r.runs && r.runs.length) {
+                trials += r.runs.length;
+                trialPasses += r.runs.filter((x) => x.passed).length;
+            }
+            else {
+                trials += 1;
+                trialPasses += r.passed ? 1 : 0;
+            }
+        }
+        const wilson = wilsonInterval(trialPasses, trials);
         const report = {
             name: this._name,
             model: firstMeta?.model,
@@ -265,6 +293,10 @@ export class AgentContext {
             timestamp: new Date().toISOString(),
             duration: Math.round(totalDuration),
             totalCases: results.length,
+            casesPassed,
+            runsPerScene: config.runs ?? 1,
+            wilsonLow: wilson.low,
+            wilsonHigh: wilson.high,
             averageInputTokensPerCase,
             averageOutputTokensPerCase,
             totalInputTokens,
@@ -286,8 +318,70 @@ export class AgentContext {
             const costSummary = totalCostUsd != null ? ` ${c.dim("·")} ${c.green(`$${Number(totalCostUsd.toFixed(4))}`)}` : "";
             logger.info(`${rateColor(`${passed}/${results.length} passed`)} ${c.dim(`(${(successRate * 100).toFixed(0)}%)`)} ${c.dim("·")} ${c.dim(`${Math.round(totalDuration)}ms`)}${costSummary}`);
         }
-        const filepath = await writeReport(formatted, report.timestamp, report.name, report.dimensions);
-        logger.info(`${c.dim("Report saved to:")} ${c.cyan(filepath)}${full ? "" : c.dim(" (run with --full to print it)")}`);
+        // A unique runId per agent() execution names the optional snapshot (so
+        // snapshots never clobber) and tags the checkpoint record.
+        const sweepId = process.env.AGEST_SWEEP_ID;
+        const runId = `${sweepId ?? "local"}-${randomUUID().slice(0, 8)}`;
+        // Heavy per-scene snapshot is opt-in via --record (AGEST_RECORD).
+        let recordPath;
+        if (process.env.AGEST_RECORD === "1") {
+            const snapPath = await writeSnapshot(formatted, runId);
+            recordPath = relative(process.cwd(), snapPath);
+            logger.info(`${c.dim("Snapshot saved to:")} ${c.cyan(recordPath)}`);
+        }
+        const checkpoint = {
+            runId,
+            sweepId,
+            timestamp: report.timestamp,
+            agentName: this._name,
+            model: report.model,
+            systemPromptHash: report.systemPromptHash,
+            tools: report.tools,
+            dimensions,
+            runsPerScene: report.runsPerScene,
+            totalCases: report.totalCases,
+            casesPassed,
+            successRate,
+            wilsonLow: report.wilsonLow,
+            wilsonHigh: report.wilsonHigh,
+            durationMs: Math.round(totalDuration),
+            costUsd: totalCostUsd ?? null,
+            totalInputTokens,
+            totalOutputTokens,
+            avgInputTokensPerCase: averageInputTokensPerCase,
+            avgOutputTokensPerCase: averageOutputTokensPerCase,
+            recordPath,
+        };
+        // When launched by `agest run`, hand the record to the parent (single writer
+        // of the checkpoint log) and let it print the cross-file footer. Standalone
+        // (`tsx foo.agest.ts`), this process is the lone writer — append directly.
+        // Best-effort throughout: never let persistence break a run.
+        const summaryFile = process.env.AGEST_SUMMARY_FILE;
+        if (summaryFile) {
+            try {
+                appendFileSync(summaryFile, JSON.stringify({
+                    file: process.argv[1],
+                    name: this._name,
+                    total: results.length,
+                    passed: casesPassed,
+                    failed: results.length - casesPassed,
+                    duration: Math.round(totalDuration),
+                    costUsd: totalCostUsd ?? null,
+                    checkpoint,
+                }) + "\n");
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        else {
+            try {
+                await appendCheckpoint(checkpoint);
+            }
+            catch {
+                /* ignore */
+            }
+        }
         return report;
     }
 }
@@ -298,6 +392,23 @@ function hashPrompt(prompt, model) {
 export function hashPromptOnly(prompt) {
     return createHash("sha256").update(prompt).digest("hex").slice(0, 12);
 }
+/**
+ * Identity hash of the test suite: prompts, suite names, assertion field names +
+ * bodies (`fn.toString()`), and schema presence. Makes the suite a first-class
+ * dimension so a comparison can never silently span a changed set of scenes.
+ * `fn.toString()` is formatting/closure-sensitive — it over-segments (a cosmetic
+ * edit yields a new hash) but never silently merges two different suites, which
+ * is the safe direction.
+ */
+export function computeSuiteHash(definitions) {
+    const canonical = definitions.map((d) => ({
+        prompt: d.prompt,
+        suite: d.suite ?? null,
+        schema: d.schema ? "1" : "0",
+        assertions: d.assertions.map((a) => ({ field: a.field, fn: a.fn.toString() })),
+    }));
+    return createHash("sha256").update(JSON.stringify(canonical)).digest("hex").slice(0, 12);
+}
 // The active context is a runtime singleton holding an executor of arbitrary
 // value type, so `any` is the honest type for the holder. The generic flows
 // through `agent()` → `AgentContext<T>` → the report at the call site.

package/dist/preview.js

@@ -1,8 +1,8 @@
-import { readFile, writeFile } from "fs/promises";
-import { join, relative } from "path";
+import { writeFile } from "fs/promises";
+import { join } from "path";
 import os from "os";
 import { exec } from "child_process";
-import { parseReport, findReports, loadDiffEntry, wilsonLowerBound, computeDiff, formatDuration, ensureDimensions, findVaryingDimensions, groupByDimension, findControlledPairs, diffConfigs, } from "./reports.js";
+import { loadReports, loadDiffEntry, wilsonLowerBound, computeDiff, formatDuration, findVaryingDimensions, groupByDimension, findControlledPairs, diffConfigs, } from "./reports.js";
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -1313,17 +1313,11 @@ function generateHTML(groups, totalReports) {
 // ---------------------------------------------------------------------------
 async function main() {
     const cwd = process.cwd();
-    const files = await findReports(cwd);
-    if (files.length === 0) {
+    const reports = await loadReports(cwd);
+    if (reports.length === 0) {
         console.log("\n  No reports found. Run some agent tests first.\n");
         return;
     }
-    const reports = await Promise.all(files.map(async (f) => {
-        const content = await readFile(f, "utf-8");
-        return parseReport(content, relative(cwd, f));
-    }));
-    // Ensure all reports have dimensions (backward compat)
-    await Promise.all(reports.map((r) => ensureDimensions(r)));
     // Group by agent name, sort each group oldest -> newest
     const groupMap = new Map();
     for (const r of reports) {

package/dist/reporter.d.ts

@@ -1,4 +1,15 @@
-import type { AgentReport } from "./types";
+import type { AgentReport, CheckpointRecord } from "./types";
 export declare function formatReport(report: AgentReport<unknown>): string;
-export declare function writeReport(content: string, timestamp: string, name?: string, dimensions?: Record<string, string>): Promise<string>;
+/**
+ * Write a full per-scene YAML report under `.reports/runs/<runId>.yaml`. The
+ * runId is unique per `agent()` execution, so snapshots never clobber and need
+ * no locking. Only written when a run opts in via `--record`.
+ */
+export declare function writeSnapshot(content: string, runId: string): Promise<string>;
+/**
+ * Append one record to the canonical append-only run log
+ * (`.reports/checkpoints.jsonl`). Used on the standalone path (a lone test-file
+ * process). When launched by `agest run`, the PARENT owns this write instead.
+ */
+export declare function appendCheckpoint(record: CheckpointRecord): Promise<void>;
 export declare function writeDiffEntry(hash: string, systemPrompt: string, tools: string[], model?: string): Promise<void>;

package/dist/reporter.js

@@ -1,5 +1,4 @@
-import { access, mkdir, writeFile } from "fs/promises";
-import { createHash } from "crypto";
+import { access, appendFile, mkdir, writeFile } from "fs/promises";
 import { join } from "path";
 import { resolveText } from "./resolve";
 export function formatReport(report) {
@@ -153,29 +152,28 @@ function formatUsd(n) {
 function escapeYaml(s) {
     return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
 }
-export async function writeReport(content, timestamp, name, dimensions) {
-    const reportsDir = join(process.cwd(), ".reports");
-    await mkdir(reportsDir, { recursive: true });
-    const safename = name ? `-${name.replace(/[^a-zA-Z0-9_-]/g, "_")}` : "";
-    let filename;
-    if (dimensions && Object.keys(dimensions).length > 0) {
-        const sorted = Object.entries(dimensions).sort(([a], [b]) => a.localeCompare(b));
-        const dimHash = createHash("sha256").update(JSON.stringify(sorted)).digest("hex").slice(0, 8);
-        filename = `report${safename}-${dimHash}.yaml`;
-    }
-    else {
-        const safestamp = timestamp.replace(/[:.]/g, "-");
-        filename = `report${safename}-${safestamp}.yaml`;
-    }
-    const filepath = join(reportsDir, filename);
-    try {
-        await access(filepath);
-        console.warn(`\x1b[33m⚠ Overwriting previous report for ${name ?? "unnamed"} (same config)\x1b[0m`);
-    }
-    catch { }
+/**
+ * Write a full per-scene YAML report under `.reports/runs/<runId>.yaml`. The
+ * runId is unique per `agent()` execution, so snapshots never clobber and need
+ * no locking. Only written when a run opts in via `--record`.
+ */
+export async function writeSnapshot(content, runId) {
+    const runsDir = join(process.cwd(), ".reports", "runs");
+    await mkdir(runsDir, { recursive: true });
+    const filepath = join(runsDir, `${runId}.yaml`);
     await writeFile(filepath, content, "utf-8");
     return filepath;
 }
+/**
+ * Append one record to the canonical append-only run log
+ * (`.reports/checkpoints.jsonl`). Used on the standalone path (a lone test-file
+ * process). When launched by `agest run`, the PARENT owns this write instead.
+ */
+export async function appendCheckpoint(record) {
+    const reportsDir = join(process.cwd(), ".reports");
+    await mkdir(reportsDir, { recursive: true });
+    await appendFile(join(reportsDir, "checkpoints.jsonl"), JSON.stringify(record) + "\n", "utf-8");
+}
 export async function writeDiffEntry(hash, systemPrompt, tools, model) {
     const diffDir = join(process.cwd(), ".diff");
     await mkdir(diffDir, { recursive: true });

package/dist/reports.d.ts

@@ -1,3 +1,4 @@
+import type { CheckpointRecord } from "./types";
 export interface ParsedSuiteResult {
     name: string;
     successRate: number;
@@ -127,9 +128,36 @@ export declare function findVaryingDimensions(reports: ParsedReport[]): string[]
  * Group reports by the value of a specific dimension.
  */
 export declare function groupByDimension(reports: ParsedReport[], dimension: string): Map<string, ParsedReport[]>;
+/** Wilson interval from a pass count over a trial count. */
+export declare function wilsonInterval(passes: number, total: number): {
+    low: number;
+    high: number;
+};
 /**
  * Wilson score interval lower bound at 95% confidence.
  * Gives a conservative success rate estimate that accounts for sample size.
  */
 export declare function wilsonLowerBound(successRate: number, totalCases: number): number;
+/** Walk for `.reports/checkpoints.jsonl` files, depth-limited like findReports. */
+export declare function findCheckpointFiles(dir: string, depth?: number): Promise<string[]>;
+/**
+ * Read every checkpoint record from the log(s) under `cwd`. Malformed lines
+ * (e.g. a crash mid-append) are skipped defensively rather than failing the
+ * whole read.
+ */
+export declare function readCheckpoints(cwd: string): Promise<CheckpointRecord[]>;
+/**
+ * Adapt a checkpoint record to the ParsedReport shape the stats/preview
+ * renderers consume. When the record references a `--record` snapshot, the
+ * per-scene detail (scenes / suites / failed cases) is loaded lazily from it;
+ * otherwise the lightweight (record-only) view is returned.
+ */
+export declare function checkpointToReport(rec: CheckpointRecord): Promise<ParsedReport>;
+/**
+ * Load all runs as ParsedReports: the canonical checkpoint log (primary) plus
+ * any legacy `report-*.yaml` still on disk (backward compat). Snapshots under
+ * `.reports/runs/` are NOT scanned directly — they are reached via a record's
+ * recordPath, so they never double-count.
+ */
+export declare function loadReports(cwd: string): Promise<ParsedReport[]>;
 export declare function formatDuration(ms: number): string;

package/dist/reports.js

@@ -1,6 +1,6 @@
 import { createHash } from "crypto";
 import { readdir, readFile } from "fs/promises";
-import { join } from "path";
+import { join, relative } from "path";
 export function extractField(content, key) {
     const regex = new RegExp(`^    ${key}:\\s*(.+)$`, "m");
     const match = content.match(regex);
@@ -474,20 +474,161 @@ export function groupByDimension(reports, dimension) {
     }
     return groups;
 }
+/**
+ * Wilson score interval bounds at 95% confidence for an observed rate `p` over
+ * `total` trials. The single source of truth for Wilson math — both the
+ * lower-bound helper and the runner's per-scene significance delegate here.
+ */
+function wilsonBounds(p, total) {
+    if (total === 0)
+        return { low: 0, high: 0 };
+    const z = 1.96;
+    const denominator = 1 + (z * z) / total;
+    const centre = p + (z * z) / (2 * total);
+    const spread = z * Math.sqrt((p * (1 - p) + (z * z) / (4 * total)) / total);
+    const clamp = (x) => Math.max(0, Math.min(1, x));
+    return {
+        low: clamp((centre - spread) / denominator),
+        high: clamp((centre + spread) / denominator),
+    };
+}
+/** Wilson interval from a pass count over a trial count. */
+export function wilsonInterval(passes, total) {
+    return wilsonBounds(total === 0 ? 0 : passes / total, total);
+}
 /**
  * Wilson score interval lower bound at 95% confidence.
  * Gives a conservative success rate estimate that accounts for sample size.
  */
 export function wilsonLowerBound(successRate, totalCases) {
-    if (totalCases === 0)
-        return 0;
-    const z = 1.96;
-    const p = successRate;
-    const denominator = 1 + (z * z) / totalCases;
-    const centre = p + (z * z) / (2 * totalCases);
-    const spread = z * Math.sqrt((p * (1 - p) + (z * z) / (4 * totalCases)) / totalCases);
-    const lower = (centre - spread) / denominator;
-    return Math.max(0, Math.min(1, lower));
+    return wilsonBounds(successRate, totalCases).low;
+}
+// ---------------------------------------------------------------------------
+// Checkpoint log (canonical run store: .reports/checkpoints.jsonl)
+// ---------------------------------------------------------------------------
+/** Walk for `.reports/checkpoints.jsonl` files, depth-limited like findReports. */
+export async function findCheckpointFiles(dir, depth = 0) {
+    if (depth > 6)
+        return [];
+    const SKIP = new Set(["node_modules", "dist", ".git", ".pnpm"]);
+    const out = [];
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    for (const entry of entries) {
+        if (SKIP.has(entry.name))
+            continue;
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (entry.name === ".reports") {
+                const files = await readdir(full);
+                if (files.includes("checkpoints.jsonl")) {
+                    out.push(join(full, "checkpoints.jsonl"));
+                }
+            }
+            else if (!entry.name.startsWith(".")) {
+                out.push(...(await findCheckpointFiles(full, depth + 1)));
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Read every checkpoint record from the log(s) under `cwd`. Malformed lines
+ * (e.g. a crash mid-append) are skipped defensively rather than failing the
+ * whole read.
+ */
+export async function readCheckpoints(cwd) {
+    const files = await findCheckpointFiles(cwd);
+    const all = [];
+    for (const f of files) {
+        let content;
+        try {
+            content = await readFile(f, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        for (const line of content.split("\n")) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            try {
+                all.push(JSON.parse(trimmed));
+            }
+            catch {
+                /* skip malformed line */
+            }
+        }
+    }
+    return all;
+}
+/**
+ * Adapt a checkpoint record to the ParsedReport shape the stats/preview
+ * renderers consume. When the record references a `--record` snapshot, the
+ * per-scene detail (scenes / suites / failed cases) is loaded lazily from it;
+ * otherwise the lightweight (record-only) view is returned.
+ */
+export async function checkpointToReport(rec) {
+    let scenes;
+    let suites;
+    let failedCases = [];
+    if (rec.recordPath) {
+        try {
+            const content = await readFile(join(process.cwd(), rec.recordPath), "utf-8");
+            const snap = parseReport(content, rec.recordPath);
+            scenes = snap.scenes;
+            suites = snap.suites;
+            failedCases = snap.failedCases;
+        }
+        catch {
+            /* snapshot missing — fall back to the lightweight view */
+        }
+    }
+    return {
+        name: rec.agentName,
+        systemPromptHash: rec.systemPromptHash,
+        promptHash: rec.dimensions?.prompt,
+        dimensions: rec.dimensions,
+        tools: rec.tools,
+        model: rec.dimensions?.model ?? rec.model ?? "unknown",
+        successRate: rec.successRate,
+        totalCases: rec.totalCases,
+        failedCasesCount: Math.max(0, rec.totalCases - rec.casesPassed),
+        failedCases,
+        duration: rec.durationMs,
+        timestamp: rec.timestamp,
+        averageInputTokensPerCase: rec.avgInputTokensPerCase,
+        averageOutputTokensPerCase: rec.avgOutputTokensPerCase,
+        totalInputTokens: rec.totalInputTokens,
+        totalOutputTokens: rec.totalOutputTokens,
+        totalCostUsd: rec.costUsd ?? undefined,
+        scenes,
+        suites,
+        source: rec.recordPath ?? rec.runId,
+    };
+}
+/**
+ * Load all runs as ParsedReports: the canonical checkpoint log (primary) plus
+ * any legacy `report-*.yaml` still on disk (backward compat). Snapshots under
+ * `.reports/runs/` are NOT scanned directly — they are reached via a record's
+ * recordPath, so they never double-count.
+ */
+export async function loadReports(cwd) {
+    const records = await readCheckpoints(cwd);
+    const fromCheckpoints = await Promise.all(records.map((r) => checkpointToReport(r)));
+    const legacyFiles = await findReports(cwd);
+    const legacy = await Promise.all(legacyFiles.map(async (f) => {
+        const content = await readFile(f, "utf-8");
+        const r = parseReport(content, relative(cwd, f));
+        await ensureDimensions(r);
+        return r;
+    }));
+    return [...fromCheckpoints, ...legacy];
 }
 export function formatDuration(ms) {
     if (ms < 1000)

package/dist/runner.js

@@ -2,6 +2,7 @@ import { collectPendingJudgements } from "./assertions";
 import { callJudge, resolveJudgeExecutor } from "./judge";
 import { resolveValue, resolveText, serializeValue, navigatePath } from "./resolve";
 import { validateAgainstSchema } from "./schema";
+import { wilsonInterval } from "./reports";
 const DEFAULT_SCENE_TIMEOUT = 10_000;
 /**
  * Extract a named field from an agent response for assertion.
@@ -31,23 +32,6 @@ export function extractField(response, field) {
         }
     }
 }
-/**
- * Compute Wilson score interval lower bound.
- * Measures confidence that the true pass rate is above 50% (random chance).
- * z = 1.96 for 95% confidence level.
- */
-function wilsonSignificance(passes, total) {
-    if (total === 0)
-        return 0;
-    const z = 1.96;
-    const p = passes / total;
-    const denominator = 1 + (z * z) / total;
-    const centre = p + (z * z) / (2 * total);
-    const spread = z * Math.sqrt((p * (1 - p) + (z * z) / (4 * total)) / total);
-    const lower = (centre - spread) / denominator;
-    // Return the lower bound clamped to [0, 1]
-    return Math.max(0, Math.min(1, lower));
-}
 async function executeSingleRun(executor, scene, timeoutMs, turns, judgeConfig) {
     // The empty sentinel uses the `text` branch of the union so it is a valid
     // AgentResponse<T> for ANY T (there is no native value yet — the executor
@@ -178,7 +162,7 @@ export async function executeScene(executor, scene, globalTimeout, judgeConfig,
     const passes = runs.filter((r) => r.passed).length;
     const passRate = passes / runs.length;
     const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
-    const statisticalSignificance = wilsonSignificance(passes, runs.length);
+    const statisticalSignificance = wilsonInterval(passes, runs.length).low;
     // Use the last run's response as representative
     const lastRun = runs[runs.length - 1];
     // Overall pass = majority passed (> 50%)

package/dist/stats.js

@@ -1,6 +1,6 @@
-import { readdir, readFile, rm } from "fs/promises";
+import { readdir, writeFile, rm } from "fs/promises";
 import { join, relative } from "path";
-import { parseReport, findReports, loadDiffEntry, computeDiff, formatDuration, ensureDimensions, findVaryingDimensions, groupByDimension, findControlledPairs, diffConfigs, } from "./reports.js";
+import { loadReports, readCheckpoints, loadDiffEntry, computeDiff, formatDuration, findVaryingDimensions, groupByDimension, findControlledPairs, diffConfigs, } from "./reports.js";
 function avg(nums) {
     return nums.length === 0
         ? undefined
@@ -193,28 +193,71 @@ async function purge(cwd) {
 // ---------------------------------------------------------------------------
 // Main
 // ---------------------------------------------------------------------------
+/** Quote a CSV cell per RFC-4180 when it contains a comma, quote, or newline. */
+function csvCell(v) {
+    const s = v == null ? "" : String(v);
+    return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+}
+/**
+ * Flatten the JSONL run log to a CSV projection on demand — the only place CSV
+ * escaping lives. For spreadsheet/eyeball use; the JSONL log stays canonical.
+ */
+async function exportCsv(cwd, outPath) {
+    const records = await readCheckpoints(cwd);
+    if (records.length === 0) {
+        console.log("\n  No checkpoints to export. Run some agent tests first.\n");
+        return;
+    }
+    const header = [
+        "runId", "sweepId", "timestamp", "agentName", "suiteHash", "model", "promptHash",
+        "tools", "judge", "runs", "runsPerScene", "totalCases", "casesPassed", "successRate",
+        "wilsonLow", "wilsonHigh", "durationMs", "costUsd", "totalInputTokens",
+        "totalOutputTokens", "recordPath",
+    ];
+    const rows = records.map((r) => [
+        r.runId, r.sweepId, r.timestamp, r.agentName,
+        r.dimensions?.suiteHash, r.dimensions?.model, r.dimensions?.prompt,
+        Array.isArray(r.tools) ? r.tools.join("|") : r.dimensions?.tools,
+        r.dimensions?.judge, r.dimensions?.runs, r.runsPerScene, r.totalCases,
+        r.casesPassed, r.successRate, r.wilsonLow, r.wilsonHigh, r.durationMs,
+        r.costUsd, r.totalInputTokens, r.totalOutputTokens, r.recordPath,
+    ].map(csvCell).join(","));
+    const csv = [header.join(","), ...rows].join("\n") + "\n";
+    await writeFile(join(cwd, outPath), csv, "utf-8");
+    console.log(`\n  Exported ${records.length} checkpoint${records.length !== 1 ? "s" : ""} to ${outPath}\n`);
+}
 async function main() {
     const args = process.argv.slice(2);
     const agentFlagIdx = args.indexOf("--agent");
     const agentFilter = agentFlagIdx !== -1 ? args[agentFlagIdx + 1] : undefined;
     const modelFlagIdx = args.indexOf("--model");
     const modelFilter = modelFlagIdx !== -1 ? args[modelFlagIdx + 1] : undefined;
+    const suiteFlagIdx = args.indexOf("--suite");
+    const suiteFilter = suiteFlagIdx !== -1 ? args[suiteFlagIdx + 1] : undefined;
     if (args.includes("--purge")) {
         await purge(process.cwd());
         return;
     }
     const cwd = process.cwd();
-    const files = await findReports(cwd);
-    if (files.length === 0) {
+    const csvFlagIdx = args.indexOf("--export-csv");
+    if (csvFlagIdx !== -1) {
+        const next = args[csvFlagIdx + 1];
+        const outPath = next && !next.startsWith("--") ? next : "agest-checkpoints.csv";
+        await exportCsv(cwd, outPath);
+        return;
+    }
+    let reports = await loadReports(cwd);
+    if (reports.length === 0) {
         console.log("\n  No reports found. Run some agent tests first.\n");
         return;
     }
-    let reports = await Promise.all(files.map(async (f) => {
-        const content = await readFile(f, "utf-8");
-        return parseReport(content, relative(cwd, f));
-    }));
-    // Ensure all reports have dimensions (backward compat)
-    await Promise.all(reports.map((r) => ensureDimensions(r)));
+    if (suiteFilter) {
+        reports = reports.filter((r) => r.dimensions?.suiteHash === suiteFilter);
+        if (reports.length === 0) {
+            console.log(`\n  No reports found for suite "${suiteFilter}".\n`);
+            return;
+        }
+    }
     if (agentFilter) {
         reports = reports.filter((r) => r.name?.toLowerCase() === agentFilter.toLowerCase());
         if (reports.length === 0) {

package/dist/types.d.ts

@@ -135,6 +135,13 @@ export interface AgentReport<T = string> {
     timestamp: string;
     duration: number;
     totalCases: number;
+    /** Cases that passed (totalCases - failures). Persisted for statistical honesty. */
+    casesPassed?: number;
+    /** Configured runs per scene (sampling count) — affects the trial basis below. */
+    runsPerScene?: number;
+    /** Wilson score interval (95%) across all trials = Σ scene runs. */
+    wilsonLow?: number;
+    wilsonHigh?: number;
     averageInputTokensPerCase?: number;
     averageOutputTokensPerCase?: number;
     totalInputTokens?: number;
@@ -142,4 +149,34 @@ export interface AgentReport<T = string> {
     totalCostUsd?: number;
     results: SceneResult<T>[];
 }
+/**
+ * One append-only line in `.reports/checkpoints.jsonl` — the canonical run log.
+ * Lightweight (cost + identity + stats), written on every run. Structured fields
+ * (`dimensions`, `tools`) stay native; adding a field later needs no migration.
+ */
+export interface CheckpointRecord {
+    runId: string;
+    sweepId?: string;
+    timestamp: string;
+    agentName?: string;
+    model?: string;
+    systemPromptHash?: string;
+    tools?: string[];
+    /** Config identity map: { suiteHash, model, prompt, tools, judge, runs }. */
+    dimensions: Record<string, string>;
+    runsPerScene?: number;
+    totalCases: number;
+    casesPassed: number;
+    successRate: number;
+    wilsonLow?: number;
+    wilsonHigh?: number;
+    durationMs: number;
+    costUsd?: number | null;
+    totalInputTokens?: number;
+    totalOutputTokens?: number;
+    avgInputTokensPerCase?: number;
+    avgOutputTokensPerCase?: number;
+    /** Relative path to the full YAML snapshot, set only when run with --record. */
+    recordPath?: string;
+}
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.3.3-next.10",
+  "version": "0.3.3-next.12",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",