@workos/quickstudy 0.3.0 → 0.5.0

package/README.md

@@ -95,6 +95,10 @@ quickstudy report <run-id>
 writes NOTES.md, so this toy check passes; against real evals every check
 fails, which is exactly what a negative control should do. Completing the
 run → score → persist loop is what it proves.
+The positive control is the built-in `oracle` agent: give an eval a
+`solution/` reference solution and `quickstudy validate --oracle` checks that
+it scores 1.0 before any paid run (see
+[Writing an eval](docs/writing-an-eval.md#prove-the-scorer-with-a-reference-solution)).
 
 Real measurements come from a benchmark repository built on quickstudy: an
 `evals/` root of real integration tasks (tiny HTTP apps under `local/`) and

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/echo.ts

@@ -13,6 +13,8 @@ import type { AgentAdapter, AttemptContext, AttemptOutcome, TranscriptEvent } fr
 
 export class EchoAdapter implements AgentAdapter {
   readonly name: string;
+  /** Does nothing, so every eval must fail it: a scorer it passes rewards doing nothing. */
+  readonly role = "negative-control" as const;
   readonly identity = {
     provider: "local", model: "echo-v1", reasoning: "none", cliVersion: "built-in",
     surfacePolicies: { docs: "no-network", mcp: "no-network", cli: "no-network" },

package/src/adapters/oracle.ts

@@ -0,0 +1,69 @@
+/**
+ * The oracle adapter: applies the eval's reference solution and does nothing
+ * else.
+ *
+ * A bench is only trustworthy when a reference solution scores 1.0 on every
+ * eval (and a no-op — the echo adapter — scores 0). The oracle declares
+ * `startingState: "solution"`, so the runner's shared workspace prep overlays
+ * `solution/` on `local/` before the adapter runs; both the host and the
+ * container variant are then no-ops. No model runs: turns and cost are true
+ * zeros, token counts are unavailable (null).
+ */
+
+import { cp } from "node:fs/promises";
+import type { AgentAdapter, AttemptContext, AttemptMetrics, AttemptOutcome, ContainerAgentAdapter } from "./types.ts";
+
+const ORACLE_IDENTITY = {
+  provider: "local", model: "oracle-v1", reasoning: "none", cliVersion: "built-in",
+  surfacePolicies: { docs: "no-network", mcp: "no-network", cli: "no-network" },
+  nativeWebControl: "controlled" as const,
+  reproducible: true,
+};
+
+function oracleMetrics(): AttemptMetrics {
+  return { tokensIn: null, tokensOut: null, turns: 0, costUsd: 0 };
+}
+
+/** Host variant: the prepared fixture already carries the solution overlay. */
+export class OracleAdapter implements AgentAdapter {
+  readonly name = "oracle";
+  readonly role = "positive-control" as const;
+  readonly identity = ORACLE_IDENTITY;
+  readonly startingState = "solution" as const;
+
+  async runAttempt(ctx: AttemptContext): Promise<AttemptOutcome> {
+    await cp(ctx.fixtureDir, ctx.workspaceDir, { recursive: true });
+    return {
+      transcript: [{ role: "assistant", content: "oracle: applied the reference solution", at: new Date().toISOString() }],
+      diff: "",
+      metrics: oracleMetrics(),
+    };
+  }
+}
+
+/**
+ * Container variant: the runner `docker cp`s the overlaid source into
+ * /workspace, so the agent command is a no-op. Experiments using it must not
+ * set `runtime.command` (that bypasses adapter resolution).
+ */
+export class OracleContainerAdapter implements ContainerAgentAdapter {
+  readonly kind = "container" as const;
+  readonly name = "oracle";
+  readonly identity = ORACLE_IDENTITY;
+  readonly startingState = "solution" as const;
+  readonly role = "positive-control" as const;
+  readonly requiredHostEnv: readonly string[] = [];
+
+  setupFiles(): Record<string, string> {
+    return {};
+  }
+
+  command(): string[] {
+    return ["true"];
+  }
+
+  /** The no-op command prints nothing; input is ignored. */
+  parseStream(_raw: string): AttemptMetrics {
+    return oracleMetrics();
+  }
+}

package/src/adapters/types.ts

@@ -73,10 +73,33 @@ export interface AttemptOutcome {
   metrics: AttemptMetrics;
 }
 
+/**
+ * Which tree the runner prepares as the attempt's starting workspace:
+ * `"fixture"` (the default) copies the eval's `local/`; `"solution"` copies
+ * `local/` and then overlays the eval's `solution/` reference solution (the
+ * built-in oracle adapter). A plan pairing a `"solution"` adapter with an
+ * eval that has no `solution/` is refused at preflight.
+ */
+export type StartingState = "fixture" | "solution";
+
+/**
+ * What an adapter's attempts are for. "agent" is an arm under measurement; a
+ * control checks the instrument instead: "positive-control" must pass (the
+ * built-in oracle applies each eval's reference solution) and
+ * "negative-control" must fail (the built-in echo adapter does nothing). The
+ * runner records the role of the adapter that actually runs each experiment
+ * (RunConfig.roles); the report never guesses it from a name.
+ */
+export type AdapterRole = "agent" | "positive-control" | "negative-control";
+
 export interface AgentAdapter {
   /** The adapter name experiments select via `agent.adapter` (e.g. "echo"). */
   name: string;
   identity?: AdapterIdentityConfig;
+  /** Starting workspace the runner prepares for this adapter (default "fixture"). */
+  startingState?: StartingState;
+  /** What the adapter's attempts are for (default "agent"). */
+  role?: AdapterRole;
   runAttempt(ctx: AttemptContext): Promise<AttemptOutcome>;
 }
 
@@ -115,6 +138,10 @@ export interface ContainerAgentAdapter {
   /** The adapter name experiments select via `agent.adapter` (e.g. "claude", "codex"). */
   name: string;
   identity?: AdapterIdentityConfig;
+  /** Starting workspace the runner prepares for this adapter (default "fixture"). */
+  startingState?: StartingState;
+  /** What the adapter's attempts are for (default "agent"). */
+  role?: AdapterRole;
   /** Command used during preflight to record the executable's exact version. */
   versionCommand?: readonly string[];
   /**

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.3.0", // x-release-please-version
+  version: "0.5.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/cli.ts

@@ -59,6 +59,7 @@ import { compareGroups } from "./experiments/groups.ts";
 import { buildRunManifest } from "./manifest.ts";
 import { filterEvals, planRun } from "./plan.ts";
 import { executeRunV3 } from "./runner/execute.ts";
+import { formatOracleOutcome, validateOracle } from "./runner/oracle-validate.ts";
 import { collectProviderKeys, InvalidProviderKeyError, MissingProviderKeyError } from "./secrets.ts";
 import { serveSite } from "./serve.ts";
 import { UI_INDEX_HTML_BASE64 } from "./ui-bundle.generated.ts";
@@ -71,6 +72,7 @@ Usage:
   quickstudy validate                     Validate the evals and experiments roots
     [--evals-root <dir>]                  Evals root (default: evals)
     [--experiments-root <dir>]            Experiments root (default: experiments)
+    [--oracle]                            Score each eval's solution/ reference solution (expects 1.0)
   quickstudy run [--eval <id>...]         Execute evals against experiments
     [--experiment <id>...]                Omit either selector to include every discovered item
     [--evals-root <dir>]                  Evals root (default: evals)
@@ -104,7 +106,7 @@ Usage:
 `;
 
 const COMMAND_HELP: Record<string, string> = {
-  validate: `Usage: quickstudy validate [--evals-root <dir>] [--experiments-root <dir>]\n\nDiscover and validate the evals root (frontmatter, scorer default exports)\nand the experiments root (defineExperiment shapes, comparison groups).\nDefaults: evals, experiments.`,
+  validate: `Usage: quickstudy validate [--evals-root <dir>] [--experiments-root <dir>] [--oracle]\n\nDiscover and validate the evals root (frontmatter, scorer default exports)\nand the experiments root (defineExperiment shapes, comparison groups).\nDefaults: evals, experiments.\n\n--oracle scores each eval's solution/ reference solution on the host and\nprints 1.0, deferred (the scorer needs a runtime capability), FAILED or\nMISSING per eval; exits 1 on any FAILED or MISSING.`,
   run: `Usage: quickstudy run [--eval <id>...] [--experiment <id>...] [options]\n\nExecute the selected eval × experiment pairs. Omit --eval to select every\ndiscovered eval; omit --experiment to select every discovered experiment.\n\nOptions:\n  --eval <id>... --experiment <id>...\n  --evals-root <dir> --experiments-root <dir>   (defaults: evals, experiments)\n  --suite <s>... --framework <f>...\n  --trials <n> --egress-proxy\n  --concurrency <n> --seed <value>             (default concurrency: 1)\n  --resume <run-id> --retry-unfinished        (explicit error/timeout retries)\n  --provision-ms <n> --agent-ms <n> --score-ms <n>\n  --export-ms <n> --cleanup-ms <n>\n  --db <path> --results <dir>`,
   report: `Usage: quickstudy report <run-id> [options]\n\nBuild report.json into the results dir and print the text report\n(pass \`--latest\` to pick the most recent run; \`--json\` prints the JSON).\n\nOptions:\n  --latest --json --strict\n  --strict exits non-zero for incomplete runs or any attempt error\n  --pricing <file>   Estimate missing Codex token costs using runtime pricing JSON\n  --db <path> --results <dir>`,
   diagnose: `Usage: quickstudy diagnose <run-id> [options]\n\nSend each failing (eval × experiment) pair's harness-selected evidence to a\nstructured model call and write diagnosis.json beside report.json\n(pass \`--latest\` to pick the most recent run). Findings are hypotheses with\nevidence links — never verdicts. Requires ANTHROPIC_API_KEY; override the\nanalyst model with QUICKSTUDY_ANALYST_MODEL.\n\nOptions:\n  --latest\n  --db <path> --results <dir>`,
@@ -126,7 +128,7 @@ function fail(message: string): never {
 }
 
 async function commandValidate(args: string[]): Promise<void> {
-  let values: { "evals-root": string; "experiments-root": string };
+  let values: { "evals-root": string; "experiments-root": string; oracle: boolean };
   let positionals: string[];
   try {
     ({ values, positionals } = parseArgs({
@@ -135,6 +137,7 @@ async function commandValidate(args: string[]): Promise<void> {
       options: {
         "evals-root": { type: "string", default: "evals" },
         "experiments-root": { type: "string", default: "experiments" },
+        oracle: { type: "boolean", default: false },
       },
     }));
   } catch (err) {
@@ -157,6 +160,10 @@ async function commandValidate(args: string[]): Promise<void> {
         `("${experimentsRoot}") exists\n\n${USAGE}`,
     );
   }
+  if (values.oracle && !haveEvals) {
+    fail(`validate --oracle: evals root "${evalsRoot}" does not exist`);
+  }
+  let oracleFailure: string | undefined;
   if (haveEvals) {
     const evals = discoverEvals(evalsRoot);
     // A missing/non-function default export is a validation error HERE, not
@@ -170,9 +177,24 @@ async function commandValidate(args: string[]): Promise<void> {
         `    - ${meta.id} (suite: ${meta.suite}` +
           `${meta.product !== undefined ? `; product: ${meta.product}` : ""}` +
           `${meta.framework !== undefined ? `; framework: ${meta.framework}` : ""}` +
-          `${loaded.localDir !== null ? "; local/" : ""})`,
+          `${loaded.localDir !== null ? "; local/" : ""}` +
+          `${loaded.solutionDir !== null ? "; solution/" : ""})`,
       );
     }
+    if (values.oracle) {
+      console.log(`  oracle:`);
+      let failed = 0;
+      let missing = 0;
+      for (const loaded of evals) {
+        const outcome = await validateOracle(loaded);
+        if (outcome.kind === "failed") failed++;
+        if (outcome.kind === "missing") missing++;
+        console.log(`    - ${formatOracleOutcome(loaded.metadata.id, outcome)}`);
+      }
+      if (failed + missing > 0) {
+        oracleFailure = `validate --oracle: ${failed} eval(s) FAILED, ${missing} eval(s) MISSING a solution/`;
+      }
+    }
   }
   if (haveExperiments) {
     const experiments = await discoverExperiments(experimentsRoot);
@@ -200,6 +222,7 @@ async function commandValidate(args: string[]): Promise<void> {
       }
     }
   }
+  if (oracleFailure !== undefined) fail(oracleFailure);
 }
 
 function parsePositiveInt(value: string, flag: string): number {

package/src/diagnose/excerpt.ts

@@ -29,8 +29,9 @@ export const BUNDLE_DIFF_CAP = 3_000;
 
 /**
  * Heuristic error scan. Matches benign prose too ("failed" in discussion) —
- * accepted: the window stays deterministic and human-verifiable, and the
- * tail fallback bounds the damage.
+ * accepted: the window stays deterministic and human-verifiable, and
+ * anchoring on the last marker cluster keeps an early false positive from
+ * pinning the window to the transcript's head.
  */
 export const ERROR_MARKER = /"is_error"\s*:\s*true|error:|failed|unauthorized|timed out|not found|exited [1-9]/i;
 
@@ -42,8 +43,11 @@ export interface ArtifactExcerpt {
 }
 
 /**
- * Deterministically select the excerpt window: context around the span of
- * error markers (capped), or the tail when nothing matches.
+ * Deterministically select the excerpt window: context around the last
+ * cluster of error markers (walking back from the final marker while the
+ * window stays within {@link EXCERPT_MAX_LINES}), or the tail when nothing
+ * matches. Earlier markers that do not fit are dropped; the window always
+ * ends at the last marker plus context.
  */
 export function excerptTranscript(content: string): ArtifactExcerpt {
   const lines = content.split("\n");
@@ -63,11 +67,18 @@ export function excerptTranscript(content: string): ArtifactExcerpt {
     start = Math.max(0, totalLines - EXCERPT_TAIL_LINES);
     end = totalLines - 1;
   } else {
-    start = Math.max(0, (matches[0] as number) - EXCERPT_CONTEXT_LINES);
-    end = Math.min(totalLines - 1, (matches.at(-1) as number) + EXCERPT_CONTEXT_LINES);
-    if (end - start + 1 > EXCERPT_MAX_LINES) {
-      end = start + EXCERPT_MAX_LINES - 1;
+    // Anchor on the last marker cluster: walk back from the final marker while
+    // the span plus context still fits the window. Early markers (often
+    // system-prompt prose) no longer pin the window to the transcript's head.
+    const last = matches.at(-1) as number;
+    let first = last;
+    for (let i = matches.length - 2; i >= 0; i -= 1) {
+      const candidate = matches[i] as number;
+      if (last - candidate + 2 * EXCERPT_CONTEXT_LINES + 1 > EXCERPT_MAX_LINES) break;
+      first = candidate;
     }
+    start = Math.max(0, first - EXCERPT_CONTEXT_LINES);
+    end = Math.min(totalLines - 1, last + EXCERPT_CONTEXT_LINES);
   }
 
   return {

package/src/diagnose/select.ts

@@ -100,7 +100,7 @@ export function selectDiagnosablePairs(attempts: AttemptRecord[]): PairSelection
       experimentId: report.experiment_id,
       tallies: report,
       flaky: report.flaky,
-      failedChecks: report.checks.filter((check) => check.passed < check.total).map((check) => check.name),
+      failedChecks: report.checks.filter((check) => check.graded !== false && check.passed < check.total).map((check) => check.name),
       bundle,
     });
   }

package/src/diagnose/types.ts

@@ -20,8 +20,11 @@ export class DiagnoseError extends Error {
 
 export const DIAGNOSIS_SCHEMA_VERSION = 1;
 
-/** Bumped whenever the system/user prompt text changes shape. */
-export const DIAGNOSIS_PROMPT_VERSION = 1;
+/**
+ * Bumped whenever the system/user prompt text changes shape. v2: excerpt
+ * windows anchor on the last marker cluster.
+ */
+export const DIAGNOSIS_PROMPT_VERSION = 2;
 
 /** Fixed self-labeling carried inside every diagnosis.json. */
 export const DIAGNOSIS_DISCLAIMER =

package/src/evals/discovery.ts

@@ -1,6 +1,7 @@
 /**
  * Eval discovery: scan each `<root>/<dir>/PROMPT.md`, pair it with its EVAL.ts, and
- * detect the optional `local/` starting-state directory. Discovery is dumb on
+ * detect the optional `local/` starting-state and `solution/` reference-solution
+ * directories. Discovery is dumb on
  * purpose — one visible expansion (a `frameworks:` frontmatter list yields
  * one eval per framework, id `<dirname>-<framework>`), no placeholders, no
  * binding — and loud: the first error aborts the scan.
@@ -53,8 +54,18 @@ export function discoverEvals(root: string): LoadedEval[] {
     if (frameworks === undefined) {
       const localDir = join(dir, "local");
       const hasLocal = existsSync(localDir) && statSync(localDir).isDirectory();
+      const solutionDir = join(dir, "solution");
+      const hasSolution = existsSync(solutionDir) && statSync(solutionDir).isDirectory();
       collected.push({
-        loaded: { metadata, dir, promptPath, promptBody, scorerPath, localDir: hasLocal ? localDir : null },
+        loaded: {
+          metadata,
+          dir,
+          promptPath,
+          promptBody,
+          scorerPath,
+          localDir: hasLocal ? localDir : null,
+          solutionDir: hasSolution ? solutionDir : null,
+        },
       });
       continue;
     }
@@ -85,6 +96,11 @@ export function discoverEvals(root: string): LoadedEval[] {
         );
       }
     }
+    // Reference solutions always live under the eval's own solution/ (no
+    // shared-path form). When solution/ exists it must cover every listed
+    // framework; when absent every variant has no solution.
+    const solutionBase = join(dir, "solution");
+    const hasSolutionBase = existsSync(solutionBase) && statSync(solutionBase).isDirectory();
     // No starting state at all (no local/ and no `local:`) is valid — every
     // variant starts from an empty workspace, like a tools-only eval.
     for (const framework of frameworks) {
@@ -101,6 +117,19 @@ export function discoverEvals(root: string): LoadedEval[] {
         }
         localDir = frameworkDir;
       }
+      let solutionDir: string | null = null;
+      if (hasSolutionBase) {
+        const frameworkSolution = join(solutionBase, framework);
+        if (!existsSync(frameworkSolution) || !statSync(frameworkSolution).isDirectory()) {
+          throw new EvalLoadError(
+            promptPath,
+            1,
+            `eval "${metadata.id}" has solution/ but no solution for framework "${framework}" ` +
+              `(expected ${frameworkSolution})`,
+          );
+        }
+        solutionDir = frameworkSolution;
+      }
       collected.push({
         loaded: {
           metadata: { ...metadata, id: `${metadata.id}-${framework}`, framework },
@@ -109,6 +138,7 @@ export function discoverEvals(root: string): LoadedEval[] {
           promptBody,
           scorerPath,
           localDir,
+          solutionDir,
         },
         framework,
       });

package/src/evals/result.ts

@@ -3,7 +3,7 @@ export function validEvalResult(value: unknown): value is EvalResult {
   if (!value || typeof value !== "object") return false;
   const result = value as EvalResult;
   if (typeof result.passed !== "boolean" || !Array.isArray(result.checks)) return false;
-  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean")) return false;
+  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean"))) return false;
   if (new Set(result.checks.map((c) => c.name)).size !== result.checks.length) return false;
   return result.judgments === undefined || (Array.isArray(result.judgments) && result.judgments.every((j) => j && typeof j === "object" && j.provenance === "semantic-judge" &&
     typeof j.provider === "string" && typeof j.model === "string" && [j.tokensIn, j.tokensOut, j.costUsd].every((v) => v === null || (typeof v === "number" && Number.isFinite(v) && v >= 0))));

package/src/evals/types.ts

@@ -32,6 +32,13 @@ export interface LoadedEval {
   scorerPath: string;
   /** Absolute path of the optional `local/` starting-state directory, or null. */
   localDir: string | null;
+  /**
+   * Absolute path of the optional `solution/` reference solution (or
+   * `solution/<framework>/` for a framework-expanded eval), or null. Overlaid
+   * on the starting state only for adapters with `startingState: "solution"`
+   * (the built-in oracle) and by `validate --oracle`; agents never see it.
+   */
+  solutionDir: string | null;
 }
 
 /** A pointer at evidence backing a check verdict (workspace-relative path). */
@@ -44,6 +51,11 @@ export interface EvidenceReference {
 export interface CheckResult {
   name: string;
   passed: boolean;
+  /**
+   * false = recorded but not graded: the report tallies it and derives SDK
+   * recall from it, and the scorer must leave it out of `passed`. Absent = true.
+   */
+  graded?: boolean;
   notes?: string;
   evidence?: EvidenceReference[];
 }

package/src/export-types.ts

@@ -89,6 +89,13 @@ export interface DataAttempt {
   finished_at: string | null;
   /** Present when status = "error" (and for recorded teardown notes). */
   error: string | null;
+  /** "timeout" when the agent hit its wall-clock limit and the tree it left was graded; null otherwise. */
+  limit: "timeout" | null;
+  /** Which stage failed when status = "error"; null for completed/incomplete rows and for rows written before this field existed. */
+  error_kind: "environment" | "agent" | "scorer" | "harness" | null;
+  /** Agent phase only (excludes provisioning, container start, export, scoring); null = not recorded. */
+  agent_started_at: string | null;
+  agent_finished_at: string | null;
   /** Relative to the run's data dir (e.g. `attempts/<id>.transcript.txt`); null = artifact missing. */
   transcript: string | null;
   diff: string | null;

package/src/export.ts

@@ -33,7 +33,7 @@ import {
 } from "./export-types.ts";
 import { trialCoverage } from "./completeness.ts";
 import { hashString } from "./hash.ts";
-import { buildReport, renderReportText, type ReportJson } from "./report/report.ts";
+import { buildReport, REPORT_SCHEMA_VERSION, renderReportText, type ReportJson } from "./report/report.ts";
 import { parsePricing, type Pricing } from "./report/costs.ts";
 import { activeAttempts, type AttemptRecord, type ResultsStore, type RunRecord } from "./store/db.ts";
 
@@ -162,6 +162,10 @@ function toDataAttempt(
     started_at: iso(attempt.startedAt),
     finished_at: iso(attempt.finishedAt),
     error: attempt.error,
+    limit: attempt.limitHit ?? null,
+    error_kind: attempt.errorKind ?? null,
+    agent_started_at: iso(attempt.agentStartedAt ?? null),
+    agent_finished_at: iso(attempt.agentFinishedAt ?? null),
     transcript,
     diff,
   };
@@ -184,11 +188,18 @@ function resolveReport(
   const path = join(resultsDir, runId, "report.json");
   if (existsSync(path)) {
     try {
-      // Export is an artifact copier, not report validation. Preserve the
-      // stored JSON byte-for-byte in meaning, including forward-compatible
-      // fields a newer report producer may add.
+      // Export is an artifact copier, not report validation. A current-schema
+      // report is preserved byte-for-byte in meaning; any other schema is
+      // rebuilt from the store so the bundled UI reads the shape it knows.
       const stored = JSON.parse(readFileSync(path, "utf8")) as ReportJson;
       const run = store.getRun(runId)!;
+      // A report written under another schema is rebuilt so every schema bump
+      // reaches the exported site; a stored pricing snapshot is reused.
+      if (stored.schema_version !== REPORT_SCHEMA_VERSION) {
+        log(`stored report.json for ${runId} is schema ${stored.schema_version ?? "unversioned"} — rebuilding at ${REPORT_SCHEMA_VERSION}`);
+        return buildReport({ store, runId, resultsDir, generatedAt: generatedAt ?? iso(run.finishedAt) ?? iso(run.startedAt)!,
+          ...(stored.cost_estimate ? { pricing: parsePricing(stored.cost_estimate.pricing) } : {}) });
+      }
       // Historical reports retain their methodology. New reports are reused
       // only when the entire source record still matches (including retries).
       if (run.manifest.manifestVersion === "v3" && stored.cost_estimate === undefined) return stored;

package/src/manifest.ts

@@ -57,6 +57,11 @@ export interface EvalIdentity {
   metadata: EvalMetadata;
   components: EvalIdentityComponents;
   hash: string;
+  /**
+   * Tree hash of `solution/` (the reference solution) when present; null
+   * otherwise. Recorded, never hashed into `hash` — agents never see it.
+   */
+  solution: string | null;
 }
 
 /** Per-experiment identity components — phase 2's comparison-group diff reads these. */
@@ -167,6 +172,7 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
       metadata: loaded.metadata,
       components,
       hash: contentHash(components),
+      solution: loaded.solutionDir === null ? null : hashTree(loaded.solutionDir).sha256,
     };
   }
 

package/src/report/aggregate.ts

@@ -0,0 +1,136 @@
+/**
+ * Per-experiment leaderboard aggregate — one row per experiment, derived from
+ * the same attempts and (banded) pairs as the rest of the report.
+ *
+ * Every number is restricted to benchmark-suite attempts, so regression and
+ * other suites never move the headline row. Telemetry follows the report's
+ * rule: sums and means cover only attempts that reported a value, and are
+ * null (never 0) when none did.
+ */
+import type { AttemptRecord } from "../store/db.ts";
+import type { RunCostEstimate } from "./costs.ts";
+import type { ReportExperiment, ReportPair } from "./report.ts";
+
+export interface ExperimentAggregate {
+  /** Benchmark-suite attempts of this experiment (every status). */
+  attempts: number;
+  completed: number;
+  passed: number;
+  failed: number;
+  incomplete: number;
+  errors: number;
+  /** Attempts whose agent phase hit the timeout limit (graded timeouts are also completed). */
+  timeouts: number;
+  /** passed / (passed + failed) over benchmark-suite pairs; null when nothing scored. */
+  pass_rate_all: number | null;
+  /** Same, restricted to evals whose band is "discriminating". */
+  pass_rate_discriminating: number | null;
+  /** Discriminating benchmark evals where this experiment scored at least one attempt. */
+  discriminating_evals: number;
+  /** Numerator and denominator of pass_rate_discriminating. */
+  discriminating_passed: number;
+  discriminating_scored: number;
+  /** Median of (agent_finished_at - agent_started_at) over scored attempts with both stamps; null when none. */
+  median_agent_ms: number | null;
+  agent_time_attempts: number;
+  /** Each mean divides by the attempts that reported that value. */
+  mean_tokens_in: number | null;
+  mean_tokens_out: number | null;
+  token_attempts: number;
+  /** Provider-reported, summed over the attempts that reported it. */
+  cost_usd: number | null;
+  cost_attempts: number;
+  /** Present only when the report was built with --pricing. */
+  estimated_cost_usd?: number | null;
+  estimated_attempts?: number;
+}
+
+const sum = (values: number[]): number => values.reduce((total, v) => total + v, 0);
+const mean = (values: number[]): number | null => (values.length > 0 ? sum(values) / values.length : null);
+
+/** Median; an even count takes the mean of the two middle values. null when empty. */
+export function median(values: number[]): number | null {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2;
+}
+
+/**
+ * Aggregate one experiment. `pairs` must already carry their bands (the way
+ * `buildReport` assigns them); `costEstimate` is the report's --pricing layer,
+ * whose per-attempt entries are keyed by attempt id only.
+ */
+export function collectExperimentAggregate(
+  attempts: AttemptRecord[],
+  pairs: ReportPair[],
+  experimentId: string,
+  costEstimate?: RunCostEstimate,
+): ExperimentAggregate {
+  const mine = attempts.filter((a) => a.experimentId === experimentId && a.suite === "benchmark");
+  const completed = mine.filter((a) => a.status === "completed");
+  const passed = completed.filter((a) => a.result?.passed).length;
+
+  const benchmarkPairs = pairs.filter((p) => p.experiment_id === experimentId && p.suite === "benchmark");
+  const scoredAll = sum(benchmarkPairs.map((p) => p.scored));
+  const passedAll = sum(benchmarkPairs.map((p) => p.passed));
+  const discriminating = benchmarkPairs.filter((p) => p.band === "discriminating" && p.scored > 0);
+  const discScored = sum(discriminating.map((p) => p.scored));
+  const discPassed = sum(discriminating.map((p) => p.passed));
+
+  const durations = completed.flatMap((a) =>
+    typeof a.agentStartedAt === "number" && typeof a.agentFinishedAt === "number" ? [a.agentFinishedAt - a.agentStartedAt] : [],
+  );
+  const tokensIn = mine.map((a) => a.tokensIn).filter((v): v is number => v !== null && v !== undefined);
+  const tokensOut = mine.map((a) => a.tokensOut).filter((v): v is number => v !== null && v !== undefined);
+  const costs = mine.map((a) => a.costUsd).filter((v): v is number => v !== null && v !== undefined);
+
+  let estimated: Pick<ExperimentAggregate, "estimated_cost_usd" | "estimated_attempts"> = {};
+  if (costEstimate !== undefined) {
+    const ids = new Set(mine.map((a) => a.id));
+    const values = costEstimate.attempts
+      .filter((e) => ids.has(e.attempt_id))
+      .flatMap((e) => (e.cost_usd === null ? [] : [e.cost_usd]));
+    estimated = { estimated_cost_usd: values.length > 0 ? sum(values) : null, estimated_attempts: values.length };
+  }
+
+  return {
+    attempts: mine.length,
+    completed: completed.length,
+    passed,
+    failed: completed.length - passed,
+    incomplete: mine.filter((a) => a.status === "incomplete").length,
+    errors: mine.filter((a) => a.status === "error").length,
+    timeouts: mine.filter((a) => a.status === "completed" && a.limitHit === "timeout").length,
+    pass_rate_all: scoredAll > 0 ? passedAll / scoredAll : null,
+    pass_rate_discriminating: discScored > 0 ? discPassed / discScored : null,
+    discriminating_evals: new Set(discriminating.map((p) => p.eval_id)).size,
+    discriminating_passed: discPassed,
+    discriminating_scored: discScored,
+    median_agent_ms: median(durations),
+    agent_time_attempts: durations.length,
+    mean_tokens_in: mean(tokensIn),
+    mean_tokens_out: mean(tokensOut),
+    token_attempts: Math.max(tokensIn.length, tokensOut.length),
+    cost_usd: costs.length > 0 ? sum(costs) : null,
+    cost_attempts: costs.length,
+    ...estimated,
+  };
+}
+
+/**
+ * Leaderboard order: agent arms before controls (the oracle would otherwise
+ * lead every board), then discriminating pass rate desc (nulls last), then the
+ * all-eval pass rate desc (nulls last), then experiment id — a total order,
+ * so every consumer renders the same rows in the same sequence.
+ */
+export function compareLeaderboard(a: ReportExperiment, b: ReportExperiment): number {
+  const rate = (value: number | null | undefined): number => (value === null || value === undefined ? -1 : value);
+  const control = (experiment: ReportExperiment): number => ((experiment.role ?? "agent") === "agent" ? 0 : 1);
+  return (
+    control(a) - control(b) ||
+    rate(b.aggregate?.pass_rate_discriminating) - rate(a.aggregate?.pass_rate_discriminating) ||
+    rate(b.aggregate?.pass_rate_all) - rate(a.aggregate?.pass_rate_all) ||
+    a.experiment_id.localeCompare(b.experiment_id)
+  );
+}