@workos/quickstudy 0.2.1 → 0.4.0

package/README.md

@@ -95,6 +95,10 @@ quickstudy report <run-id>
 writes NOTES.md, so this toy check passes; against real evals every check
 fails, which is exactly what a negative control should do. Completing the
 run → score → persist loop is what it proves.
+The positive control is the built-in `oracle` agent: give an eval a
+`solution/` reference solution and `quickstudy validate --oracle` checks that
+it scores 1.0 before any paid run (see
+[Writing an eval](docs/writing-an-eval.md#prove-the-scorer-with-a-reference-solution)).
 
 Real measurements come from a benchmark repository built on quickstudy: an
 `evals/` root of real integration tasks (tiny HTTP apps under `local/`) and
@@ -231,6 +235,27 @@ quickstudy export <run-id> --out ./site
 npx gh-pages -d site        # or push ./site to any static host
 ```
 
+### Embedding the viewers
+
+A host that renders exported bundles in its own UI can use this UI's transcript and
+diff viewers directly instead of copying them:
+
+```tsx
+import { DiffViewer, TranscriptViewer } from "@workos/quickstudy/ui/viewers";
+import "@workos/quickstudy/ui/viewers.css";
+
+// Fetch data/runs/<run>/attempts/<id>.transcript.txt yourself (the bundle lazy-loads it
+// the same way), then render it; `lines` highlights and scrolls to a cited range.
+<div className="my-theme"><TranscriptViewer text={text} lines={[21, 60]} /></div>
+```
+
+The components import React only (an optional peer dependency). Every rule in
+`viewers.css` is scoped under the components' root class `.quickstudy-viewer`, so importing
+it restyles nothing else in the host. It reads the theme through custom properties the host sets on a wrapping element: `--surface`, `--bg`,
+`--line`, `--muted`, `--accent`, `--green`, `--amber`, `--amber-bg`, `--green-bg`,
+`--red-bg`, `--gray-bg` and `--mono`. Turn parsing, roles, collapsing and the `L<n>`
+anchors are exactly what this UI shows, so evidence links carry over unchanged.
+
 ## Development
 
 Run the CLI from source as `bun src/cli.ts <cmd>` (what `quickstudy` is,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.2.1",
+  "version": "0.4.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",
@@ -9,7 +9,9 @@
   },
   "exports": {
     ".": "./src/index.ts",
-    "./package.json": "./package.json"
+    "./package.json": "./package.json",
+    "./ui/viewers": "./ui/src/viewers.ts",
+    "./ui/viewers.css": "./ui/src/components/viewers.css"
   },
   "files": [
     "src",
@@ -17,7 +19,11 @@
     "images",
     "examples",
     "README.md",
-    "LICENSE"
+    "LICENSE",
+    "ui/src/viewers.ts",
+    "ui/src/components/TranscriptViewer.tsx",
+    "ui/src/components/DiffViewer.tsx",
+    "ui/src/components/viewers.css"
   ],
   "engines": {
     "bun": ">=1.1"
@@ -49,5 +55,13 @@
     "react-dom": "^19.3.0",
     "typescript": "^7.0.2",
     "vite": "^8.3.0"
+  },
+  "peerDependencies": {
+    "react": ">=18"
+  },
+  "peerDependenciesMeta": {
+    "react": {
+      "optional": true
+    }
   }
 }

package/src/adapters/oracle.ts

@@ -0,0 +1,67 @@
+/**
+ * The oracle adapter: applies the eval's reference solution and does nothing
+ * else.
+ *
+ * A bench is only trustworthy when a reference solution scores 1.0 on every
+ * eval (and a no-op — the echo adapter — scores 0). The oracle declares
+ * `startingState: "solution"`, so the runner's shared workspace prep overlays
+ * `solution/` on `local/` before the adapter runs; both the host and the
+ * container variant are then no-ops. No model runs: turns and cost are true
+ * zeros, token counts are unavailable (null).
+ */
+
+import { cp } from "node:fs/promises";
+import type { AgentAdapter, AttemptContext, AttemptMetrics, AttemptOutcome, ContainerAgentAdapter } from "./types.ts";
+
+const ORACLE_IDENTITY = {
+  provider: "local", model: "oracle-v1", reasoning: "none", cliVersion: "built-in",
+  surfacePolicies: { docs: "no-network", mcp: "no-network", cli: "no-network" },
+  nativeWebControl: "controlled" as const,
+  reproducible: true,
+};
+
+function oracleMetrics(): AttemptMetrics {
+  return { tokensIn: null, tokensOut: null, turns: 0, costUsd: 0 };
+}
+
+/** Host variant: the prepared fixture already carries the solution overlay. */
+export class OracleAdapter implements AgentAdapter {
+  readonly name = "oracle";
+  readonly identity = ORACLE_IDENTITY;
+  readonly startingState = "solution" as const;
+
+  async runAttempt(ctx: AttemptContext): Promise<AttemptOutcome> {
+    await cp(ctx.fixtureDir, ctx.workspaceDir, { recursive: true });
+    return {
+      transcript: [{ role: "assistant", content: "oracle: applied the reference solution", at: new Date().toISOString() }],
+      diff: "",
+      metrics: oracleMetrics(),
+    };
+  }
+}
+
+/**
+ * Container variant: the runner `docker cp`s the overlaid source into
+ * /workspace, so the agent command is a no-op. Experiments using it must not
+ * set `runtime.command` (that bypasses adapter resolution).
+ */
+export class OracleContainerAdapter implements ContainerAgentAdapter {
+  readonly kind = "container" as const;
+  readonly name = "oracle";
+  readonly identity = ORACLE_IDENTITY;
+  readonly startingState = "solution" as const;
+  readonly requiredHostEnv: readonly string[] = [];
+
+  setupFiles(): Record<string, string> {
+    return {};
+  }
+
+  command(): string[] {
+    return ["true"];
+  }
+
+  /** The no-op command prints nothing; input is ignored. */
+  parseStream(_raw: string): AttemptMetrics {
+    return oracleMetrics();
+  }
+}

package/src/adapters/types.ts

@@ -73,10 +73,21 @@ export interface AttemptOutcome {
   metrics: AttemptMetrics;
 }
 
+/**
+ * Which tree the runner prepares as the attempt's starting workspace:
+ * `"fixture"` (the default) copies the eval's `local/`; `"solution"` copies
+ * `local/` and then overlays the eval's `solution/` reference solution (the
+ * built-in oracle adapter). A plan pairing a `"solution"` adapter with an
+ * eval that has no `solution/` is refused at preflight.
+ */
+export type StartingState = "fixture" | "solution";
+
 export interface AgentAdapter {
   /** The adapter name experiments select via `agent.adapter` (e.g. "echo"). */
   name: string;
   identity?: AdapterIdentityConfig;
+  /** Starting workspace the runner prepares for this adapter (default "fixture"). */
+  startingState?: StartingState;
   runAttempt(ctx: AttemptContext): Promise<AttemptOutcome>;
 }
 
@@ -115,6 +126,8 @@ export interface ContainerAgentAdapter {
   /** The adapter name experiments select via `agent.adapter` (e.g. "claude", "codex"). */
   name: string;
   identity?: AdapterIdentityConfig;
+  /** Starting workspace the runner prepares for this adapter (default "fixture"). */
+  startingState?: StartingState;
   /** Command used during preflight to record the executable's exact version. */
   versionCommand?: readonly string[];
   /**

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.2.1", // x-release-please-version
+  version: "0.4.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/cli.ts

@@ -59,6 +59,7 @@ import { compareGroups } from "./experiments/groups.ts";
 import { buildRunManifest } from "./manifest.ts";
 import { filterEvals, planRun } from "./plan.ts";
 import { executeRunV3 } from "./runner/execute.ts";
+import { formatOracleOutcome, validateOracle } from "./runner/oracle-validate.ts";
 import { collectProviderKeys, InvalidProviderKeyError, MissingProviderKeyError } from "./secrets.ts";
 import { serveSite } from "./serve.ts";
 import { UI_INDEX_HTML_BASE64 } from "./ui-bundle.generated.ts";
@@ -71,6 +72,7 @@ Usage:
   quickstudy validate                     Validate the evals and experiments roots
     [--evals-root <dir>]                  Evals root (default: evals)
     [--experiments-root <dir>]            Experiments root (default: experiments)
+    [--oracle]                            Score each eval's solution/ reference solution (expects 1.0)
   quickstudy run [--eval <id>...]         Execute evals against experiments
     [--experiment <id>...]                Omit either selector to include every discovered item
     [--evals-root <dir>]                  Evals root (default: evals)
@@ -104,7 +106,7 @@ Usage:
 `;
 
 const COMMAND_HELP: Record<string, string> = {
-  validate: `Usage: quickstudy validate [--evals-root <dir>] [--experiments-root <dir>]\n\nDiscover and validate the evals root (frontmatter, scorer default exports)\nand the experiments root (defineExperiment shapes, comparison groups).\nDefaults: evals, experiments.`,
+  validate: `Usage: quickstudy validate [--evals-root <dir>] [--experiments-root <dir>] [--oracle]\n\nDiscover and validate the evals root (frontmatter, scorer default exports)\nand the experiments root (defineExperiment shapes, comparison groups).\nDefaults: evals, experiments.\n\n--oracle scores each eval's solution/ reference solution on the host and\nprints 1.0, deferred (the scorer needs a runtime capability), FAILED or\nMISSING per eval; exits 1 on any FAILED or MISSING.`,
   run: `Usage: quickstudy run [--eval <id>...] [--experiment <id>...] [options]\n\nExecute the selected eval × experiment pairs. Omit --eval to select every\ndiscovered eval; omit --experiment to select every discovered experiment.\n\nOptions:\n  --eval <id>... --experiment <id>...\n  --evals-root <dir> --experiments-root <dir>   (defaults: evals, experiments)\n  --suite <s>... --framework <f>...\n  --trials <n> --egress-proxy\n  --concurrency <n> --seed <value>             (default concurrency: 1)\n  --resume <run-id> --retry-unfinished        (explicit error/timeout retries)\n  --provision-ms <n> --agent-ms <n> --score-ms <n>\n  --export-ms <n> --cleanup-ms <n>\n  --db <path> --results <dir>`,
   report: `Usage: quickstudy report <run-id> [options]\n\nBuild report.json into the results dir and print the text report\n(pass \`--latest\` to pick the most recent run; \`--json\` prints the JSON).\n\nOptions:\n  --latest --json --strict\n  --strict exits non-zero for incomplete runs or any attempt error\n  --pricing <file>   Estimate missing Codex token costs using runtime pricing JSON\n  --db <path> --results <dir>`,
   diagnose: `Usage: quickstudy diagnose <run-id> [options]\n\nSend each failing (eval × experiment) pair's harness-selected evidence to a\nstructured model call and write diagnosis.json beside report.json\n(pass \`--latest\` to pick the most recent run). Findings are hypotheses with\nevidence links — never verdicts. Requires ANTHROPIC_API_KEY; override the\nanalyst model with QUICKSTUDY_ANALYST_MODEL.\n\nOptions:\n  --latest\n  --db <path> --results <dir>`,
@@ -126,7 +128,7 @@ function fail(message: string): never {
 }
 
 async function commandValidate(args: string[]): Promise<void> {
-  let values: { "evals-root": string; "experiments-root": string };
+  let values: { "evals-root": string; "experiments-root": string; oracle: boolean };
   let positionals: string[];
   try {
     ({ values, positionals } = parseArgs({
@@ -135,6 +137,7 @@ async function commandValidate(args: string[]): Promise<void> {
       options: {
         "evals-root": { type: "string", default: "evals" },
         "experiments-root": { type: "string", default: "experiments" },
+        oracle: { type: "boolean", default: false },
       },
     }));
   } catch (err) {
@@ -157,6 +160,10 @@ async function commandValidate(args: string[]): Promise<void> {
         `("${experimentsRoot}") exists\n\n${USAGE}`,
     );
   }
+  if (values.oracle && !haveEvals) {
+    fail(`validate --oracle: evals root "${evalsRoot}" does not exist`);
+  }
+  let oracleFailure: string | undefined;
   if (haveEvals) {
     const evals = discoverEvals(evalsRoot);
     // A missing/non-function default export is a validation error HERE, not
@@ -170,9 +177,24 @@ async function commandValidate(args: string[]): Promise<void> {
         `    - ${meta.id} (suite: ${meta.suite}` +
           `${meta.product !== undefined ? `; product: ${meta.product}` : ""}` +
           `${meta.framework !== undefined ? `; framework: ${meta.framework}` : ""}` +
-          `${loaded.localDir !== null ? "; local/" : ""})`,
+          `${loaded.localDir !== null ? "; local/" : ""}` +
+          `${loaded.solutionDir !== null ? "; solution/" : ""})`,
       );
     }
+    if (values.oracle) {
+      console.log(`  oracle:`);
+      let failed = 0;
+      let missing = 0;
+      for (const loaded of evals) {
+        const outcome = await validateOracle(loaded);
+        if (outcome.kind === "failed") failed++;
+        if (outcome.kind === "missing") missing++;
+        console.log(`    - ${formatOracleOutcome(loaded.metadata.id, outcome)}`);
+      }
+      if (failed + missing > 0) {
+        oracleFailure = `validate --oracle: ${failed} eval(s) FAILED, ${missing} eval(s) MISSING a solution/`;
+      }
+    }
   }
   if (haveExperiments) {
     const experiments = await discoverExperiments(experimentsRoot);
@@ -200,6 +222,7 @@ async function commandValidate(args: string[]): Promise<void> {
       }
     }
   }
+  if (oracleFailure !== undefined) fail(oracleFailure);
 }
 
 function parsePositiveInt(value: string, flag: string): number {

package/src/diagnose/excerpt.ts

@@ -29,8 +29,9 @@ export const BUNDLE_DIFF_CAP = 3_000;
 
 /**
  * Heuristic error scan. Matches benign prose too ("failed" in discussion) —
- * accepted: the window stays deterministic and human-verifiable, and the
- * tail fallback bounds the damage.
+ * accepted: the window stays deterministic and human-verifiable, and
+ * anchoring on the last marker cluster keeps an early false positive from
+ * pinning the window to the transcript's head.
  */
 export const ERROR_MARKER = /"is_error"\s*:\s*true|error:|failed|unauthorized|timed out|not found|exited [1-9]/i;
 
@@ -42,8 +43,11 @@ export interface ArtifactExcerpt {
 }
 
 /**
- * Deterministically select the excerpt window: context around the span of
- * error markers (capped), or the tail when nothing matches.
+ * Deterministically select the excerpt window: context around the last
+ * cluster of error markers (walking back from the final marker while the
+ * window stays within {@link EXCERPT_MAX_LINES}), or the tail when nothing
+ * matches. Earlier markers that do not fit are dropped; the window always
+ * ends at the last marker plus context.
  */
 export function excerptTranscript(content: string): ArtifactExcerpt {
   const lines = content.split("\n");
@@ -63,11 +67,18 @@ export function excerptTranscript(content: string): ArtifactExcerpt {
     start = Math.max(0, totalLines - EXCERPT_TAIL_LINES);
     end = totalLines - 1;
   } else {
-    start = Math.max(0, (matches[0] as number) - EXCERPT_CONTEXT_LINES);
-    end = Math.min(totalLines - 1, (matches.at(-1) as number) + EXCERPT_CONTEXT_LINES);
-    if (end - start + 1 > EXCERPT_MAX_LINES) {
-      end = start + EXCERPT_MAX_LINES - 1;
+    // Anchor on the last marker cluster: walk back from the final marker while
+    // the span plus context still fits the window. Early markers (often
+    // system-prompt prose) no longer pin the window to the transcript's head.
+    const last = matches.at(-1) as number;
+    let first = last;
+    for (let i = matches.length - 2; i >= 0; i -= 1) {
+      const candidate = matches[i] as number;
+      if (last - candidate + 2 * EXCERPT_CONTEXT_LINES + 1 > EXCERPT_MAX_LINES) break;
+      first = candidate;
     }
+    start = Math.max(0, first - EXCERPT_CONTEXT_LINES);
+    end = Math.min(totalLines - 1, last + EXCERPT_CONTEXT_LINES);
   }
 
   return {

package/src/diagnose/select.ts

@@ -100,7 +100,7 @@ export function selectDiagnosablePairs(attempts: AttemptRecord[]): PairSelection
       experimentId: report.experiment_id,
       tallies: report,
       flaky: report.flaky,
-      failedChecks: report.checks.filter((check) => check.passed < check.total).map((check) => check.name),
+      failedChecks: report.checks.filter((check) => check.graded !== false && check.passed < check.total).map((check) => check.name),
       bundle,
     });
   }

package/src/diagnose/types.ts

@@ -20,8 +20,11 @@ export class DiagnoseError extends Error {
 
 export const DIAGNOSIS_SCHEMA_VERSION = 1;
 
-/** Bumped whenever the system/user prompt text changes shape. */
-export const DIAGNOSIS_PROMPT_VERSION = 1;
+/**
+ * Bumped whenever the system/user prompt text changes shape. v2: excerpt
+ * windows anchor on the last marker cluster.
+ */
+export const DIAGNOSIS_PROMPT_VERSION = 2;
 
 /** Fixed self-labeling carried inside every diagnosis.json. */
 export const DIAGNOSIS_DISCLAIMER =

package/src/evals/discovery.ts

@@ -1,6 +1,7 @@
 /**
  * Eval discovery: scan each `<root>/<dir>/PROMPT.md`, pair it with its EVAL.ts, and
- * detect the optional `local/` starting-state directory. Discovery is dumb on
+ * detect the optional `local/` starting-state and `solution/` reference-solution
+ * directories. Discovery is dumb on
  * purpose — one visible expansion (a `frameworks:` frontmatter list yields
  * one eval per framework, id `<dirname>-<framework>`), no placeholders, no
  * binding — and loud: the first error aborts the scan.
@@ -53,8 +54,18 @@ export function discoverEvals(root: string): LoadedEval[] {
     if (frameworks === undefined) {
       const localDir = join(dir, "local");
       const hasLocal = existsSync(localDir) && statSync(localDir).isDirectory();
+      const solutionDir = join(dir, "solution");
+      const hasSolution = existsSync(solutionDir) && statSync(solutionDir).isDirectory();
       collected.push({
-        loaded: { metadata, dir, promptPath, promptBody, scorerPath, localDir: hasLocal ? localDir : null },
+        loaded: {
+          metadata,
+          dir,
+          promptPath,
+          promptBody,
+          scorerPath,
+          localDir: hasLocal ? localDir : null,
+          solutionDir: hasSolution ? solutionDir : null,
+        },
       });
       continue;
     }
@@ -85,6 +96,11 @@ export function discoverEvals(root: string): LoadedEval[] {
         );
       }
     }
+    // Reference solutions always live under the eval's own solution/ (no
+    // shared-path form). When solution/ exists it must cover every listed
+    // framework; when absent every variant has no solution.
+    const solutionBase = join(dir, "solution");
+    const hasSolutionBase = existsSync(solutionBase) && statSync(solutionBase).isDirectory();
     // No starting state at all (no local/ and no `local:`) is valid — every
     // variant starts from an empty workspace, like a tools-only eval.
     for (const framework of frameworks) {
@@ -101,6 +117,19 @@ export function discoverEvals(root: string): LoadedEval[] {
         }
         localDir = frameworkDir;
       }
+      let solutionDir: string | null = null;
+      if (hasSolutionBase) {
+        const frameworkSolution = join(solutionBase, framework);
+        if (!existsSync(frameworkSolution) || !statSync(frameworkSolution).isDirectory()) {
+          throw new EvalLoadError(
+            promptPath,
+            1,
+            `eval "${metadata.id}" has solution/ but no solution for framework "${framework}" ` +
+              `(expected ${frameworkSolution})`,
+          );
+        }
+        solutionDir = frameworkSolution;
+      }
       collected.push({
         loaded: {
           metadata: { ...metadata, id: `${metadata.id}-${framework}`, framework },
@@ -109,6 +138,7 @@ export function discoverEvals(root: string): LoadedEval[] {
           promptBody,
           scorerPath,
           localDir,
+          solutionDir,
         },
         framework,
       });

package/src/evals/result.ts

@@ -3,7 +3,7 @@ export function validEvalResult(value: unknown): value is EvalResult {
   if (!value || typeof value !== "object") return false;
   const result = value as EvalResult;
   if (typeof result.passed !== "boolean" || !Array.isArray(result.checks)) return false;
-  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean")) return false;
+  if (result.checks.some((c) => !c || typeof c.name !== "string" || c.name === "" || typeof c.passed !== "boolean" || (c.graded !== undefined && typeof c.graded !== "boolean"))) return false;
   if (new Set(result.checks.map((c) => c.name)).size !== result.checks.length) return false;
   return result.judgments === undefined || (Array.isArray(result.judgments) && result.judgments.every((j) => j && typeof j === "object" && j.provenance === "semantic-judge" &&
     typeof j.provider === "string" && typeof j.model === "string" && [j.tokensIn, j.tokensOut, j.costUsd].every((v) => v === null || (typeof v === "number" && Number.isFinite(v) && v >= 0))));

package/src/evals/types.ts

@@ -32,6 +32,13 @@ export interface LoadedEval {
   scorerPath: string;
   /** Absolute path of the optional `local/` starting-state directory, or null. */
   localDir: string | null;
+  /**
+   * Absolute path of the optional `solution/` reference solution (or
+   * `solution/<framework>/` for a framework-expanded eval), or null. Overlaid
+   * on the starting state only for adapters with `startingState: "solution"`
+   * (the built-in oracle) and by `validate --oracle`; agents never see it.
+   */
+  solutionDir: string | null;
 }
 
 /** A pointer at evidence backing a check verdict (workspace-relative path). */
@@ -44,6 +51,11 @@ export interface EvidenceReference {
 export interface CheckResult {
   name: string;
   passed: boolean;
+  /**
+   * false = recorded but not graded: the report tallies it and derives SDK
+   * recall from it, and the scorer must leave it out of `passed`. Absent = true.
+   */
+  graded?: boolean;
   notes?: string;
   evidence?: EvidenceReference[];
 }

package/src/export-types.ts

@@ -89,6 +89,13 @@ export interface DataAttempt {
   finished_at: string | null;
   /** Present when status = "error" (and for recorded teardown notes). */
   error: string | null;
+  /** "timeout" when the agent hit its wall-clock limit and the tree it left was graded; null otherwise. */
+  limit: "timeout" | null;
+  /** Which stage failed when status = "error"; null for completed/incomplete rows and for rows written before this field existed. */
+  error_kind: "environment" | "agent" | "scorer" | "harness" | null;
+  /** Agent phase only (excludes provisioning, container start, export, scoring); null = not recorded. */
+  agent_started_at: string | null;
+  agent_finished_at: string | null;
   /** Relative to the run's data dir (e.g. `attempts/<id>.transcript.txt`); null = artifact missing. */
   transcript: string | null;
   diff: string | null;

package/src/export.ts

@@ -33,7 +33,7 @@ import {
 } from "./export-types.ts";
 import { trialCoverage } from "./completeness.ts";
 import { hashString } from "./hash.ts";
-import { buildReport, renderReportText, type ReportJson } from "./report/report.ts";
+import { buildReport, REPORT_SCHEMA_VERSION, renderReportText, type ReportJson } from "./report/report.ts";
 import { parsePricing, type Pricing } from "./report/costs.ts";
 import { activeAttempts, type AttemptRecord, type ResultsStore, type RunRecord } from "./store/db.ts";
 
@@ -162,6 +162,10 @@ function toDataAttempt(
     started_at: iso(attempt.startedAt),
     finished_at: iso(attempt.finishedAt),
     error: attempt.error,
+    limit: attempt.limitHit ?? null,
+    error_kind: attempt.errorKind ?? null,
+    agent_started_at: iso(attempt.agentStartedAt ?? null),
+    agent_finished_at: iso(attempt.agentFinishedAt ?? null),
     transcript,
     diff,
   };
@@ -184,11 +188,18 @@ function resolveReport(
   const path = join(resultsDir, runId, "report.json");
   if (existsSync(path)) {
     try {
-      // Export is an artifact copier, not report validation. Preserve the
-      // stored JSON byte-for-byte in meaning, including forward-compatible
-      // fields a newer report producer may add.
+      // Export is an artifact copier, not report validation. A current-schema
+      // report is preserved byte-for-byte in meaning; any other schema is
+      // rebuilt from the store so the bundled UI reads the shape it knows.
       const stored = JSON.parse(readFileSync(path, "utf8")) as ReportJson;
       const run = store.getRun(runId)!;
+      // A report written under another schema is rebuilt so every schema bump
+      // reaches the exported site; a stored pricing snapshot is reused.
+      if (stored.schema_version !== REPORT_SCHEMA_VERSION) {
+        log(`stored report.json for ${runId} is schema ${stored.schema_version ?? "unversioned"} — rebuilding at ${REPORT_SCHEMA_VERSION}`);
+        return buildReport({ store, runId, resultsDir, generatedAt: generatedAt ?? iso(run.finishedAt) ?? iso(run.startedAt)!,
+          ...(stored.cost_estimate ? { pricing: parsePricing(stored.cost_estimate.pricing) } : {}) });
+      }
       // Historical reports retain their methodology. New reports are reused
       // only when the entire source record still matches (including retries).
       if (run.manifest.manifestVersion === "v3" && stored.cost_estimate === undefined) return stored;

package/src/manifest.ts

@@ -57,6 +57,11 @@ export interface EvalIdentity {
   metadata: EvalMetadata;
   components: EvalIdentityComponents;
   hash: string;
+  /**
+   * Tree hash of `solution/` (the reference solution) when present; null
+   * otherwise. Recorded, never hashed into `hash` — agents never see it.
+   */
+  solution: string | null;
 }
 
 /** Per-experiment identity components — phase 2's comparison-group diff reads these. */
@@ -167,6 +172,7 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
       metadata: loaded.metadata,
       components,
       hash: contentHash(components),
+      solution: loaded.solutionDir === null ? null : hashTree(loaded.solutionDir).sha256,
     };
   }