@hackerrank/astra-cli 0.1.2 → 0.1.4

package/README.md

@@ -115,24 +115,69 @@ astra \
   --repeat 3 \
   --tar                       # write a bench/ tarball locally
 
-# push results to S3 (tars bench/ then `aws s3 cp` to the URI)
+# push results to S3 (tars bench/ and uploads it + report.html to the URI)
 astra -m m1,m2 -p tasks/dummy-slugify --repeat 3 \
   --push s3://hackerrank-astra-bench-results-dev/runs/2025-09-02/
 ```
 
-Leaderboard columns: `solved` (pass rate), average `steps` / `tokens`, and
+Leaderboard columns: completed generation rate, average `steps` / `tokens`, and
 `cost`. Cost is per-run summed; a model whose route reports no cost and has no
 price entry shows **`n/a`** (never `$0`), and estimated costs (native routes) are
 marked `~`. `--push` requires an `s3://` URI and the `aws` CLI on PATH; if the
 upload fails the local tarball is kept.
 
+Next to the tarball, `--push` also uploads the self-contained `report.html`
+dashboard with `Content-Type: text/html` and prints a **7-day presigned
+"view report" link** at the end of the run. The bucket is private (no public
+read), so that presigned URL is the only way to open the dashboard — e.g.
+clickable straight from a Jenkins console log.
+
+### Project-configured bench runs
+
+For repeatable benchmarks, keep the benchmark definition with the task and run
+it through the existing bench persona:
+
+```bash
+astra bench --project ./task
+```
+
+`./task/.astra/project.toml` selects the public workspace, instruction,
+models, limits, output root, and optional bench template/extensions. CLI flags
+still override bench limits and output paths. Astra copies only the configured
+workspace; container lifecycle, services, secrets, verification, and scoring
+remain harness responsibilities. Run artifacts include `project.json` with the
+project and input hashes. `completed` means the agent emitted its completion
+sentinel—not that the candidate is correct; `model_error` and `request_error`
+distinguish generation/provider failures from invalid invocation/configuration.
+
+```toml
+[project]
+id = "peoplecore-scim"
+version = 1
+instruction = "instruction.md"
+workspace = "public/codebase"
+
+[bench]
+models = ["gpt-5.6-sol"]
+reasoning = ["medium"]
+repeat = 3
+steps = 60
+wall_seconds = 3600
+command_timeout_seconds = 120
+
+[templates]
+bench = ".astra/templates/bench.md"
+[extensions]
+directories = [".astra/extensions"]
+```
+
 ### HTML report
 
 Every matrix run (and every single `-p`/`-t`/`-f` bench run) rebuilds
 `bench/summary.json` and a self-contained `bench/report.html` dashboard —
 no server, no build step, no external JS: open it straight from `file://` or
 from inside the `bench-*.tgz` tarball. It has a leaderboard, a model × task
-pass-rate heatmap, cost/token charts, per-step trend lines, and an
+completion-rate heatmap, cost/token charts, per-step trend lines, and an
 expandable command timeline for every run.
 
 Regenerate it on demand (e.g. after manually editing/pruning `bench/`) with:
@@ -144,7 +189,7 @@ astra --report --bench-root ./other-bench
 
 `summary.json` follows the `astra-bench-1` schema: `kpis` (totals), a tidy
 `leaderboard[]` (one row per model×reasoning), `matrix[]` (model×reasoning×task
-pass@k), `runs[]` (every attempt, with a compact per-step `timeline`), and
+completion telemetry), `runs[]` (every attempt, with a compact per-step `timeline`), and
 `step_series[]` (token/cost distributions by step, for the trend charts). Each
 run folder also gets a standalone `run.json` for drill-down without loading
 the full `trajectory.json`.
@@ -172,10 +217,13 @@ credits — that's a quota issue, not a bug.
 -t, --task <text>       Task text -> autonomous mode
 -f, --task-file <path>  Read task text from a file -> autonomous mode
 -p, --path <dir>        Task directory -> isolated bench workspace
+    --project <path>     Load .astra/project.toml for an explicit bench run
     --repeat <n>        Attempts per (model,reasoning) in a matrix (default: 1)
     --bench-root <dir>  Root folder for bench runs (default: ./bench)
     --tar               After a matrix, write a bench/ tarball locally
-    --push <s3-uri>     After a matrix, tar bench/ and `aws s3 cp` to the URI
+    --push <s3-uri>     After a matrix, tar bench/ and `aws s3 cp` it (plus
+                        report.html) to the URI; prints presigned
+                        download/view links
     --report            Rebuild bench/summary.json + bench/report.html from
                         whatever runs already exist on disk, then exit
 -C, --cwd <path>        Working directory for commands (default: cwd)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {
@@ -16,7 +16,8 @@
   },
   "scripts": {
     "start": "node src/cli.js",
-    "astra": "node src/cli.js"
+    "astra": "node src/cli.js",
+    "test": "node --test"
   },
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
@@ -26,5 +27,8 @@
     "type": "git",
     "url": "git+ssh://git@github.com/interviewstreet/hrbench.git"
   },
-  "license": "MIT"
+  "license": "MIT",
+  "dependencies": {
+    "@iarna/toml": "2.2.5"
+  }
 }

package/src/bench.js

@@ -30,7 +30,8 @@ export const METRIC_COLUMNS = [
   "model",
   "reasoning",
   "exit_status",
-  "resolved",
+  "status",
+  "completed",
   "steps",
   "n_commands",
   "n_failed_commands",
@@ -104,6 +105,7 @@ export function allocateRun({ model, reasoning, root } = {}) {
  */
 export function seedWorkspace(workspace, { taskPath, taskFile, taskText } = {}) {
   let text = taskText ?? "";
+  const hasExplicitTask = taskText !== undefined && taskText !== null;
   let copyFrom = null;
   let instructionName = null;
 
@@ -114,7 +116,7 @@ export function seedWorkspace(workspace, { taskPath, taskFile, taskText } = {})
       copyFrom = resolved;
       // Prefer a conventional instruction file for the task text.
       for (const name of ["instruction.md", "INSTRUCTION.md", "task.md", "README.md"]) {
-        if (fs.existsSync(path.join(resolved, name))) {
+        if (!hasExplicitTask && fs.existsSync(path.join(resolved, name))) {
           text = fs.readFileSync(path.join(resolved, name), "utf8");
           break;
         }
@@ -122,13 +124,13 @@ export function seedWorkspace(workspace, { taskPath, taskFile, taskText } = {})
     } else {
       copyFrom = path.dirname(resolved);
       instructionName = path.basename(resolved);
-      text = fs.readFileSync(resolved, "utf8");
+      if (!hasExplicitTask) text = fs.readFileSync(resolved, "utf8");
     }
   } else if (taskFile) {
     const resolved = path.resolve(taskFile);
     copyFrom = path.dirname(resolved);
     instructionName = path.basename(resolved);
-    text = fs.readFileSync(resolved, "utf8");
+    if (!hasExplicitTask) text = fs.readFileSync(resolved, "utf8");
   }
 
   if (copyFrom) copyDir(copyFrom, workspace);
@@ -162,13 +164,14 @@ function copyDir(from, to) {
 /** Build a metrics row object from a finished agent + run info. */
 export function collectMetrics({ agent, model, reasoning }) {
   const info = agent.serialize().info;
-  const resolved = info.exit_status === "Submitted" ? 1 : 0;
+  const status = generationStatus(info.exit_status);
   return {
     timestamp: new Date().toISOString(),
     model: model.model,
     reasoning: reasoning || "none",
     exit_status: info.exit_status || "",
-    resolved,
+    status,
+    completed: status === "completed" ? 1 : 0,
     steps: info.n_steps ?? 0,
     n_commands: agent.nCommands ?? 0,
     n_failed_commands: agent.nFailedCommands ?? 0,
@@ -188,6 +191,11 @@ export function collectMetrics({ agent, model, reasoning }) {
   };
 }
 
+/** Classify autonomous generation without making a correctness claim. */
+export function generationStatus(exitStatus) {
+  return exitStatus === "Submitted" ? "completed" : "model_error";
+}
+
 /** Serialize a metrics row to a single CSV line (columns in METRIC_COLUMNS order). */
 export function metricsRow(m) {
   return METRIC_COLUMNS.map((c) => csvCell(m[c])).join(",");
@@ -219,6 +227,14 @@ export function writeMetrics({ runDir, root, metrics }) {
   return { runMetrics: path.join(runDir, "metrics.csv"), index: indexPath };
 }
 
+/** Persist the project-owned inputs that defined a bench attempt. */
+export function writeProjectMetadata(runDir, metadata) {
+  if (!metadata) return null;
+  const file = path.join(runDir, "project.json");
+  fs.writeFileSync(file, JSON.stringify({ schema: "astra-project-1", ...metadata }, null, 2) + "\n");
+  return file;
+}
+
 function csvCell(v) {
   const s = v == null ? "" : String(v);
   return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
@@ -264,6 +280,7 @@ export async function runCell({
   task,
   taskPath,
   taskFile,
+  projectMetadata,
   root,
   steps = 40,
   wall = 0,
@@ -278,6 +295,7 @@ export async function runCell({
   const seeded = seedWorkspace(alloc.workspace, { taskPath, taskFile, taskText: task });
   const taskText = seeded.taskText || task || "";
   fs.writeFileSync(path.join(alloc.dir, "task.md"), taskText);
+  writeProjectMetadata(alloc.dir, projectMetadata);
 
   const gw = new GatewayModel({
     model,
@@ -340,7 +358,7 @@ export async function runMatrix({
 
 /**
  * Aggregate metrics rows into a per-(model,reasoning) leaderboard.
- * `resolved` counts as a solve. Cost is summed only over runs that reported a
+ * `completed` counts a completed generation. Cost is summed only over runs that reported a
  * cost; a group with no cost data is reported as null (rendered "n/a"), never $0.
  */
 export function aggregate(rows) {
@@ -352,7 +370,7 @@ export function aggregate(rows) {
         model: r.model,
         reasoning: r.reasoning || "none",
         runs: 0,
-        solved: 0,
+        completed: 0,
         steps: 0,
         totalTokens: 0,
         costUsd: 0,
@@ -362,7 +380,7 @@ export function aggregate(rows) {
     }
     const g = groups.get(key);
     g.runs++;
-    g.solved += Number(r.resolved) ? 1 : 0;
+    g.completed += Number(r.completed) ? 1 : 0;
     g.steps += Number(r.steps) || 0;
     g.totalTokens += Number(r.total_tokens) || 0;
     if (r.cost_source && r.cost_source !== "") {
@@ -375,8 +393,8 @@ export function aggregate(rows) {
     model: g.model,
     reasoning: g.reasoning,
     runs: g.runs,
-    solved: g.solved,
-    solved_rate: g.runs ? g.solved / g.runs : 0,
+    completed: g.completed,
+    completion_rate: g.runs ? g.completed / g.runs : 0,
     avg_steps: g.runs ? g.steps / g.runs : 0,
     avg_tokens: g.runs ? g.totalTokens / g.runs : 0,
     cost_usd: g.costRuns ? g.costUsd : null,
@@ -455,17 +473,38 @@ function s3ConsoleUrl(s3Uri) {
   return `https://s3.console.aws.amazon.com/s3/object/${bucket}?prefix=${encodeURIComponent(key)}`;
 }
 
+/**
+ * Generate a time-limited presigned URL for an S3 object via the AWS CLI
+ * (`aws s3 presign`). `expiresIn` defaults to 7 days (604800s). Returns null
+ * (rather than throwing) if the AWS CLI is missing or presigning fails, so
+ * a presign failure never blocks the run.
+ */
+export function presignS3Url(s3Uri, expiresIn = 604800) {
+  const res = spawnSync(
+    "aws",
+    ["s3", "presign", s3Uri, "--expires-in", String(expiresIn)],
+    { stdio: ["ignore", "pipe", "pipe"] }
+  );
+  if (res.error && res.error.code === "ENOENT") return null;
+  if (res.status !== 0) return null;
+  return (res.stdout?.toString() || "").trim() || null;
+}
+
 /**
  * Upload a file to S3 via the AWS CLI. Returns { uri, url } where `uri` is the
  * final s3:// destination and `url` is a browsable https console link.
- * Fails gracefully (throws a descriptive error) when the AWS CLI is missing or
- * the copy fails; callers should keep the local tarball on failure.
+ * Optionally sets the object's Content-Type (e.g. "text/html" so presigned
+ * URLs open in a browser instead of downloading). Fails gracefully (throws a
+ * descriptive error) when the AWS CLI is missing or the copy fails; callers
+ * should keep the local tarball on failure.
  */
-export function pushToS3(file, s3Uri) {
+export function pushToS3(file, s3Uri, { contentType } = {}) {
   if (!/^s3:\/\//.test(s3Uri)) throw new Error(`--push destination must be an s3:// URI, got: ${s3Uri}`);
   // If the URI ends with "/", upload into it under the file's basename.
   const dest = s3Uri.endsWith("/") ? s3Uri + path.basename(file) : s3Uri;
-  const res = spawnSync("aws", ["s3", "cp", file, dest], {
+  const cmd = ["s3", "cp", file, dest];
+  if (contentType) cmd.push("--content-type", contentType);
+  const res = spawnSync("aws", cmd, {
     stdio: ["ignore", "pipe", "pipe"],
   });
   if (res.error && res.error.code === "ENOENT") {
@@ -474,5 +513,5 @@ export function pushToS3(file, s3Uri) {
   if (res.status !== 0) {
     throw new Error(`aws s3 cp failed: ${res.stderr?.toString() || "unknown"}`);
   }
-  return { uri: dest, url: s3ConsoleUrl(dest) };
+  return { uri: dest, url: s3ConsoleUrl(dest), presigned: presignS3Url(dest) };
 }

package/src/cli.js

@@ -16,6 +16,7 @@
  *   astra -m <model> -t "<task text>"        bench run from task text
  *   astra -m <model> -f path/to/instruction.md
  *   astra -m <model> -p tasks/dummy-slugify  bench run from a task directory
+ *   astra bench --project ./task         project-configured bench run
  *   astra --resume <session-id>             resume a saved session
  *   astra --sessions                        list saved sessions
  *
@@ -28,13 +29,15 @@
  *                           comma-separated to sweep levels in bench mode).
  *                           Sent to the model and recorded in the bench name.
  *       --repeat <n>        Attempts per (model,reasoning) in a matrix (1)
- *       --push              After a matrix, tar bench/ and upload to
- *                           s3://astra-bench-results/<timestamp>/
+ *       --push              After a matrix, tar bench/ and upload it (plus
+ *                           report.html) to s3://astra-bench-results/<timestamp>/;
+ *                           prints presigned download/view links
  *       --push-uri <s3-uri> Override the S3 push destination
  *       --tar               After a matrix, write a bench/ tarball locally
  *       --report            Rebuild bench/summary.json + bench/report.html
  *                           from whatever runs already exist on disk, then
  *                           exit (no model/API key needed).
+ *       --project <path>    Load .astra/project.toml and run its bench setup
  *   -t, --task <text>       Task text -> bench mode (run to completion)
  *   -f, --task-file <path>  Read task text from a file -> bench mode
  *   -p, --path <dir>        Task directory (copied into an isolated bench
@@ -85,11 +88,13 @@ import {
   deriveTitle,
 } from "./session.js";
 import { runRepl } from "./repl.js";
+import { ProjectConfigError, loadProject } from "./project.js";
 import {
   allocateRun,
   seedWorkspace,
   collectMetrics,
   writeMetrics,
+  writeProjectMetadata,
   runMatrix,
   aggregate,
   packBench,
@@ -108,7 +113,7 @@ function reasoningKwargs(level) {
 }
 
 function parseArgs(argv) {
-  const args = { steps: 40, wall: 0, timeout: 60, quiet: false, "max-output": 16000 };
+  const args = { steps: 40, wall: 0, timeout: 60, quiet: false, "max-output": 16000, _provided: new Set() };
   const alias = {
     "-m": "model", "--model": "model",
     "-t": "task", "--task": "task",
@@ -120,6 +125,7 @@ function parseArgs(argv) {
     "--push-uri": "push-uri",
     "--tar": "tar",
     "--report": "report",
+    "--project": "project",
     "--bench-root": "bench-root",
     "-C": "cwd", "--cwd": "cwd",
     "-o": "output", "--output": "output",
@@ -136,11 +142,17 @@ function parseArgs(argv) {
     "-h": "help", "--help": "help",
   };
   const flags = new Set(["quiet", "yolo", "help", "sessions", "tar", "push", "report"]);
-  for (let i = 2; i < argv.length; i++) {
+  let start = 2;
+  if (argv[start] === "bench") {
+    args.bench = true;
+    start++;
+  }
+  for (let i = start; i < argv.length; i++) {
     const key = alias[argv[i]];
     if (!key) { console.error(`Unknown option: ${argv[i]}`); process.exit(2); }
     if (flags.has(key)) { args[key] = true; continue; }
     args[key] = argv[++i];
+    args._provided.add(key);
   }
   return args;
 }
@@ -182,11 +194,28 @@ async function main() {
     resumeDoc = loadSession(args.resume);
   }
 
-  // A task (-t/-f) or task path (-p) selects bench mode; otherwise agent mode.
+  // A task (-t/-f), task path (-p), or explicit bench project selects bench mode.
   // Bench mode uses the autonomous engine; agent mode uses the interactive one.
   let task = args.task;
   if (args["task-file"]) task = fs.readFileSync(args["task-file"], "utf8");
-  const wantsBench = !!(task || args.path);
+  let project = null;
+  if (args.project) {
+    try {
+      project = loadProject(args.project);
+    } catch (error) {
+      const message = error instanceof ProjectConfigError ? error.message : String(error?.message || error);
+      console.error(`\x1b[31m[astra] request_error: ${message}\x1b[0m`);
+      process.exit(2);
+    }
+    task = project.taskText;
+    args.path = project.workspace;
+    if (!args._provided.has("bench-root")) args["bench-root"] = project.outputRoot;
+    if (!args._provided.has("steps")) args.steps = project.bench.steps;
+    if (!args._provided.has("wall")) args.wall = project.bench.wall;
+    if (!args._provided.has("timeout")) args.timeout = project.bench.timeout;
+    if (!args._provided.has("repeat")) args.repeat = project.bench.repeat;
+  }
+  const wantsBench = !!(task || args.path || args.bench);
   const mode = wantsBench
     ? "autonomous"
     : resumeDoc?.info?.mode === "autonomous"
@@ -235,6 +264,8 @@ async function main() {
   let reasoning = args.reasoning || "";
 
   if (mode === "autonomous") {
+    if (project && !modelId) modelId = project.bench.models.join(",");
+    if (project && !reasoning) reasoning = project.bench.reasoning.join(",");
     if (!modelId) {
       console.error("\x1b[31m[astra] --model is required for bench mode.\x1b[0m\n");
       console.log(HELP);
@@ -253,7 +284,7 @@ async function main() {
     const isMatrix = models.length > 1 || reasonings.length > 1 || repeat > 1;
 
     if (isMatrix && !resumeDoc) {
-      await runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, quiet: !!args.quiet });
+      await runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, project, quiet: !!args.quiet });
       return; // runBenchMatrix exits the process
     }
   } else {
@@ -309,6 +340,7 @@ async function main() {
     task = seeded.taskText || task || "";
     // Persist the task text alongside the run for reproducibility.
     fs.writeFileSync(path.join(benchRun.dir, "task.md"), task || "");
+    writeProjectMetadata(benchRun.dir, project?.metadata);
   }
 
   const cmdCwd = benchRun ? benchRun.workspace : cwd;
@@ -365,6 +397,12 @@ async function main() {
       const metrics = collectMetrics({ agent: benchAgent, model, reasoning });
       const paths = writeMetrics({ runDir: alloc.dir, root: alloc.root, metrics });
       log(`[astra] bench done: exit=${res.exit_status} · metrics → ${paths.runMetrics}`);
+      try {
+        const report = refreshReport(alloc.root);
+        log(`[astra] report → ${report.html}`);
+      } catch (err) {
+        log(`[astra] report generation skipped: ${err.message}`);
+      }
     };
     await runRepl(agent, { model, autoRun: !!args.yolo, fresh: !resumeDoc, reasoning, runBench });
     process.exit(0);
@@ -465,6 +503,30 @@ function archiveAndPush(args, root, links) {
         const pushed = pushToS3(tarball, dest);
         links.push(["s3 uri", pushed.uri]);
         links.push(["s3 url", pushed.url]);
+        if (pushed.presigned) {
+          links.push(["download (presigned, 7d)", pushed.presigned]);
+        }
+        // Also push the self-contained HTML dashboard next to the tarball and
+        // emit a presigned view link: the bucket is private (no public read),
+        // so the presigned URL is the only way to open the report — e.g. by
+        // clicking it straight from the Jenkins console output. Uploaded with
+        // Content-Type: text/html so browsers render it instead of downloading.
+        const reportHtml = path.join(benchRoot(root), "report.html");
+        if (fs.existsSync(reportHtml)) {
+          const htmlDest = dest.endsWith("/")
+            ? dest + "report.html"
+            : dest.slice(0, dest.lastIndexOf("/") + 1) + "report.html";
+          try {
+            const html = pushToS3(reportHtml, htmlDest, { contentType: "text/html" });
+            if (html.presigned) {
+              links.push(["view report (presigned, 7d)", html.presigned]);
+            } else {
+              links.push(["report (s3 url)", html.url]);
+            }
+          } catch (err) {
+            console.error(`\x1b[31m[astra] S3 report push failed: ${err.message}\x1b[0m`);
+          }
+        }
       } catch (err) {
         console.error(`\x1b[31m[astra] S3 push failed: ${err.message}\x1b[0m`);
         console.error(`\x1b[33m[astra] tarball kept locally: ${tarball}\x1b[0m`);
@@ -479,7 +541,7 @@ function archiveAndPush(args, root, links) {
  * Run a multi-model / multi-reasoning / repeated bench matrix, print a
  * leaderboard, optionally tar + upload the bench/ folder, then exit.
  */
-async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, quiet }) {
+async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, project, quiet }) {
   const root = args["bench-root"];
   const total = models.length * reasonings.length * repeat;
   console.error(
@@ -497,6 +559,7 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
     task,
     taskPath: args.path,
     taskFile: args["task-file"],
+    projectMetadata: project?.metadata,
     root,
     steps: Number(args.steps),
     wall: Number(args.wall),
@@ -509,8 +572,8 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
       }
     },
     onDone: (r) => {
-      const mark = r.resolved ? "\x1b[32m✓\x1b[0m" : "\x1b[31m✗\x1b[0m";
-      const status = r.error ? `Error: ${r.error.split("\n")[0]}` : r.exit_status;
+      const mark = r.completed ? "\x1b[32m✓\x1b[0m" : "\x1b[31m✗\x1b[0m";
+      const status = r.error ? `model_error: ${r.error.split("\n")[0]}` : r.status;
       console.error(
         `  ${mark} ${String(r.model).padEnd(20)} ${String(r.reasoning).padEnd(8)} ` +
           `${String(status).padEnd(16)} ${String(r.steps).padStart(3)} steps · ` +
@@ -539,8 +602,8 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
   // Tar + optional S3 push of the whole bench/ folder.
   archiveAndPush(args, root, links);
 
-  const solved = rows.filter((r) => r.resolved).length;
-  console.error(`\n\x1b[1m[astra] matrix done · ${solved}/${rows.length} resolved\x1b[0m`);
+  const completed = rows.filter((r) => r.completed).length;
+  console.error(`\n\x1b[1m[astra] matrix done · ${completed}/${rows.length} completed\x1b[0m`);
 
   // Print the important URLs / paths, aligned, at the very end.
   const w = Math.max(...links.map(([k]) => k.length));
@@ -549,23 +612,23 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
     console.error(`\x1b[1m  ${k.padEnd(w)}\x1b[0m  ${v}`);
   }
 
-  process.exit(solved === rows.length ? 0 : 1);
+  process.exit(completed === rows.length ? 0 : 1);
 }
 
 /** Print the per-(model,reasoning) leaderboard table. */
 function printLeaderboard(board) {
-  board.sort((a, b) => b.solved_rate - a.solved_rate || a.avg_steps - b.avg_steps);
+  board.sort((a, b) => b.completion_rate - a.completion_rate || a.avg_steps - b.avg_steps);
   console.error(
-    `\n\x1b[1m${"model".padEnd(20)} ${"reason".padEnd(8)} ${"solved".padEnd(8)} ` +
+    `\n\x1b[1m${"model".padEnd(20)} ${"reason".padEnd(8)} ${"complete".padEnd(8)} ` +
       `${"steps".padStart(6)} ${"tokens".padStart(9)}  ${"cost".padStart(9)}  source\x1b[0m`
   );
   for (const g of board) {
-    const solved = `${g.solved}/${g.runs}`;
-    const rate = `${Math.round(g.solved_rate * 100)}%`;
+    const completed = `${g.completed}/${g.runs}`;
+    const rate = `${Math.round(g.completion_rate * 100)}%`;
     const cost = g.cost_usd == null ? "n/a" : fmtUsd(g.cost_usd);
     console.error(
       `${String(g.model).padEnd(20)} ${String(g.reasoning).padEnd(8)} ` +
-        `${(solved + " " + rate).padEnd(8)} ${g.avg_steps.toFixed(1).padStart(6)} ` +
+        `${(completed + " " + rate).padEnd(8)} ${g.avg_steps.toFixed(1).padStart(6)} ` +
         `${fmt(Math.round(g.avg_tokens)).padStart(9)}  ${cost.padStart(9)}  ${g.cost_source}`
     );
   }

package/src/project.js

@@ -0,0 +1,177 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import toml from "@iarna/toml";
+
+export class ProjectConfigError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ProjectConfigError";
+  }
+}
+
+const DEFAULT_BENCH = {
+  models: [],
+  reasoning: [""],
+  repeat: 1,
+  steps: 40,
+  wall_seconds: 0,
+  command_timeout_seconds: 60,
+};
+
+function requireObject(value, name) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new ProjectConfigError(`${name} must be a TOML table`);
+  }
+  return value;
+}
+
+function requireString(value, name) {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new ProjectConfigError(`${name} must be a non-empty string`);
+  }
+  return value;
+}
+
+function positiveInteger(value, name, fallback, { allowZero = false } = {}) {
+  if (value === undefined) return fallback;
+  if (!Number.isInteger(value) || value < (allowZero ? 0 : 1)) {
+    throw new ProjectConfigError(`${name} must be ${allowZero ? "a non-negative" : "a positive"} integer`);
+  }
+  return value;
+}
+
+function stringArray(value, name, fallback) {
+  if (value === undefined) return fallback;
+  if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || !item.trim())) {
+    throw new ProjectConfigError(`${name} must be an array of non-empty strings`);
+  }
+  return value;
+}
+
+function resolveInside(root, relative, name) {
+  const candidate = path.resolve(root, requireString(relative, name));
+  if (candidate !== root && !candidate.startsWith(root + path.sep)) {
+    throw new ProjectConfigError(`${name} must stay inside the project directory`);
+  }
+  return candidate;
+}
+
+function readRegularFile(file, name, { optional = false } = {}) {
+  if (!fs.existsSync(file)) {
+    if (optional) return null;
+    throw new ProjectConfigError(`${name} does not exist: ${file}`);
+  }
+  const stat = fs.lstatSync(file);
+  if (!stat.isFile() || stat.isSymbolicLink()) {
+    throw new ProjectConfigError(`${name} must be a regular file: ${file}`);
+  }
+  return fs.readFileSync(file, "utf8");
+}
+
+function sha256(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function extensionFiles(root, directories) {
+  const seen = new Set();
+  const extensions = [];
+  for (const directory of directories) {
+    const resolved = resolveInside(root, directory, "extensions.directories entry");
+    if (!fs.existsSync(resolved)) {
+      throw new ProjectConfigError(`extension directory does not exist: ${resolved}`);
+    }
+    const stat = fs.lstatSync(resolved);
+    if (!stat.isDirectory() || stat.isSymbolicLink()) {
+      throw new ProjectConfigError(`extension directory must be a regular directory: ${resolved}`);
+    }
+    for (const entry of fs.readdirSync(resolved, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      if (!entry.isFile() || entry.isSymbolicLink()) continue;
+      const file = path.join(resolved, entry.name);
+      if (seen.has(file)) throw new ProjectConfigError(`duplicate extension path: ${file}`);
+      seen.add(file);
+      const content = readRegularFile(file, "extension");
+      extensions.push({ path: file, content, sha256: sha256(content) });
+    }
+  }
+  return extensions;
+}
+
+export function loadProject(projectPath) {
+  const root = path.resolve(requireString(projectPath, "project path"));
+  if (!fs.existsSync(root) || !fs.lstatSync(root).isDirectory()) {
+    throw new ProjectConfigError(`project directory does not exist: ${root}`);
+  }
+
+  const configPath = path.join(root, ".astra", "project.toml");
+  const configText = readRegularFile(configPath, "project configuration");
+  let config;
+  try {
+    config = toml.parse(configText);
+  } catch (error) {
+    throw new ProjectConfigError(`invalid project TOML: ${error.message}`);
+  }
+
+  const project = requireObject(config.project, "project");
+  const bench = requireObject(config.bench ?? {}, "bench");
+  const output = requireObject(config.output ?? {}, "output");
+  const templates = requireObject(config.templates ?? {}, "templates");
+  const extensionConfig = requireObject(config.extensions ?? {}, "extensions");
+  const provenance = requireObject(config.provenance ?? {}, "provenance");
+  const id = requireString(project.id, "project.id");
+  const version = positiveInteger(project.version, "project.version", undefined);
+
+  const instructionPath = resolveInside(root, project.instruction, "project.instruction");
+  const workspace = resolveInside(root, project.workspace, "project.workspace");
+  const instruction = readRegularFile(instructionPath, "instruction");
+  if (!instruction.trim()) throw new ProjectConfigError("instruction must not be empty");
+  if (!fs.existsSync(workspace) || !fs.lstatSync(workspace).isDirectory() || fs.lstatSync(workspace).isSymbolicLink()) {
+    throw new ProjectConfigError(`project.workspace must be a regular directory: ${workspace}`);
+  }
+
+  const templatePath = templates.bench === undefined ? null : resolveInside(root, templates.bench, "templates.bench");
+  const template = templatePath === null ? null : readRegularFile(templatePath, "bench template");
+  const extensions = extensionFiles(root, stringArray(extensionConfig.directories, "extensions.directories", []));
+  const taskText = [instruction, template, ...extensions.map((extension) => extension.content)]
+    .filter(Boolean)
+    .join("\n\n");
+
+  const baselineSha256 = provenance.baseline_sha256;
+  if (baselineSha256 !== undefined && (typeof baselineSha256 !== "string" || !/^[a-f0-9]{64}$/.test(baselineSha256))) {
+    throw new ProjectConfigError("provenance.baseline_sha256 must be a lowercase SHA-256 hash");
+  }
+
+  return {
+    root,
+    id,
+    version,
+    instructionPath,
+    instruction,
+    instructionSha256: sha256(instruction),
+    workspace,
+    outputRoot: resolveInside(root, output.root ?? "bench", "output.root"),
+    template: template === null ? null : { path: templatePath, content: template, sha256: sha256(template) },
+    extensions,
+    taskText,
+    provenance: { baselineSha256: baselineSha256 ?? null },
+    metadata: {
+      id,
+      version,
+      instruction_sha256: sha256(instruction),
+      baseline_sha256: baselineSha256 ?? null,
+      template_sha256: template === null ? null : sha256(template),
+      extension_sha256: extensions.map((extension) => ({
+        path: path.relative(root, extension.path),
+        sha256: extension.sha256,
+      })),
+    },
+    bench: {
+      models: stringArray(bench.models, "bench.models", DEFAULT_BENCH.models),
+      reasoning: stringArray(bench.reasoning, "bench.reasoning", DEFAULT_BENCH.reasoning),
+      repeat: positiveInteger(bench.repeat, "bench.repeat", DEFAULT_BENCH.repeat),
+      steps: positiveInteger(bench.steps, "bench.steps", DEFAULT_BENCH.steps),
+      wall: positiveInteger(bench.wall_seconds, "bench.wall_seconds", DEFAULT_BENCH.wall_seconds, { allowZero: true }),
+      timeout: positiveInteger(bench.command_timeout_seconds, "bench.command_timeout_seconds", DEFAULT_BENCH.command_timeout_seconds),
+    },
+  };
+}

package/src/report.html

@@ -4,41 +4,15 @@
 <meta charset="utf-8" />
 <meta name="viewport" content="width=device-width, initial-scale=1" />
 <title>astra bench report</title>
+<link rel="stylesheet" href="https://api.fontshare.com/v2/css?f[]=satoshi@400,500,700&display=swap" />
 <style>
-  :root {
-    --bg: #0f1115;
-    --panel: #171a21;
-    --panel-2: #1d212b;
-    --border: #262b36;
-    --text: #e6e8ee;
-    --muted: #8b93a7;
-    --accent: #5b9dff;
-    --good: #3ecf8e;
-    --bad: #ef5a6f;
-    --warn: #e8b339;
-    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
-  }
-  * { box-sizing: border-box; }
-  body { margin: 0; background: var(--bg); color: var(--text); font-size: 14px; line-height: 1.45; }
-  a { color: var(--accent); }
-  .wrap { max-width: 1180px; margin: 0 auto; padding: 28px 20px 64px; }
-  header.top { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 4px; flex-wrap: wrap; gap: 8px; }
-  header.top h1 { font-size: 20px; margin: 0; font-weight: 650; letter-spacing: .2px; }
-  header.top .meta { color: var(--muted); font-size: 12.5px; }
-  .kpis { display: grid; grid-template-columns: repeat(auto-fit, minmax(130px, 1fr)); gap: 10px; margin: 20px 0 28px; }
-  .kpi { background: var(--panel); border: 1px solid var(--border); border-radius: 10px; padding: 12px 14px; }
-  .kpi .v { font-size: 22px; font-weight: 700; }
-  .kpi .l { color: var(--muted); font-size: 11.5px; text-transform: uppercase; letter-spacing: .06em; margin-top: 2px; }
-  section { margin: 34px 0; }
-  section > h2 { font-size: 15px; margin: 0 0 12px; font-weight: 650; color: var(--text); display: flex; align-items: center; gap: 8px; }
-  section > h2 .sub { color: var(--muted); font-weight: 400; font-size: 12px; }
-  .grid2 { display: grid; grid-template-columns: 1.1fr 1fr; gap: 16px; }
-  .grid3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; }
-  @media (max-width: 900px) { .grid2, .grid3 { grid-template-columns: 1fr; } }
-  .panel { background: var(--panel); border: 1px solid var(--border); border-radius: 10px; padding: 14px 16px; }
-  .panel h3 { margin: 0 0 10px; font-size: 12.5px; color: var(--muted); font-weight: 600; text-transform: uppercase; letter-spacing: .05em; }
+  :root { --ink:#11131a; --muted:#69707d; --wash:#f5f6f8; --line:#e4e7eb; --lime:#a7ff63; --blue:#5285f7; --purple:#9160d8; --green:#39ae76; --orange:#f3a021; --red:#d85668; font-family:Satoshi,"Avenir Next",Arial,sans-serif; }
+  * { box-sizing:border-box; } body { margin:0; background:#fff; color:var(--ink); font-size:14px; line-height:1.45; } .wrap { max-width:1240px; margin:auto; padding:28px 32px 68px; }
+  header.top { display:flex; justify-content:space-between; gap:20px; align-items:flex-start; padding-bottom:28px; border-bottom:1px solid var(--line); } header.top h1 { font-size:32px; letter-spacing:-.045em; line-height:1; margin:8px 0 0; font-weight:700; } .eyebrow { display:inline-block; padding:5px 14px; border-radius:99px; background:var(--lime); font-size:12px; font-weight:700; } header.top .meta { color:var(--muted); font-size:12px; text-align:right; }
+  .kpis { display:grid; grid-template-columns:repeat(4,1fr); margin:24px 0 52px; } .kpi { min-height:96px; padding:8px 18px; border-left:1px solid var(--line); } .kpi:first-child { border:0; padding-left:0; } .kpi .v { font-size:28px; letter-spacing:-.045em; font-weight:700; } .kpi .l { color:var(--muted); font-size:11px; font-weight:700; letter-spacing:.08em; text-transform:uppercase; margin-top:8px; }
+  section { margin:56px 0; } section > h2 { font-size:20px; letter-spacing:-.035em; margin:0 0 22px; } section > h2 .sub { color:var(--muted); font-size:13px; font-weight:400; letter-spacing:0; } .grid2 { display:grid; grid-template-columns:1.1fr 1fr; gap:44px; } .panel { padding:0; } .panel h3 { color:var(--muted); letter-spacing:.07em; text-transform:uppercase; font-size:12px; margin:0 0 16px; }
   table { width: 100%; border-collapse: collapse; font-size: 13px; }
-  th, td { text-align: left; padding: 7px 10px; border-bottom: 1px solid var(--border); white-space: nowrap; }
+  th, td { text-align: left; padding: 10px 8px; border-bottom: 1px solid var(--line); white-space: nowrap; }
   th { color: var(--muted); font-weight: 600; font-size: 11.5px; text-transform: uppercase; letter-spacing: .03em; cursor: pointer; user-select: none; }
   th:hover { color: var(--text); }
   th .arrow { opacity: .5; font-size: 10px; margin-left: 3px; }
@@ -52,7 +26,7 @@
   .muted { color: var(--muted); }
   .bar-row { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
   .bar-row .lbl { width: 150px; flex: 0 0 150px; font-size: 12px; overflow: hidden; text-overflow: ellipsis; }
-  .bar-track { flex: 1; height: 14px; background: var(--panel-2); border-radius: 6px; overflow: hidden; display: flex; }
+  .bar-track { flex: 1; height: 9px; background: var(--wash); border-radius: 99px; overflow: hidden; display: flex; }
   .bar-seg { height: 100%; }
   .bar-val { width: 64px; flex: 0 0 64px; text-align: right; font-size: 12px; font-variant-numeric: tabular-nums; }
   .legend { display: flex; gap: 14px; flex-wrap: wrap; margin-top: 10px; font-size: 11.5px; color: var(--muted); }
@@ -61,7 +35,7 @@
   .heat-cell { display: inline-block; min-width: 44px; padding: 3px 6px; border-radius: 5px; font-size: 12px; }
   details.run-detail { margin: 0; }
   details.run-detail > summary { display: none; }
-  .timeline { margin: 0 0 18px; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
+  .timeline { margin: 0 0 18px; background:var(--wash); padding:0 14px; overflow: hidden; }
   .timeline .step { padding: 10px 14px; border-bottom: 1px solid var(--border); }
   .timeline .step:last-child { border-bottom: none; }
   .timeline .step .head { display: flex; justify-content: space-between; gap: 10px; align-items: baseline; margin-bottom: 4px; }
@@ -71,35 +45,30 @@
   .timeline .step pre { background: var(--panel-2); border-radius: 6px; padding: 8px 10px; margin: 4px 0; font-size: 12px; overflow-x: auto; white-space: pre-wrap; word-break: break-word; }
   .rc0 { color: var(--good); } .rcN { color: var(--bad); }
   .toolbar { display: flex; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; align-items: center; }
-  .toolbar select, .toolbar input { background: var(--panel-2); border: 1px solid var(--border); color: var(--text); border-radius: 6px; padding: 5px 9px; font-size: 12.5px; }
+  .toolbar select, .toolbar input { background: var(--wash); border:0; color:var(--ink); border-radius:4px; padding:7px 9px; font-size:12px; } .model-filter { align-items:end; } .model-filter > span { color:var(--muted); font-size:11px; } .model-menu { position:relative; } .model-menu summary { cursor:pointer; list-style:none; min-width:220px; padding:9px 32px 9px 11px; background:var(--wash); border-radius:5px; font-size:12px; position:relative; } .model-menu summary::-webkit-details-marker { display:none; } .model-menu summary::after { content:"⌄"; position:absolute; right:11px; color:var(--muted); } .model-options { position:absolute; z-index:2; top:calc(100% + 6px); left:0; width:260px; max-height:240px; overflow:auto; padding:6px; background:#fff; border:1px solid var(--line); box-shadow:0 10px 24px rgba(17,19,26,.1); } .model-option { display:flex; gap:9px; align-items:center; padding:8px; font-size:12px; cursor:pointer; } .model-option:hover { background:var(--wash); } .model-option input { accent-color:var(--blue); }
   .toolbar label { color: var(--muted); font-size: 12px; }
   footer { color: var(--muted); font-size: 11.5px; margin-top: 40px; text-align: center; }
   svg text { fill: var(--muted); font-size: 10.5px; }
-  .axis line, .axis path { stroke: var(--border); }
+  .axis line, .axis path { stroke: var(--line); }
   .n-a { color: var(--muted); }
 </style>
 </head>
 <body>
 <div class="wrap">
   <header class="top">
-    <h1>astra bench report</h1>
-    <div class="meta" id="generated"></div>
+    <div><h1>Benchmark report</h1><div class="meta" id="generated"></div></div>
   </header>
 
   <div class="kpis" id="kpis"></div>
 
   <section>
-    <h2>Leaderboard <span class="sub">one row per model · reasoning</span></h2>
-    <div class="grid2">
-      <div class="panel">
-        <h3>Solve rate</h3>
-        <div id="chart-solve"></div>
-      </div>
-      <div class="panel">
-        <h3>Outcomes</h3>
-        <div id="chart-outcomes"></div>
-        <div class="legend" id="legend-outcomes"></div>
-      </div>
+    <h2>Performance <span class="sub">run volume and reliability by model</span></h2>
+    <div class="toolbar model-filter">
+      <span>Models</span>
+      <details class="model-menu">
+        <summary id="model-filter-label">All models</summary>
+        <div id="leaderboard-model-options" class="model-options"></div>
+      </details>
     </div>
     <div class="panel" style="margin-top:16px;">
       <table id="tbl-leaderboard"></table>
@@ -107,22 +76,29 @@
   </section>
 
   <section>
-    <h2>Cost &amp; tokens</h2>
+    <h2>Efficiency <span class="sub">spend and token composition</span></h2>
     <div class="grid2">
       <div class="panel">
-        <h3>Cost vs. solve rate</h3>
+        <h3>Cost vs. average duration</h3>
         <div id="chart-scatter"></div>
       </div>
       <div class="panel">
-        <h3>Token mix (prompt / completion / reasoning / cached)</h3>
-        <div id="chart-tokens"></div>
-        <div class="legend" id="legend-tokens"></div>
+        <h3>Cost vs. average steps</h3>
+        <div id="chart-scatter-steps"></div>
       </div>
     </div>
   </section>
 
+  <section>
+    <h2>Token mix <span class="sub">prompt, completion, reasoning, and cached tokens</span></h2>
+    <div class="panel">
+      <div id="chart-tokens"></div>
+      <div class="legend" id="legend-tokens"></div>
+    </div>
+  </section>
+
   <section id="section-series">
-    <h2>Per-step trend <span class="sub">median tokens &amp; cumulative cost across the trajectory</span></h2>
+    <h2>Trajectory <span class="sub">where context and cost accumulate</span></h2>
     <div class="grid2">
       <div class="panel">
         <h3>Prompt tokens (context) vs. step</h3>
@@ -136,31 +112,6 @@
     <div class="legend" id="legend-series"></div>
   </section>
 
-  <section id="section-matrix">
-    <h2>Model &times; task <span class="sub">pass@k</span></h2>
-    <div class="panel">
-      <table id="tbl-matrix" class="heat-table"></table>
-    </div>
-  </section>
-
-  <section>
-    <h2>Runs <span class="sub">every attempt · click a row to expand the command timeline</span></h2>
-    <div class="toolbar">
-      <label>Model
-        <select id="filter-model"><option value="">all</option></select>
-      </label>
-      <label>Task
-        <select id="filter-task"><option value="">all</option></select>
-      </label>
-      <label>Outcome
-        <select id="filter-outcome"><option value="">all</option></select>
-      </label>
-    </div>
-    <div class="panel">
-      <table id="tbl-runs"></table>
-    </div>
-  </section>
-
   <footer>generated by <span class="mono">astra bench --report</span> · schema <span id="schema"></span></footer>
 </div>
 
@@ -169,8 +120,19 @@
 </script>
 <script>
 (function () {
-  var DATA = window.DATA || { kpis: {}, leaderboard: [], matrix: [], runs: [], step_series: [] };
-  var PALETTE = ["#5b9dff", "#3ecf8e", "#e8b339", "#ef5a6f", "#b57bff", "#33c2c2", "#ff8a5b", "#a3d977"];
+  function loadReportData() {
+    try {
+      var request = new XMLHttpRequest();
+      request.open("GET", "summary.json", false);
+      request.send(null);
+      if ((request.status >= 200 && request.status < 300) || request.status === 0) return JSON.parse(request.responseText);
+    } catch (_) {
+      // Direct file access can block JSON reads; use the embedded snapshot then.
+    }
+    return window.__ASTRA_EMBEDDED_DATA__ || { kpis: {}, leaderboard: [], matrix: [], runs: [], step_series: [] };
+  }
+  var DATA = loadReportData();
+  var PALETTE = ["#5285f7", "#9160d8", "#39ae76", "#f3a021", "#d85668", "#37a8ad"];
   var colorCache = {};
   function colorFor(key) {
     if (!colorCache[key]) {
@@ -209,21 +171,19 @@
   }
 
   // ---------------------------------------------------------------- header
-  document.getElementById("generated").textContent = "generated " + (DATA.generated_at || "");
+  document.getElementById("generated").textContent = DATA.generated_at
+    ? "Generated " + new Date(DATA.generated_at).toLocaleString("en-US", { dateStyle: "medium", timeStyle: "short" })
+    : "";
   document.getElementById("schema").textContent = DATA.schema || "";
 
   // ---------------------------------------------------------------- KPIs
   (function renderKpis() {
     var k = DATA.kpis || {};
     var cards = [
-      ["Runs", fmt(k.runs)],
-      ["Models", fmt(k.models)],
-      ["Tasks", fmt(k.tasks)],
-      ["Solved", fmt(k.solved) + " / " + fmt(k.runs)],
-      ["Solve rate", pct(k.solved_rate)],
-      ["Total tokens", fmt(k.tokens)],
+      ["Models Benchmarked", fmt(k.models)],
+      ["Total duration", fmt(k.elapsed_seconds) + "s"],
       ["Total cost", fmtUsd(k.cost_usd) + (k.cost_source === "estimated" ? "~" : "")],
-      ["Elapsed", fmt(k.elapsed_seconds) + "s"],
+      ["Tokens used", fmt(k.tokens)],
     ];
     var host = document.getElementById("kpis");
     cards.forEach(function (c) {
@@ -266,20 +226,22 @@
     draw();
   }
 
-  // ---------------------------------------------------------------- leaderboard chart: solve rate bars
+  // ---------------------------------------------------------------- leaderboard chart: run volume bars
   (function () {
     var host = document.getElementById("chart-solve");
+    if (!host) return;
+    var maxRuns = Math.max(1, Math.max.apply(null, (DATA.leaderboard || []).map(function (g) { return g.runs || 0; })));
     (DATA.leaderboard || []).forEach(function (g) {
       var row = el("div", { class: "bar-row" });
       row.appendChild(el("div", { class: "lbl mono" }, esc(g.slug)));
       var track = el("div", { class: "bar-track" });
       var seg = el("div", {
         class: "bar-seg",
-        style: "width:" + Math.round((g.solved_rate || 0) * 100) + "%;background:" + colorFor(g.slug),
+        style: "width:" + Math.round((g.runs || 0) / maxRuns * 100) + "%;background:" + colorFor(g.slug),
       });
       track.appendChild(seg);
       row.appendChild(track);
-      row.appendChild(el("div", { class: "bar-val" }, pct(g.solved_rate) + " (" + g.solved + "/" + g.runs + ")"));
+      row.appendChild(el("div", { class: "bar-val" }, fmt(g.runs) + " runs"));
       host.appendChild(row);
     });
   })();
@@ -288,6 +250,7 @@
   (function () {
     var host = document.getElementById("chart-outcomes");
     var legend = document.getElementById("legend-outcomes");
+    if (!host || !legend) return;
     var outcomeColors = { Submitted: "#3ecf8e", LimitsExceeded: "#e8b339", TimeExceeded: "#e8b339", RepeatedFormatError: "#ef5a6f", ContextWindowExceeded: "#ef5a6f", Error: "#ef5a6f", Unknown: "#8b93a7" };
     function colorForOutcome(k) { return outcomeColors[k] || "#8b93a7"; }
     var allOutcomes = {};
@@ -313,18 +276,31 @@
   // ---------------------------------------------------------------- leaderboard table
   (function () {
     var container = document.getElementById("tbl-leaderboard");
+    var options = document.getElementById("leaderboard-model-options");
+    var filterLabel = document.getElementById("model-filter-label");
     var columns = [
       { key: "model", label: "Model", render: function (r) { return esc(r.model); } },
       { key: "reasoning", label: "Reasoning", render: function (r) { return esc(r.reasoning); } },
       { key: "runs", label: "Runs", render: function (r) { return fmt(r.runs); } },
-      { key: "solved_rate", label: "Solved", render: function (r) { return r.solved + "/" + r.runs + " (" + pct(r.solved_rate) + ")"; } },
-      { key: "avg_steps", label: "Avg steps", render: function (r) { return fmt1(r.avg_steps); } },
-      { key: "avg_elapsed_seconds", label: "Avg time", render: function (r) { return fmt1(r.avg_elapsed_seconds) + "s"; } },
-      { key: "avg_tokens", label: "Avg tokens", render: function (r) { return fmt(Math.round(r.avg_tokens)); } },
+      { key: "sum_steps", label: "Total steps", render: function (r) { return fmt(r.sum_steps); } },
+      { key: "sum_elapsed_seconds", label: "Time taken", render: function (r) { return fmt(r.sum_elapsed_seconds) + "s"; } },
+      { key: "sum_tokens", label: "Tokens", render: function (r) { return fmt(r.sum_tokens); } },
       { key: "cost_usd", label: "Cost", render: function (r) { return fmtUsd(r.cost_usd) + (r.cost_source === "estimated" ? "~" : ""); } },
-      { key: "avg_n_failed_commands", label: "Fail/run", render: function (r) { return fmt1(r.avg_n_failed_commands); } },
     ];
-    sortableTable(container, columns, DATA.leaderboard || [], { defaultKey: "solved_rate" });
+    (DATA.leaderboard || []).forEach(function (r) {
+      var option = el("label", { class: "model-option" });
+      option.appendChild(el("input", { type: "checkbox", value: r.slug }));
+      option.appendChild(document.createTextNode(r.model + " · " + r.reasoning));
+      options.appendChild(option);
+    });
+    function draw() {
+      var selected = Array.prototype.slice.call(options.querySelectorAll("input:checked")).map(function (o) { return o.value; });
+      var rows = selected.length ? (DATA.leaderboard || []).filter(function (r) { return selected.indexOf(r.slug) !== -1; }) : DATA.leaderboard || [];
+      filterLabel.textContent = selected.length ? selected.length + " model" + (selected.length === 1 ? "" : "s") + " selected" : "All models";
+      sortableTable(container, columns, rows, { defaultKey: "sum_tokens" });
+    }
+    options.addEventListener("change", draw);
+    draw();
   })();
 
   // ---------------------------------------------------------------- token mix stacked bar
@@ -354,29 +330,30 @@
     });
   })();
 
-  // ---------------------------------------------------------------- scatter: cost vs solve rate
+  // ---------------------------------------------------------------- scatter: cost vs duration
   (function () {
     var host = document.getElementById("chart-scatter");
     var W = host.clientWidth || 460, H = 260, pad = { l: 42, r: 16, t: 14, b: 30 };
     var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
     var data = (DATA.leaderboard || []).filter(function (g) { return g.cost_usd != null; });
     var maxCost = Math.max(0.001, Math.max.apply(null, data.map(function (g) { return g.cost_usd; }).concat([0.001])));
+    var maxDuration = Math.max(1, Math.max.apply(null, data.map(function (g) { return g.avg_elapsed_seconds || 0; }).concat([1])));
     function x(v) { return pad.l + (v / maxCost) * (W - pad.l - pad.r); }
-    function y(v) { return H - pad.b - v * (H - pad.t - pad.b); }
+    function y(v) { return H - pad.b - (v / maxDuration) * (H - pad.t - pad.b); }
     // axes
     svg.appendChild(svgEl("line", { x1: pad.l, x2: W - pad.r, y1: H - pad.b, y2: H - pad.b, stroke: "#262b36" }));
     svg.appendChild(svgEl("line", { x1: pad.l, x2: pad.l, y1: pad.t, y2: H - pad.b, stroke: "#262b36" }));
     [0, 0.25, 0.5, 0.75, 1].forEach(function (t) {
-      var ty = y(t);
+      var ty = y(maxDuration * t);
       var lab = svgEl("text", { x: pad.l - 8, y: ty + 3, "text-anchor": "end" });
-      lab.textContent = Math.round(t * 100) + "%";
+      lab.textContent = Math.round(maxDuration * t) + "s";
       svg.appendChild(lab);
     });
     var lab2 = svgEl("text", { x: W - pad.r, y: H - pad.b + 20, "text-anchor": "end" });
     lab2.textContent = "cost \u2192 (max " + fmtUsd(maxCost) + ")";
     svg.appendChild(lab2);
     data.forEach(function (g) {
-      var cx = x(g.cost_usd), cy = y(g.solved_rate);
+      var cx = x(g.cost_usd), cy = y(g.avg_elapsed_seconds || 0);
       var r = 6 + Math.min(10, (g.avg_tokens || 0) / 4000);
       var c = svgEl("circle", { cx: cx, cy: cy, r: r, fill: colorFor(g.slug), "fill-opacity": 0.85 });
       svg.appendChild(c);
@@ -387,6 +364,38 @@
     host.appendChild(svg);
   })();
 
+  // ---------------------------------------------------------------- scatter: cost vs average steps
+  (function () {
+    var host = document.getElementById("chart-scatter-steps");
+    var W = host.clientWidth || 460, H = 260, pad = { l: 42, r: 16, t: 14, b: 30 };
+    var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
+    var data = (DATA.leaderboard || []).filter(function (g) { return g.cost_usd != null; });
+    var maxCost = Math.max(0.001, Math.max.apply(null, data.map(function (g) { return g.cost_usd; }).concat([0.001])));
+    var maxSteps = Math.max(1, Math.max.apply(null, data.map(function (g) { return g.avg_steps || 0; }).concat([1])));
+    function x(v) { return pad.l + (v / maxCost) * (W - pad.l - pad.r); }
+    function y(v) { return H - pad.b - (v / maxSteps) * (H - pad.t - pad.b); }
+    svg.appendChild(svgEl("line", { x1: pad.l, x2: W - pad.r, y1: H - pad.b, y2: H - pad.b, stroke: "#e4e7eb" }));
+    svg.appendChild(svgEl("line", { x1: pad.l, x2: pad.l, y1: pad.t, y2: H - pad.b, stroke: "#e4e7eb" }));
+    [0, 0.25, 0.5, 0.75, 1].forEach(function (t) {
+      var ty = y(maxSteps * t);
+      var lab = svgEl("text", { x: pad.l - 8, y: ty + 3, "text-anchor": "end" });
+      lab.textContent = fmt1(maxSteps * t);
+      svg.appendChild(lab);
+    });
+    var lab2 = svgEl("text", { x: W - pad.r, y: H - pad.b + 20, "text-anchor": "end" });
+    lab2.textContent = "cost \u2192 (max " + fmtUsd(maxCost) + ")";
+    svg.appendChild(lab2);
+    data.forEach(function (g) {
+      var cx = x(g.cost_usd), cy = y(g.avg_steps || 0);
+      var r = 6 + Math.min(10, (g.avg_tokens || 0) / 4000);
+      svg.appendChild(svgEl("circle", { cx: cx, cy: cy, r: r, fill: colorFor(g.slug), "fill-opacity": 0.85 }));
+      var label = svgEl("text", { x: cx + r + 4, y: cy + 3 });
+      label.textContent = g.slug;
+      svg.appendChild(label);
+    });
+    host.appendChild(svg);
+  })();
+
   // ---------------------------------------------------------------- per-step line charts
   function lineChart(host, series, valueFn, opts) {
     opts = opts || {};
@@ -444,6 +453,7 @@
   (function () {
     var matrix = DATA.matrix || [];
     var container = document.getElementById("tbl-matrix");
+    if (!container) return;
     if (matrix.length === 0) {
       document.getElementById("section-matrix").style.display = "none";
       return;
@@ -467,7 +477,7 @@
       var cells = tasks.map(function (t) {
         var m = cell[slug + "\u0000" + t];
         if (!m) return "<td>&mdash;</td>";
-        return '<td><span class="heat-cell" style="background:' + heatColor(m.pass_at_k) + '">' + m.passed + "/" + m.k + "<br>" + pct(m.pass_at_k) + "</span></td>";
+        return '<td><span class="heat-cell" style="background:' + heatColor(m.completion_rate) + '">' + m.completed + "/" + m.k + "<br>" + pct(m.completion_rate) + "</span></td>";
       }).join("");
       return "<tr><td class=\"mono\">" + esc(slug) + "</td>" + cells + "</tr>";
     }).join("") + "</tbody>";
@@ -481,6 +491,7 @@
     var fModel = document.getElementById("filter-model");
     var fTask = document.getElementById("filter-task");
     var fOutcome = document.getElementById("filter-outcome");
+    if (!container || !fModel || !fTask || !fOutcome) return;
 
     function uniq(fn) {
       var seen = {}, out = [];
@@ -489,15 +500,15 @@
     }
     uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
     uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
-    uniq(function (r) { return r.exit_status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
+    uniq(function (r) { return r.status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
 
     var columns = [
       { key: "slug", label: "Model", render: function (r) { return '<span class="mono">' + esc(r.slug) + "</span>"; } },
       { key: "task_id", label: "Task", render: function (r) { return esc(r.task_id); } },
       { key: "run_id", label: "Run", render: function (r) { return esc(r.run_id); } },
-      { key: "exit_status", label: "Outcome", render: function (r) {
-          var cls = r.resolved ? "ok" : (r.exit_status === "Error" ? "bad" : "warn");
-          return '<span class="pill ' + cls + '">' + esc(r.exit_status || "Unknown") + "</span>";
+      { key: "status", label: "Status", render: function (r) {
+          var cls = r.completed ? "ok" : (r.status === "request_error" ? "bad" : "warn");
+          return '<span class="pill ' + cls + '">' + esc(r.status || "Unknown") + "</span>";
         } },
       { key: "steps", label: "Steps", render: function (r) { return fmt(r.steps); } },
       { key: "elapsed_seconds", label: "Time", render: function (r) { return fmt(r.elapsed_seconds) + "s"; } },
@@ -530,7 +541,7 @@
       var filtered = runs.filter(function (r) {
         if (fModel.value && r.slug !== fModel.value) return false;
         if (fTask.value && r.task_id !== fTask.value) return false;
-        if (fOutcome.value && r.exit_status !== fOutcome.value) return false;
+        if (fOutcome.value && r.status !== fOutcome.value) return false;
         return true;
       });
       sortableTable(container, columns, filtered, {

package/src/report.js

@@ -91,17 +91,19 @@ export function buildSummary(runs) {
       const k = r.exit_status || "Unknown";
       outcomes[k] = (outcomes[k] || 0) + 1;
     }
-    const solved = rs.filter((r) => r.resolved).length;
+    const completed = rs.filter((r) => r.completed).length;
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
       slug,
       runs: rs.length,
-      solved,
-      solved_rate: rs.length ? solved / rs.length : 0,
+      completed,
+      completion_rate: rs.length ? completed / rs.length : 0,
       outcomes,
       avg_steps: avg(rs, (r) => r.steps),
+      sum_steps: sum(rs, (r) => r.steps),
       avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
+      sum_elapsed_seconds: sum(rs, (r) => r.elapsed_seconds),
       avg_tokens: avg(rs, (r) => r.tokens.total),
       sum_tokens: sum(rs, (r) => r.tokens.total),
       tokens: {
@@ -118,10 +120,10 @@ export function buildSummary(runs) {
       avg_n_retries: avg(rs, (r) => r.n_retries),
     };
   });
-  leaderboard.sort((a, b) => b.solved_rate - a.solved_rate || a.avg_steps - b.avg_steps);
+  leaderboard.sort((a, b) => b.completion_rate - a.completion_rate || a.avg_steps - b.avg_steps);
 
   const matrix = groupBy(runs, (r) => `${r.slug}\u0000${r.task_id}`).map(([, rs]) => {
-    const passed = rs.filter((r) => r.resolved).length;
+    const completed = rs.filter((r) => r.completed).length;
     const costRuns = rs.filter((r) => r.cost_source);
     return {
       model: rs[0].model,
@@ -130,8 +132,8 @@ export function buildSummary(runs) {
       task_id: rs[0].task_id,
       task_title: rs[0].task_title,
       k: rs.length,
-      passed,
-      pass_at_k: rs.length ? passed / rs.length : 0,
+      completed,
+      completion_rate: rs.length ? completed / rs.length : 0,
       avg_steps: avg(rs, (r) => r.steps),
       avg_tokens: avg(rs, (r) => r.tokens.total),
       avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
@@ -190,7 +192,7 @@ export function buildSummary(runs) {
     .sort((a, b) => a.slug.localeCompare(b.slug) || a.step - b.step);
 
   const costRuns = runs.filter((r) => r.cost_source);
-  const solved = runs.filter((r) => r.resolved).length;
+  const completed = runs.filter((r) => r.completed).length;
   const sources = new Set(costRuns.map((r) => r.cost_source));
 
   return {
@@ -200,8 +202,8 @@ export function buildSummary(runs) {
       models: new Set(runs.map((r) => r.slug)).size,
       tasks: new Set(runs.map((r) => r.task_id)).size,
       runs: runs.length,
-      solved,
-      solved_rate: runs.length ? solved / runs.length : 0,
+      completed,
+      completion_rate: runs.length ? completed / runs.length : 0,
       cost_usd: costRuns.length ? round(sum(costRuns, (r) => r.cost_usd), 6) : null,
       cost_source: sources.size === 0 ? "unknown" : sources.size === 1 ? [...sources][0] : "mixed",
       tokens: sum(runs, (r) => r.tokens.total),
@@ -221,7 +223,7 @@ export function renderReportHtml(summary) {
   if (!tpl.includes("/*__ASTRA_DATA__*/")) {
     throw new Error("src/report.html is missing the /*__ASTRA_DATA__*/ injection marker");
   }
-  return tpl.replace("/*__ASTRA_DATA__*/", `window.DATA = ${data};`);
+  return tpl.replace("/*__ASTRA_DATA__*/", `window.__ASTRA_EMBEDDED_DATA__ = ${data};`);
 }
 
 function toRunDoc(run) {
@@ -232,6 +234,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   const trajPath = path.join(dir, "trajectory.json");
   const metricsPath = path.join(dir, "metrics.csv");
   const taskPath = path.join(dir, "task.md");
+  const projectPath = path.join(dir, "project.json");
   if (!fs.existsSync(trajPath) && !fs.existsSync(metricsPath)) return null;
 
   let traj = null;
@@ -243,6 +246,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     }
   }
   const metricsRow = fs.existsSync(metricsPath) ? parseCsv(fs.readFileSync(metricsPath, "utf8"))[0] : null;
+  const project = readJson(projectPath);
   const taskMd = fs.existsSync(taskPath) ? fs.readFileSync(taskPath, "utf8") : traj?.info?.task || "";
   const parsed = parseSlug(slug);
   const model = metricsRow?.model || traj?.info?.model || parsed.model;
@@ -259,11 +263,11 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     last_context: num(metricsRow?.last_context_tokens ?? traj?.info?.tokens?.last_context),
   };
   const costSource = metricsRow?.cost_source || traj?.info?.cost?.source || "";
-  const resolvedRaw = metricsRow?.resolved;
-  const resolved =
-    resolvedRaw == null
-      ? traj?.info?.exit_status === "Submitted"
-      : resolvedRaw === true || resolvedRaw === "1" || resolvedRaw === 1;
+  const legacyResolved = metricsRow?.resolved;
+  const status = metricsRow?.status ||
+    ((legacyResolved == null ? traj?.info?.exit_status === "Submitted" : legacyResolved === true || legacyResolved === "1" || legacyResolved === 1)
+      ? "completed"
+      : "model_error");
 
   return {
     id: `${slug}/${runId}`,
@@ -271,12 +275,16 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     model,
     reasoning,
     slug,
-    task_id: inferTaskId({ taskMd }),
+    task_id: typeof project?.id === "string" && project.id ? project.id : inferTaskId({ taskMd }),
     task_title: inferTaskTitle(taskMd),
+    project: project && typeof project.id === "string"
+      ? { id: project.id, version: project.version }
+      : null,
     run: runNum,
     run_id: runId,
     exit_status: metricsRow?.exit_status || traj?.info?.exit_status || "",
-    resolved: !!resolved,
+    status,
+    completed: status === "completed",
     error: null,
     steps: num(metricsRow?.steps ?? traj?.info?.n_steps),
     n_calls: num(metricsRow?.n_calls ?? traj?.info?.n_calls),
@@ -299,6 +307,15 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   };
 }
 
+function readJson(file) {
+  if (!fs.existsSync(file)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
 export function extractTimeline(doc) {
   const messages = doc?.messages || [];
   const steps = [];