@hackerrank/astra-cli 0.1.3 → 0.1.4

package/README.md

@@ -120,7 +120,7 @@ astra -m m1,m2 -p tasks/dummy-slugify --repeat 3 \
   --push s3://hackerrank-astra-bench-results-dev/runs/2025-09-02/
 ```
 
-Leaderboard columns: `solved` (pass rate), average `steps` / `tokens`, and
+Leaderboard columns: completed generation rate, average `steps` / `tokens`, and
 `cost`. Cost is per-run summed; a model whose route reports no cost and has no
 price entry shows **`n/a`** (never `$0`), and estimated costs (native routes) are
 marked `~`. `--push` requires an `s3://` URI and the `aws` CLI on PATH; if the
@@ -132,13 +132,52 @@ dashboard with `Content-Type: text/html` and prints a **7-day presigned
 read), so that presigned URL is the only way to open the dashboard — e.g.
 clickable straight from a Jenkins console log.
 
+### Project-configured bench runs
+
+For repeatable benchmarks, keep the benchmark definition with the task and run
+it through the existing bench persona:
+
+```bash
+astra bench --project ./task
+```
+
+`./task/.astra/project.toml` selects the public workspace, instruction,
+models, limits, output root, and optional bench template/extensions. CLI flags
+still override bench limits and output paths. Astra copies only the configured
+workspace; container lifecycle, services, secrets, verification, and scoring
+remain harness responsibilities. Run artifacts include `project.json` with the
+project and input hashes. `completed` means the agent emitted its completion
+sentinel—not that the candidate is correct; `model_error` and `request_error`
+distinguish generation/provider failures from invalid invocation/configuration.
+
+```toml
+[project]
+id = "peoplecore-scim"
+version = 1
+instruction = "instruction.md"
+workspace = "public/codebase"
+
+[bench]
+models = ["gpt-5.6-sol"]
+reasoning = ["medium"]
+repeat = 3
+steps = 60
+wall_seconds = 3600
+command_timeout_seconds = 120
+
+[templates]
+bench = ".astra/templates/bench.md"
+[extensions]
+directories = [".astra/extensions"]
+```
+
 ### HTML report
 
 Every matrix run (and every single `-p`/`-t`/`-f` bench run) rebuilds
 `bench/summary.json` and a self-contained `bench/report.html` dashboard —
 no server, no build step, no external JS: open it straight from `file://` or
 from inside the `bench-*.tgz` tarball. It has a leaderboard, a model × task
-pass-rate heatmap, cost/token charts, per-step trend lines, and an
+completion-rate heatmap, cost/token charts, per-step trend lines, and an
 expandable command timeline for every run.
 
 Regenerate it on demand (e.g. after manually editing/pruning `bench/`) with:
@@ -150,7 +189,7 @@ astra --report --bench-root ./other-bench
 
 `summary.json` follows the `astra-bench-1` schema: `kpis` (totals), a tidy
 `leaderboard[]` (one row per model×reasoning), `matrix[]` (model×reasoning×task
-pass@k), `runs[]` (every attempt, with a compact per-step `timeline`), and
+completion telemetry), `runs[]` (every attempt, with a compact per-step `timeline`), and
 `step_series[]` (token/cost distributions by step, for the trend charts). Each
 run folder also gets a standalone `run.json` for drill-down without loading
 the full `trajectory.json`.
@@ -178,6 +217,7 @@ credits — that's a quota issue, not a bug.
 -t, --task <text>       Task text -> autonomous mode
 -f, --task-file <path>  Read task text from a file -> autonomous mode
 -p, --path <dir>        Task directory -> isolated bench workspace
+    --project <path>     Load .astra/project.toml for an explicit bench run
     --repeat <n>        Attempts per (model,reasoning) in a matrix (default: 1)
     --bench-root <dir>  Root folder for bench runs (default: ./bench)
     --tar               After a matrix, write a bench/ tarball locally

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {
@@ -16,7 +16,8 @@
   },
   "scripts": {
     "start": "node src/cli.js",
-    "astra": "node src/cli.js"
+    "astra": "node src/cli.js",
+    "test": "node --test"
   },
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
@@ -26,5 +27,8 @@
     "type": "git",
     "url": "git+ssh://git@github.com/interviewstreet/hrbench.git"
   },
-  "license": "MIT"
+  "license": "MIT",
+  "dependencies": {
+    "@iarna/toml": "2.2.5"
+  }
 }

package/src/bench.js

@@ -30,7 +30,8 @@ export const METRIC_COLUMNS = [
   "model",
   "reasoning",
   "exit_status",
-  "resolved",
+  "status",
+  "completed",
   "steps",
   "n_commands",
   "n_failed_commands",
@@ -104,6 +105,7 @@ export function allocateRun({ model, reasoning, root } = {}) {
  */
 export function seedWorkspace(workspace, { taskPath, taskFile, taskText } = {}) {
   let text = taskText ?? "";
+  const hasExplicitTask = taskText !== undefined && taskText !== null;
   let copyFrom = null;
   let instructionName = null;
 
@@ -114,7 +116,7 @@ export function seedWorkspace(workspace, { taskPath, taskFile, taskText } = {})
       copyFrom = resolved;
       // Prefer a conventional instruction file for the task text.
       for (const name of ["instruction.md", "INSTRUCTION.md", "task.md", "README.md"]) {
-        if (fs.existsSync(path.join(resolved, name))) {
+        if (!hasExplicitTask && fs.existsSync(path.join(resolved, name))) {
           text = fs.readFileSync(path.join(resolved, name), "utf8");
           break;
         }
@@ -122,13 +124,13 @@ export function seedWorkspace(workspace, { taskPath, taskFile, taskText } = {})
     } else {
       copyFrom = path.dirname(resolved);
       instructionName = path.basename(resolved);
-      text = fs.readFileSync(resolved, "utf8");
+      if (!hasExplicitTask) text = fs.readFileSync(resolved, "utf8");
     }
   } else if (taskFile) {
     const resolved = path.resolve(taskFile);
     copyFrom = path.dirname(resolved);
     instructionName = path.basename(resolved);
-    text = fs.readFileSync(resolved, "utf8");
+    if (!hasExplicitTask) text = fs.readFileSync(resolved, "utf8");
   }
 
   if (copyFrom) copyDir(copyFrom, workspace);
@@ -162,13 +164,14 @@ function copyDir(from, to) {
 /** Build a metrics row object from a finished agent + run info. */
 export function collectMetrics({ agent, model, reasoning }) {
   const info = agent.serialize().info;
-  const resolved = info.exit_status === "Submitted" ? 1 : 0;
+  const status = generationStatus(info.exit_status);
   return {
     timestamp: new Date().toISOString(),
     model: model.model,
     reasoning: reasoning || "none",
     exit_status: info.exit_status || "",
-    resolved,
+    status,
+    completed: status === "completed" ? 1 : 0,
     steps: info.n_steps ?? 0,
     n_commands: agent.nCommands ?? 0,
     n_failed_commands: agent.nFailedCommands ?? 0,
@@ -188,6 +191,11 @@ export function collectMetrics({ agent, model, reasoning }) {
   };
 }
 
+/** Classify autonomous generation without making a correctness claim. */
+export function generationStatus(exitStatus) {
+  return exitStatus === "Submitted" ? "completed" : "model_error";
+}
+
 /** Serialize a metrics row to a single CSV line (columns in METRIC_COLUMNS order). */
 export function metricsRow(m) {
   return METRIC_COLUMNS.map((c) => csvCell(m[c])).join(",");
@@ -219,6 +227,14 @@ export function writeMetrics({ runDir, root, metrics }) {
   return { runMetrics: path.join(runDir, "metrics.csv"), index: indexPath };
 }
 
+/** Persist the project-owned inputs that defined a bench attempt. */
+export function writeProjectMetadata(runDir, metadata) {
+  if (!metadata) return null;
+  const file = path.join(runDir, "project.json");
+  fs.writeFileSync(file, JSON.stringify({ schema: "astra-project-1", ...metadata }, null, 2) + "\n");
+  return file;
+}
+
 function csvCell(v) {
   const s = v == null ? "" : String(v);
   return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
@@ -264,6 +280,7 @@ export async function runCell({
   task,
   taskPath,
   taskFile,
+  projectMetadata,
   root,
   steps = 40,
   wall = 0,
@@ -278,6 +295,7 @@ export async function runCell({
   const seeded = seedWorkspace(alloc.workspace, { taskPath, taskFile, taskText: task });
   const taskText = seeded.taskText || task || "";
   fs.writeFileSync(path.join(alloc.dir, "task.md"), taskText);
+  writeProjectMetadata(alloc.dir, projectMetadata);
 
   const gw = new GatewayModel({
     model,
@@ -340,7 +358,7 @@ export async function runMatrix({
 
 /**
  * Aggregate metrics rows into a per-(model,reasoning) leaderboard.
- * `resolved` counts as a solve. Cost is summed only over runs that reported a
+ * `completed` counts a completed generation. Cost is summed only over runs that reported a
  * cost; a group with no cost data is reported as null (rendered "n/a"), never $0.
  */
 export function aggregate(rows) {
@@ -352,7 +370,7 @@ export function aggregate(rows) {
         model: r.model,
         reasoning: r.reasoning || "none",
         runs: 0,
-        solved: 0,
+        completed: 0,
         steps: 0,
         totalTokens: 0,
         costUsd: 0,
@@ -362,7 +380,7 @@ export function aggregate(rows) {
     }
     const g = groups.get(key);
     g.runs++;
-    g.solved += Number(r.resolved) ? 1 : 0;
+    g.completed += Number(r.completed) ? 1 : 0;
     g.steps += Number(r.steps) || 0;
     g.totalTokens += Number(r.total_tokens) || 0;
     if (r.cost_source && r.cost_source !== "") {
@@ -375,8 +393,8 @@ export function aggregate(rows) {
     model: g.model,
     reasoning: g.reasoning,
     runs: g.runs,
-    solved: g.solved,
-    solved_rate: g.runs ? g.solved / g.runs : 0,
+    completed: g.completed,
+    completion_rate: g.runs ? g.completed / g.runs : 0,
     avg_steps: g.runs ? g.steps / g.runs : 0,
     avg_tokens: g.runs ? g.totalTokens / g.runs : 0,
     cost_usd: g.costRuns ? g.costUsd : null,

package/src/cli.js

@@ -16,6 +16,7 @@
  *   astra -m <model> -t "<task text>"        bench run from task text
  *   astra -m <model> -f path/to/instruction.md
  *   astra -m <model> -p tasks/dummy-slugify  bench run from a task directory
+ *   astra bench --project ./task         project-configured bench run
  *   astra --resume <session-id>             resume a saved session
  *   astra --sessions                        list saved sessions
  *
@@ -36,6 +37,7 @@
  *       --report            Rebuild bench/summary.json + bench/report.html
  *                           from whatever runs already exist on disk, then
  *                           exit (no model/API key needed).
+ *       --project <path>    Load .astra/project.toml and run its bench setup
  *   -t, --task <text>       Task text -> bench mode (run to completion)
  *   -f, --task-file <path>  Read task text from a file -> bench mode
  *   -p, --path <dir>        Task directory (copied into an isolated bench
@@ -86,11 +88,13 @@ import {
   deriveTitle,
 } from "./session.js";
 import { runRepl } from "./repl.js";
+import { ProjectConfigError, loadProject } from "./project.js";
 import {
   allocateRun,
   seedWorkspace,
   collectMetrics,
   writeMetrics,
+  writeProjectMetadata,
   runMatrix,
   aggregate,
   packBench,
@@ -109,7 +113,7 @@ function reasoningKwargs(level) {
 }
 
 function parseArgs(argv) {
-  const args = { steps: 40, wall: 0, timeout: 60, quiet: false, "max-output": 16000 };
+  const args = { steps: 40, wall: 0, timeout: 60, quiet: false, "max-output": 16000, _provided: new Set() };
   const alias = {
     "-m": "model", "--model": "model",
     "-t": "task", "--task": "task",
@@ -121,6 +125,7 @@ function parseArgs(argv) {
     "--push-uri": "push-uri",
     "--tar": "tar",
     "--report": "report",
+    "--project": "project",
     "--bench-root": "bench-root",
     "-C": "cwd", "--cwd": "cwd",
     "-o": "output", "--output": "output",
@@ -137,11 +142,17 @@ function parseArgs(argv) {
     "-h": "help", "--help": "help",
   };
   const flags = new Set(["quiet", "yolo", "help", "sessions", "tar", "push", "report"]);
-  for (let i = 2; i < argv.length; i++) {
+  let start = 2;
+  if (argv[start] === "bench") {
+    args.bench = true;
+    start++;
+  }
+  for (let i = start; i < argv.length; i++) {
     const key = alias[argv[i]];
     if (!key) { console.error(`Unknown option: ${argv[i]}`); process.exit(2); }
     if (flags.has(key)) { args[key] = true; continue; }
     args[key] = argv[++i];
+    args._provided.add(key);
   }
   return args;
 }
@@ -183,11 +194,28 @@ async function main() {
     resumeDoc = loadSession(args.resume);
   }
 
-  // A task (-t/-f) or task path (-p) selects bench mode; otherwise agent mode.
+  // A task (-t/-f), task path (-p), or explicit bench project selects bench mode.
   // Bench mode uses the autonomous engine; agent mode uses the interactive one.
   let task = args.task;
   if (args["task-file"]) task = fs.readFileSync(args["task-file"], "utf8");
-  const wantsBench = !!(task || args.path);
+  let project = null;
+  if (args.project) {
+    try {
+      project = loadProject(args.project);
+    } catch (error) {
+      const message = error instanceof ProjectConfigError ? error.message : String(error?.message || error);
+      console.error(`\x1b[31m[astra] request_error: ${message}\x1b[0m`);
+      process.exit(2);
+    }
+    task = project.taskText;
+    args.path = project.workspace;
+    if (!args._provided.has("bench-root")) args["bench-root"] = project.outputRoot;
+    if (!args._provided.has("steps")) args.steps = project.bench.steps;
+    if (!args._provided.has("wall")) args.wall = project.bench.wall;
+    if (!args._provided.has("timeout")) args.timeout = project.bench.timeout;
+    if (!args._provided.has("repeat")) args.repeat = project.bench.repeat;
+  }
+  const wantsBench = !!(task || args.path || args.bench);
   const mode = wantsBench
     ? "autonomous"
     : resumeDoc?.info?.mode === "autonomous"
@@ -236,6 +264,8 @@ async function main() {
   let reasoning = args.reasoning || "";
 
   if (mode === "autonomous") {
+    if (project && !modelId) modelId = project.bench.models.join(",");
+    if (project && !reasoning) reasoning = project.bench.reasoning.join(",");
     if (!modelId) {
       console.error("\x1b[31m[astra] --model is required for bench mode.\x1b[0m\n");
       console.log(HELP);
@@ -254,7 +284,7 @@ async function main() {
     const isMatrix = models.length > 1 || reasonings.length > 1 || repeat > 1;
 
     if (isMatrix && !resumeDoc) {
-      await runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, quiet: !!args.quiet });
+      await runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, project, quiet: !!args.quiet });
       return; // runBenchMatrix exits the process
     }
   } else {
@@ -310,6 +340,7 @@ async function main() {
     task = seeded.taskText || task || "";
     // Persist the task text alongside the run for reproducibility.
     fs.writeFileSync(path.join(benchRun.dir, "task.md"), task || "");
+    writeProjectMetadata(benchRun.dir, project?.metadata);
   }
 
   const cmdCwd = benchRun ? benchRun.workspace : cwd;
@@ -510,7 +541,7 @@ function archiveAndPush(args, root, links) {
  * Run a multi-model / multi-reasoning / repeated bench matrix, print a
  * leaderboard, optionally tar + upload the bench/ folder, then exit.
  */
-async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, quiet }) {
+async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task, project, quiet }) {
   const root = args["bench-root"];
   const total = models.length * reasonings.length * repeat;
   console.error(
@@ -528,6 +559,7 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
     task,
     taskPath: args.path,
     taskFile: args["task-file"],
+    projectMetadata: project?.metadata,
     root,
     steps: Number(args.steps),
     wall: Number(args.wall),
@@ -540,8 +572,8 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
       }
     },
     onDone: (r) => {
-      const mark = r.resolved ? "\x1b[32m✓\x1b[0m" : "\x1b[31m✗\x1b[0m";
-      const status = r.error ? `Error: ${r.error.split("\n")[0]}` : r.exit_status;
+      const mark = r.completed ? "\x1b[32m✓\x1b[0m" : "\x1b[31m✗\x1b[0m";
+      const status = r.error ? `model_error: ${r.error.split("\n")[0]}` : r.status;
       console.error(
         `  ${mark} ${String(r.model).padEnd(20)} ${String(r.reasoning).padEnd(8)} ` +
           `${String(status).padEnd(16)} ${String(r.steps).padStart(3)} steps · ` +
@@ -570,8 +602,8 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
   // Tar + optional S3 push of the whole bench/ folder.
   archiveAndPush(args, root, links);
 
-  const solved = rows.filter((r) => r.resolved).length;
-  console.error(`\n\x1b[1m[astra] matrix done · ${solved}/${rows.length} resolved\x1b[0m`);
+  const completed = rows.filter((r) => r.completed).length;
+  console.error(`\n\x1b[1m[astra] matrix done · ${completed}/${rows.length} completed\x1b[0m`);
 
   // Print the important URLs / paths, aligned, at the very end.
   const w = Math.max(...links.map(([k]) => k.length));
@@ -580,23 +612,23 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
     console.error(`\x1b[1m  ${k.padEnd(w)}\x1b[0m  ${v}`);
   }
 
-  process.exit(solved === rows.length ? 0 : 1);
+  process.exit(completed === rows.length ? 0 : 1);
 }
 
 /** Print the per-(model,reasoning) leaderboard table. */
 function printLeaderboard(board) {
-  board.sort((a, b) => b.solved_rate - a.solved_rate || a.avg_steps - b.avg_steps);
+  board.sort((a, b) => b.completion_rate - a.completion_rate || a.avg_steps - b.avg_steps);
   console.error(
-    `\n\x1b[1m${"model".padEnd(20)} ${"reason".padEnd(8)} ${"solved".padEnd(8)} ` +
+    `\n\x1b[1m${"model".padEnd(20)} ${"reason".padEnd(8)} ${"complete".padEnd(8)} ` +
       `${"steps".padStart(6)} ${"tokens".padStart(9)}  ${"cost".padStart(9)}  source\x1b[0m`
   );
   for (const g of board) {
-    const solved = `${g.solved}/${g.runs}`;
-    const rate = `${Math.round(g.solved_rate * 100)}%`;
+    const completed = `${g.completed}/${g.runs}`;
+    const rate = `${Math.round(g.completion_rate * 100)}%`;
     const cost = g.cost_usd == null ? "n/a" : fmtUsd(g.cost_usd);
     console.error(
       `${String(g.model).padEnd(20)} ${String(g.reasoning).padEnd(8)} ` +
-        `${(solved + " " + rate).padEnd(8)} ${g.avg_steps.toFixed(1).padStart(6)} ` +
+        `${(completed + " " + rate).padEnd(8)} ${g.avg_steps.toFixed(1).padStart(6)} ` +
         `${fmt(Math.round(g.avg_tokens)).padStart(9)}  ${cost.padStart(9)}  ${g.cost_source}`
     );
   }

package/src/project.js

@@ -0,0 +1,177 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import toml from "@iarna/toml";
+
+export class ProjectConfigError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ProjectConfigError";
+  }
+}
+
+const DEFAULT_BENCH = {
+  models: [],
+  reasoning: [""],
+  repeat: 1,
+  steps: 40,
+  wall_seconds: 0,
+  command_timeout_seconds: 60,
+};
+
+function requireObject(value, name) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new ProjectConfigError(`${name} must be a TOML table`);
+  }
+  return value;
+}
+
+function requireString(value, name) {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new ProjectConfigError(`${name} must be a non-empty string`);
+  }
+  return value;
+}
+
+function positiveInteger(value, name, fallback, { allowZero = false } = {}) {
+  if (value === undefined) return fallback;
+  if (!Number.isInteger(value) || value < (allowZero ? 0 : 1)) {
+    throw new ProjectConfigError(`${name} must be ${allowZero ? "a non-negative" : "a positive"} integer`);
+  }
+  return value;
+}
+
+function stringArray(value, name, fallback) {
+  if (value === undefined) return fallback;
+  if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || !item.trim())) {
+    throw new ProjectConfigError(`${name} must be an array of non-empty strings`);
+  }
+  return value;
+}
+
+function resolveInside(root, relative, name) {
+  const candidate = path.resolve(root, requireString(relative, name));
+  if (candidate !== root && !candidate.startsWith(root + path.sep)) {
+    throw new ProjectConfigError(`${name} must stay inside the project directory`);
+  }
+  return candidate;
+}
+
+function readRegularFile(file, name, { optional = false } = {}) {
+  if (!fs.existsSync(file)) {
+    if (optional) return null;
+    throw new ProjectConfigError(`${name} does not exist: ${file}`);
+  }
+  const stat = fs.lstatSync(file);
+  if (!stat.isFile() || stat.isSymbolicLink()) {
+    throw new ProjectConfigError(`${name} must be a regular file: ${file}`);
+  }
+  return fs.readFileSync(file, "utf8");
+}
+
+function sha256(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function extensionFiles(root, directories) {
+  const seen = new Set();
+  const extensions = [];
+  for (const directory of directories) {
+    const resolved = resolveInside(root, directory, "extensions.directories entry");
+    if (!fs.existsSync(resolved)) {
+      throw new ProjectConfigError(`extension directory does not exist: ${resolved}`);
+    }
+    const stat = fs.lstatSync(resolved);
+    if (!stat.isDirectory() || stat.isSymbolicLink()) {
+      throw new ProjectConfigError(`extension directory must be a regular directory: ${resolved}`);
+    }
+    for (const entry of fs.readdirSync(resolved, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      if (!entry.isFile() || entry.isSymbolicLink()) continue;
+      const file = path.join(resolved, entry.name);
+      if (seen.has(file)) throw new ProjectConfigError(`duplicate extension path: ${file}`);
+      seen.add(file);
+      const content = readRegularFile(file, "extension");
+      extensions.push({ path: file, content, sha256: sha256(content) });
+    }
+  }
+  return extensions;
+}
+
+export function loadProject(projectPath) {
+  const root = path.resolve(requireString(projectPath, "project path"));
+  if (!fs.existsSync(root) || !fs.lstatSync(root).isDirectory()) {
+    throw new ProjectConfigError(`project directory does not exist: ${root}`);
+  }
+
+  const configPath = path.join(root, ".astra", "project.toml");
+  const configText = readRegularFile(configPath, "project configuration");
+  let config;
+  try {
+    config = toml.parse(configText);
+  } catch (error) {
+    throw new ProjectConfigError(`invalid project TOML: ${error.message}`);
+  }
+
+  const project = requireObject(config.project, "project");
+  const bench = requireObject(config.bench ?? {}, "bench");
+  const output = requireObject(config.output ?? {}, "output");
+  const templates = requireObject(config.templates ?? {}, "templates");
+  const extensionConfig = requireObject(config.extensions ?? {}, "extensions");
+  const provenance = requireObject(config.provenance ?? {}, "provenance");
+  const id = requireString(project.id, "project.id");
+  const version = positiveInteger(project.version, "project.version", undefined);
+
+  const instructionPath = resolveInside(root, project.instruction, "project.instruction");
+  const workspace = resolveInside(root, project.workspace, "project.workspace");
+  const instruction = readRegularFile(instructionPath, "instruction");
+  if (!instruction.trim()) throw new ProjectConfigError("instruction must not be empty");
+  if (!fs.existsSync(workspace) || !fs.lstatSync(workspace).isDirectory() || fs.lstatSync(workspace).isSymbolicLink()) {
+    throw new ProjectConfigError(`project.workspace must be a regular directory: ${workspace}`);
+  }
+
+  const templatePath = templates.bench === undefined ? null : resolveInside(root, templates.bench, "templates.bench");
+  const template = templatePath === null ? null : readRegularFile(templatePath, "bench template");
+  const extensions = extensionFiles(root, stringArray(extensionConfig.directories, "extensions.directories", []));
+  const taskText = [instruction, template, ...extensions.map((extension) => extension.content)]
+    .filter(Boolean)
+    .join("\n\n");
+
+  const baselineSha256 = provenance.baseline_sha256;
+  if (baselineSha256 !== undefined && (typeof baselineSha256 !== "string" || !/^[a-f0-9]{64}$/.test(baselineSha256))) {
+    throw new ProjectConfigError("provenance.baseline_sha256 must be a lowercase SHA-256 hash");
+  }
+
+  return {
+    root,
+    id,
+    version,
+    instructionPath,
+    instruction,
+    instructionSha256: sha256(instruction),
+    workspace,
+    outputRoot: resolveInside(root, output.root ?? "bench", "output.root"),
+    template: template === null ? null : { path: templatePath, content: template, sha256: sha256(template) },
+    extensions,
+    taskText,
+    provenance: { baselineSha256: baselineSha256 ?? null },
+    metadata: {
+      id,
+      version,
+      instruction_sha256: sha256(instruction),
+      baseline_sha256: baselineSha256 ?? null,
+      template_sha256: template === null ? null : sha256(template),
+      extension_sha256: extensions.map((extension) => ({
+        path: path.relative(root, extension.path),
+        sha256: extension.sha256,
+      })),
+    },
+    bench: {
+      models: stringArray(bench.models, "bench.models", DEFAULT_BENCH.models),
+      reasoning: stringArray(bench.reasoning, "bench.reasoning", DEFAULT_BENCH.reasoning),
+      repeat: positiveInteger(bench.repeat, "bench.repeat", DEFAULT_BENCH.repeat),
+      steps: positiveInteger(bench.steps, "bench.steps", DEFAULT_BENCH.steps),
+      wall: positiveInteger(bench.wall_seconds, "bench.wall_seconds", DEFAULT_BENCH.wall_seconds, { allowZero: true }),
+      timeout: positiveInteger(bench.command_timeout_seconds, "bench.command_timeout_seconds", DEFAULT_BENCH.command_timeout_seconds),
+    },
+  };
+}

package/src/report.html

@@ -477,7 +477,7 @@
       var cells = tasks.map(function (t) {
         var m = cell[slug + "\u0000" + t];
         if (!m) return "<td>&mdash;</td>";
-        return '<td><span class="heat-cell" style="background:' + heatColor(m.pass_at_k) + '">' + m.passed + "/" + m.k + "<br>" + pct(m.pass_at_k) + "</span></td>";
+        return '<td><span class="heat-cell" style="background:' + heatColor(m.completion_rate) + '">' + m.completed + "/" + m.k + "<br>" + pct(m.completion_rate) + "</span></td>";
       }).join("");
       return "<tr><td class=\"mono\">" + esc(slug) + "</td>" + cells + "</tr>";
     }).join("") + "</tbody>";
@@ -500,15 +500,15 @@
     }
     uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
     uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
-    uniq(function (r) { return r.exit_status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
+    uniq(function (r) { return r.status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
 
     var columns = [
       { key: "slug", label: "Model", render: function (r) { return '<span class="mono">' + esc(r.slug) + "</span>"; } },
       { key: "task_id", label: "Task", render: function (r) { return esc(r.task_id); } },
       { key: "run_id", label: "Run", render: function (r) { return esc(r.run_id); } },
-      { key: "exit_status", label: "Outcome", render: function (r) {
-          var cls = r.resolved ? "ok" : (r.exit_status === "Error" ? "bad" : "warn");
-          return '<span class="pill ' + cls + '">' + esc(r.exit_status || "Unknown") + "</span>";
+      { key: "status", label: "Status", render: function (r) {
+          var cls = r.completed ? "ok" : (r.status === "request_error" ? "bad" : "warn");
+          return '<span class="pill ' + cls + '">' + esc(r.status || "Unknown") + "</span>";
         } },
       { key: "steps", label: "Steps", render: function (r) { return fmt(r.steps); } },
       { key: "elapsed_seconds", label: "Time", render: function (r) { return fmt(r.elapsed_seconds) + "s"; } },
@@ -541,7 +541,7 @@
       var filtered = runs.filter(function (r) {
         if (fModel.value && r.slug !== fModel.value) return false;
         if (fTask.value && r.task_id !== fTask.value) return false;
-        if (fOutcome.value && r.exit_status !== fOutcome.value) return false;
+        if (fOutcome.value && r.status !== fOutcome.value) return false;
         return true;
       });
       sortableTable(container, columns, filtered, {

package/src/report.js

@@ -91,14 +91,14 @@ export function buildSummary(runs) {
       const k = r.exit_status || "Unknown";
       outcomes[k] = (outcomes[k] || 0) + 1;
     }
-    const solved = rs.filter((r) => r.resolved).length;
+    const completed = rs.filter((r) => r.completed).length;
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
       slug,
       runs: rs.length,
-      solved,
-      solved_rate: rs.length ? solved / rs.length : 0,
+      completed,
+      completion_rate: rs.length ? completed / rs.length : 0,
       outcomes,
       avg_steps: avg(rs, (r) => r.steps),
       sum_steps: sum(rs, (r) => r.steps),
@@ -120,10 +120,10 @@ export function buildSummary(runs) {
       avg_n_retries: avg(rs, (r) => r.n_retries),
     };
   });
-  leaderboard.sort((a, b) => b.solved_rate - a.solved_rate || a.avg_steps - b.avg_steps);
+  leaderboard.sort((a, b) => b.completion_rate - a.completion_rate || a.avg_steps - b.avg_steps);
 
   const matrix = groupBy(runs, (r) => `${r.slug}\u0000${r.task_id}`).map(([, rs]) => {
-    const passed = rs.filter((r) => r.resolved).length;
+    const completed = rs.filter((r) => r.completed).length;
     const costRuns = rs.filter((r) => r.cost_source);
     return {
       model: rs[0].model,
@@ -132,8 +132,8 @@ export function buildSummary(runs) {
       task_id: rs[0].task_id,
       task_title: rs[0].task_title,
       k: rs.length,
-      passed,
-      pass_at_k: rs.length ? passed / rs.length : 0,
+      completed,
+      completion_rate: rs.length ? completed / rs.length : 0,
       avg_steps: avg(rs, (r) => r.steps),
       avg_tokens: avg(rs, (r) => r.tokens.total),
       avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
@@ -192,7 +192,7 @@ export function buildSummary(runs) {
     .sort((a, b) => a.slug.localeCompare(b.slug) || a.step - b.step);
 
   const costRuns = runs.filter((r) => r.cost_source);
-  const solved = runs.filter((r) => r.resolved).length;
+  const completed = runs.filter((r) => r.completed).length;
   const sources = new Set(costRuns.map((r) => r.cost_source));
 
   return {
@@ -202,8 +202,8 @@ export function buildSummary(runs) {
       models: new Set(runs.map((r) => r.slug)).size,
       tasks: new Set(runs.map((r) => r.task_id)).size,
       runs: runs.length,
-      solved,
-      solved_rate: runs.length ? solved / runs.length : 0,
+      completed,
+      completion_rate: runs.length ? completed / runs.length : 0,
       cost_usd: costRuns.length ? round(sum(costRuns, (r) => r.cost_usd), 6) : null,
       cost_source: sources.size === 0 ? "unknown" : sources.size === 1 ? [...sources][0] : "mixed",
       tokens: sum(runs, (r) => r.tokens.total),
@@ -234,6 +234,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   const trajPath = path.join(dir, "trajectory.json");
   const metricsPath = path.join(dir, "metrics.csv");
   const taskPath = path.join(dir, "task.md");
+  const projectPath = path.join(dir, "project.json");
   if (!fs.existsSync(trajPath) && !fs.existsSync(metricsPath)) return null;
 
   let traj = null;
@@ -245,6 +246,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     }
   }
   const metricsRow = fs.existsSync(metricsPath) ? parseCsv(fs.readFileSync(metricsPath, "utf8"))[0] : null;
+  const project = readJson(projectPath);
   const taskMd = fs.existsSync(taskPath) ? fs.readFileSync(taskPath, "utf8") : traj?.info?.task || "";
   const parsed = parseSlug(slug);
   const model = metricsRow?.model || traj?.info?.model || parsed.model;
@@ -261,11 +263,11 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     last_context: num(metricsRow?.last_context_tokens ?? traj?.info?.tokens?.last_context),
   };
   const costSource = metricsRow?.cost_source || traj?.info?.cost?.source || "";
-  const resolvedRaw = metricsRow?.resolved;
-  const resolved =
-    resolvedRaw == null
-      ? traj?.info?.exit_status === "Submitted"
-      : resolvedRaw === true || resolvedRaw === "1" || resolvedRaw === 1;
+  const legacyResolved = metricsRow?.resolved;
+  const status = metricsRow?.status ||
+    ((legacyResolved == null ? traj?.info?.exit_status === "Submitted" : legacyResolved === true || legacyResolved === "1" || legacyResolved === 1)
+      ? "completed"
+      : "model_error");
 
   return {
     id: `${slug}/${runId}`,
@@ -273,12 +275,16 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     model,
     reasoning,
     slug,
-    task_id: inferTaskId({ taskMd }),
+    task_id: typeof project?.id === "string" && project.id ? project.id : inferTaskId({ taskMd }),
     task_title: inferTaskTitle(taskMd),
+    project: project && typeof project.id === "string"
+      ? { id: project.id, version: project.version }
+      : null,
     run: runNum,
     run_id: runId,
     exit_status: metricsRow?.exit_status || traj?.info?.exit_status || "",
-    resolved: !!resolved,
+    status,
+    completed: status === "completed",
     error: null,
     steps: num(metricsRow?.steps ?? traj?.info?.n_steps),
     n_calls: num(metricsRow?.n_calls ?? traj?.info?.n_calls),
@@ -301,6 +307,15 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   };
 }
 
+function readJson(file) {
+  if (!fs.existsSync(file)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
 export function extractTimeline(doc) {
   const messages = doc?.messages || [];
   const steps = [];