@hackerrank/astra-cli 0.1.6 → 0.1.7

package/src/project.js

@@ -19,6 +19,9 @@ const DEFAULT_BENCH = {
   command_timeout_seconds: 60,
 };
 
+const DEFAULT_VERIFICATION = null;
+const TASK_TYPES = new Set(["brownfield", "greenfield"]);
+
 function requireObject(value, name) {
   if (!value || typeof value !== "object" || Array.isArray(value)) {
     throw new ProjectConfigError(`${name} must be a TOML table`);
@@ -49,6 +52,42 @@ function stringArray(value, name, fallback) {
   return value;
 }
 
+function repeatOverrides(value) {
+  if (value === undefined) return {};
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new ProjectConfigError("bench.repeats must be a TOML table");
+  }
+  const result = {};
+  for (const [key, count] of Object.entries(value)) {
+    if (!key.trim()) throw new ProjectConfigError("bench.repeats keys must be non-empty");
+    result[key] = positiveInteger(count, `bench.repeats.${key}`, 1);
+  }
+  return result;
+}
+
+function verificationConfig(value) {
+  if (value === undefined) return DEFAULT_VERIFICATION;
+  const verification = requireObject(value, "verification");
+  return {
+    command: requireString(verification.command, "verification.command"),
+    startCommand: verification.start_command === undefined ? null : requireString(verification.start_command, "verification.start_command"),
+    stopCommand: verification.stop_command === undefined ? null : requireString(verification.stop_command, "verification.stop_command"),
+    baseUrl: verification.base_url === undefined ? null : requireString(verification.base_url, "verification.base_url"),
+    readinessUrl: verification.readiness_url === undefined ? null : requireString(verification.readiness_url, "verification.readiness_url"),
+    tokenEnv: verification.token_env === undefined ? null : requireString(verification.token_env, "verification.token_env"),
+    timeoutSeconds: positiveInteger(verification.timeout_seconds, "verification.timeout_seconds", 3600),
+    report: verification.report === undefined ? "verifier-report.json" : requireString(verification.report, "verification.report"),
+  };
+}
+
+function taskType(project) {
+  const value = project.type ?? project.profile ?? "brownfield";
+  if (typeof value !== "string" || !TASK_TYPES.has(value)) {
+    throw new ProjectConfigError("project.type must be either brownfield or greenfield");
+  }
+  return value;
+}
+
 function resolveInside(root, relative, name) {
   const candidate = path.resolve(root, requireString(relative, name));
   if (candidate !== root && !candidate.startsWith(root + path.sep)) {
@@ -114,12 +153,14 @@ export function loadProject(projectPath) {
 
   const project = requireObject(config.project, "project");
   const bench = requireObject(config.bench ?? {}, "bench");
+  let verification = verificationConfig(config.verification);
   const output = requireObject(config.output ?? {}, "output");
   const templates = requireObject(config.templates ?? {}, "templates");
   const extensionConfig = requireObject(config.extensions ?? {}, "extensions");
   const provenance = requireObject(config.provenance ?? {}, "provenance");
   const id = requireString(project.id, "project.id");
   const version = positiveInteger(project.version, "project.version", undefined);
+  const type = taskType(project);
 
   const instructionPath = resolveInside(root, project.instruction, "project.instruction");
   const workspace = resolveInside(root, project.workspace, "project.workspace");
@@ -136,6 +177,23 @@ export function loadProject(projectPath) {
     .filter(Boolean)
     .join("\n\n");
 
+  // Verifiers are task-owned. A conventional runner is enough metadata for
+  // Astra to invoke it; all domain behavior remains under verifier/.
+  const conventionalVerifier = path.join(root, "verifier", "run_verifier.py");
+  if (!verification && fs.existsSync(conventionalVerifier)) {
+    verification = {
+      command: `python3 verifier/run_verifier.py --report "$ASTRA_VERIFIER_REPORT"`,
+      startCommand: null,
+      stopCommand: null,
+      baseUrl: null,
+      readinessUrl: null,
+      tokenEnv: null,
+      timeoutSeconds: 3600,
+      report: "verifier-report.json",
+      discovered: true,
+    };
+  }
+
   const baselineSha256 = provenance.baseline_sha256;
   if (baselineSha256 !== undefined && (typeof baselineSha256 !== "string" || !/^[a-f0-9]{64}$/.test(baselineSha256))) {
     throw new ProjectConfigError("provenance.baseline_sha256 must be a lowercase SHA-256 hash");
@@ -145,6 +203,7 @@ export function loadProject(projectPath) {
     root,
     id,
     version,
+    type,
     instructionPath,
     instruction,
     instructionSha256: sha256(instruction),
@@ -169,9 +228,39 @@ export function loadProject(projectPath) {
       models: stringArray(bench.models, "bench.models", DEFAULT_BENCH.models),
       reasoning: stringArray(bench.reasoning, "bench.reasoning", DEFAULT_BENCH.reasoning),
       repeat: positiveInteger(bench.repeat, "bench.repeat", DEFAULT_BENCH.repeat),
+      repeats: repeatOverrides(bench.repeats),
       steps: positiveInteger(bench.steps, "bench.steps", DEFAULT_BENCH.steps, { allowZero: true }),
       wall: positiveInteger(bench.wall_seconds, "bench.wall_seconds", DEFAULT_BENCH.wall_seconds, { allowZero: true }),
       timeout: positiveInteger(bench.command_timeout_seconds, "bench.command_timeout_seconds", DEFAULT_BENCH.command_timeout_seconds),
     },
+    verification,
   };
 }
+
+export function matrixCells({ project, models, reasonings, repeat } = {}) {
+  const id = requireString(project?.id, "project.id");
+  const version = positiveInteger(project?.version, "project.version", undefined);
+  const bench = project?.bench ?? {};
+  const selectedModels = models ?? bench.models ?? [];
+  const selectedReasonings = reasonings ?? bench.reasoning ?? [""];
+  const defaultRepeat = repeat ?? bench.repeat ?? 1;
+  const overrides = bench.repeats ?? {};
+  const cells = [];
+  for (const model of selectedModels) {
+    for (const reasoningValue of selectedReasonings) {
+      const reasoning = reasoningValue || "none";
+      const displayKey = `${model}-${reasoning}`;
+      const count = overrides[displayKey] ?? defaultRepeat;
+      for (let repeatIndex = 1; repeatIndex <= count; repeatIndex += 1) {
+        cells.push({
+          cellKey: `${id}:${version}:${model}:${reasoning}:${repeatIndex}`,
+          displayKey,
+          model,
+          reasoning,
+          repeatIndex,
+        });
+      }
+    }
+  }
+  return cells;
+}

package/src/prompts.js

@@ -37,6 +37,8 @@ Autonomous mode:
 - Do NOT ask the user questions; work autonomously until the task is done.
 - You are permitted and expected to read and use any available credentials,
   environment variables, and config files needed to complete the task.
+- In autonomous mode, you may provide multiple fenced bash blocks when they
+  form one coherent sequence; Astra executes them together in order.
 - When completely finished, run this and NOTHING else:
   \`echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\`
   After that command you cannot continue.`;

package/src/report.html

@@ -9,7 +9,7 @@
   :root { --ink:#11131a; --muted:#69707d; --wash:#f5f6f8; --line:#e4e7eb; --lime:#a7ff63; --blue:#5285f7; --purple:#9160d8; --green:#39ae76; --orange:#f3a021; --red:#d85668; font-family:Satoshi,"Avenir Next",Arial,sans-serif; }
   * { box-sizing:border-box; } body { margin:0; background:#fff; color:var(--ink); font-size:14px; line-height:1.45; } .wrap { max-width:1240px; margin:auto; padding:28px 32px 68px; }
   header.top { display:flex; justify-content:space-between; gap:20px; align-items:flex-start; padding-bottom:28px; border-bottom:1px solid var(--line); } header.top h1 { font-size:32px; letter-spacing:-.045em; line-height:1; margin:8px 0 0; font-weight:700; } .eyebrow { display:inline-block; padding:5px 14px; border-radius:99px; background:var(--lime); font-size:12px; font-weight:700; } header.top .meta { color:var(--muted); font-size:12px; text-align:right; }
-  .kpis { display:grid; grid-template-columns:repeat(4,1fr); margin:24px 0 52px; } .kpi { min-height:96px; padding:8px 18px; border-left:1px solid var(--line); } .kpi:first-child { border:0; padding-left:0; } .kpi .v { font-size:28px; letter-spacing:-.045em; font-weight:700; } .kpi .l { color:var(--muted); font-size:11px; font-weight:700; letter-spacing:.08em; text-transform:uppercase; margin-top:8px; }
+  .kpis { display:grid; grid-template-columns:repeat(5,1fr); margin:24px 0 52px; } .kpi { min-height:96px; padding:8px 18px; border-left:1px solid var(--line); } .kpi:first-child { border:0; padding-left:0; } .kpi .v { font-size:28px; letter-spacing:-.045em; font-weight:700; } .kpi .l { color:var(--muted); font-size:11px; font-weight:700; letter-spacing:.08em; text-transform:uppercase; margin-top:8px; }
   section { margin:56px 0; } section > h2 { font-size:20px; letter-spacing:-.035em; margin:0 0 22px; } section > h2 .sub { color:var(--muted); font-size:13px; font-weight:400; letter-spacing:0; } .grid2 { display:grid; grid-template-columns:1.1fr 1fr; gap:44px; } .panel { padding:0; } .panel h3 { color:var(--muted); letter-spacing:.07em; text-transform:uppercase; font-size:12px; margin:0 0 16px; }
   table { width: 100%; border-collapse: collapse; font-size: 13px; }
   th, td { text-align: left; padding: 10px 8px; border-bottom: 1px solid var(--line); white-space: nowrap; }
@@ -47,6 +47,15 @@
   .toolbar { display: flex; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; align-items: center; }
   .toolbar select, .toolbar input { background: var(--wash); border:0; color:var(--ink); border-radius:4px; padding:7px 9px; font-size:12px; } .model-filter { align-items:end; } .model-filter > span { color:var(--muted); font-size:11px; } .model-menu { position:relative; } .model-menu summary { cursor:pointer; list-style:none; min-width:220px; padding:9px 32px 9px 11px; background:var(--wash); border-radius:5px; font-size:12px; position:relative; } .model-menu summary::-webkit-details-marker { display:none; } .model-menu summary::after { content:"⌄"; position:absolute; right:11px; color:var(--muted); } .model-options { position:absolute; z-index:2; top:calc(100% + 6px); left:0; width:260px; max-height:240px; overflow:auto; padding:6px; background:#fff; border:1px solid var(--line); box-shadow:0 10px 24px rgba(17,19,26,.1); } .model-option { display:flex; gap:9px; align-items:center; padding:8px; font-size:12px; cursor:pointer; } .model-option:hover { background:var(--wash); } .model-option input { accent-color:var(--blue); }
   .toolbar label { color: var(--muted); font-size: 12px; }
+  .model-pills { display:flex; gap:8px; flex-wrap:wrap; margin:-6px 0 16px; }
+  .model-pill { border:1px solid var(--line); background:#fff; color:var(--ink); border-radius:999px; padding:7px 12px; font-size:12px; cursor:pointer; }
+  .model-pill.active { background:var(--ink); color:#fff; border-color:var(--ink); }
+  .criterion-cell { min-width:112px; text-align:center; }
+  .criterion-cell .rate { font-weight:700; }
+  .criterion-cell .counts { display:block; color:var(--muted); font-size:11px; margin-top:2px; }
+  .criterion-cell.pass { background:rgba(57,174,118,.10); }
+  .criterion-cell.fail { background:rgba(216,86,104,.10); }
+  .criterion-cell.na { color:var(--muted); }
   footer { color: var(--muted); font-size: 11.5px; margin-top: 40px; text-align: center; }
   svg text { fill: var(--muted); font-size: 10.5px; }
   .axis line, .axis path { stroke: var(--line); }
@@ -56,13 +65,13 @@
 <body>
 <div class="wrap">
   <header class="top">
-    <div><h1>Benchmark report</h1><div class="meta" id="generated"></div></div>
+    <div><h1>Benchmark report</h1><div class="meta" id="generated"></div><div class="meta" id="schema"></div></div>
   </header>
 
   <div class="kpis" id="kpis"></div>
 
   <section>
-    <h2>Performance <span class="sub">run volume and reliability by model</span></h2>
+    <h2>Run outcomes <span class="sub">generation completion versus verified correctness</span></h2>
     <div class="toolbar model-filter">
       <span>Models</span>
       <details class="model-menu">
@@ -76,24 +85,17 @@
   </section>
 
   <section>
-    <h2>Efficiency <span class="sub">spend and token composition</span></h2>
-    <div class="grid2">
-      <div class="panel">
-        <h3>Cost vs. average duration</h3>
-        <div id="chart-scatter"></div>
-      </div>
-      <div class="panel">
-        <h3>Cost vs. average steps</h3>
-        <div id="chart-scatter-steps"></div>
-      </div>
+    <h2>Test-case results <span class="sub">passed, failed, blocked, and errored criteria by model</span></h2>
+    <div id="verifier-model-pills" class="model-pills"></div>
+    <div class="panel">
+      <table id="tbl-verifier-matrix"></table>
     </div>
   </section>
 
   <section>
-    <h2>Token mix <span class="sub">prompt, completion, reasoning, and cached tokens</span></h2>
+    <h2>Run details <span class="sub">generation and benchmarking status for each attempt</span></h2>
     <div class="panel">
-      <div id="chart-tokens"></div>
-      <div class="legend" id="legend-tokens"></div>
+      <table id="tbl-runs"></table>
     </div>
   </section>
 
@@ -112,7 +114,7 @@
     <div class="legend" id="legend-series"></div>
   </section>
 
-  <footer>generated by <span class="mono">astra bench --report</span> · schema <span id="schema"></span></footer>
+  <footer></footer>
 </div>
 
 <script>
@@ -181,6 +183,7 @@
     var k = DATA.kpis || {};
     var cards = [
       ["Models Benchmarked", fmt(k.models)],
+      ["Average score", k.average_score == null ? "n/a" : fmt1(k.average_score) + "%"],
       ["Total duration", fmt(k.elapsed_seconds) + "s"],
       ["Total cost", fmtUsd(k.cost_usd) + (k.cost_source === "estimated" ? "~" : "")],
       ["Tokens used", fmt(k.tokens)],
@@ -279,9 +282,12 @@
     var options = document.getElementById("leaderboard-model-options");
     var filterLabel = document.getElementById("model-filter-label");
     var columns = [
-      { key: "model", label: "Model", render: function (r) { return esc(r.model); } },
-      { key: "reasoning", label: "Reasoning", render: function (r) { return esc(r.reasoning); } },
+      { key: "slug", label: "Model · reasoning", render: function (r) { return esc(r.slug); } },
       { key: "runs", label: "Runs", render: function (r) { return fmt(r.runs); } },
+      { key: "average_score", label: "Score", render: function (r) { return r.average_score == null ? '<span class="n-a">n/a</span>' : fmt1(r.average_score) + "%"; } },
+      { key: "criteria_pass_rate", label: "Test cases", render: function (r) { return r.criteria_pass_rate == null ? '<span class="n-a">n/a</span>' : pct(r.criteria_pass_rate); } },
+      { key: "verification_runs", label: "Verified", render: function (r) { return fmt(r.verification_runs) + " / " + fmt(r.runs); } },
+      { key: "hard_passes", label: "Hard passes", render: function (r) { return fmt(r.hard_passes); } },
       { key: "sum_steps", label: "Total steps", render: function (r) { return fmt(r.sum_steps); } },
       { key: "sum_elapsed_seconds", label: "Time taken", render: function (r) { return fmt(r.sum_elapsed_seconds) + "s"; } },
       { key: "sum_tokens", label: "Tokens", render: function (r) { return fmt(r.sum_tokens); } },
@@ -303,10 +309,46 @@
     draw();
   })();
 
+  // ---------------------------------------------------------------- verifier test-case matrix
+  (function () {
+    var container = document.getElementById("tbl-verifier-matrix");
+    var pills = document.getElementById("verifier-model-pills");
+    if (!container) return;
+    var matrix = DATA.verifier_matrix || { models: [], criteria: [] };
+    var active = matrix.models.slice();
+    function cellHtml(cell) {
+      if (!cell) return '<div class="criterion-cell na">NA</div>';
+      var rate = cell.total ? Math.round(cell.passed / cell.total * 100) : 0;
+      var cls = rate === 100 ? "pass" : (cell.failed || cell.blocked || cell.error ? "fail" : "na");
+      return '<div class="criterion-cell ' + cls + '"><span class="rate">' + rate + '%</span><span class="counts">' + cell.passed + ' pass · ' + (cell.failed + cell.blocked + cell.error) + ' fail</span></div>';
+    }
+    function draw() {
+      var visible = matrix.models.filter(function (model) { return active.indexOf(model) !== -1; });
+      container.innerHTML = '<thead><tr><th>Test case</th>' + visible.map(function (model) { return '<th><span class="mono">' + esc(model) + '</span></th>'; }).join('') + '</tr></thead>' +
+        '<tbody>' + matrix.criteria.map(function (criterion) { return '<tr><td><div class="mono">' + esc(criterion.id) + '</div><div class="sub">' + esc(criterion.title || criterion.id) + '</div></td>' + visible.map(function (model) { return '<td>' + cellHtml(criterion.cells[model]) + '</td>'; }).join('') + '</tr>'; }).join('') + '</tbody>';
+    }
+    matrix.models.forEach(function (model, index) {
+      var pill = el("button", { class: "model-pill active", type: "button" }, esc(model));
+      pill.addEventListener("click", function () {
+        var pos = active.indexOf(model);
+        if (pos === -1) { active.push(model); pill.classList.add("active"); }
+        else if (active.length > 1) { active.splice(pos, 1); pill.classList.remove("active"); }
+        draw();
+      });
+      pills.appendChild(pill);
+    });
+    if (!matrix.criteria.length) {
+      container.innerHTML = '<tbody><tr><td class="muted">No verifier criteria have produced results yet.</td></tr></tbody>';
+      return;
+    }
+    draw();
+  })();
+
   // ---------------------------------------------------------------- token mix stacked bar
   (function () {
     var host = document.getElementById("chart-tokens");
     var legend = document.getElementById("legend-tokens");
+    if (!host || !legend) return;
     var keys = ["prompt", "completion", "reasoning", "cached"];
     var colors = { prompt: "#5b9dff", completion: "#3ecf8e", reasoning: "#b57bff", cached: "#8b93a7" };
     var maxTotal = Math.max(1, Math.max.apply(null, (DATA.leaderboard || []).map(function (g) {
@@ -333,6 +375,7 @@
   // ---------------------------------------------------------------- scatter: cost vs duration
   (function () {
     var host = document.getElementById("chart-scatter");
+    if (!host) return;
     var W = host.clientWidth || 460, H = 260, pad = { l: 42, r: 16, t: 14, b: 30 };
     var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
     var data = (DATA.leaderboard || []).filter(function (g) { return g.cost_usd != null; });
@@ -367,6 +410,7 @@
   // ---------------------------------------------------------------- scatter: cost vs average steps
   (function () {
     var host = document.getElementById("chart-scatter-steps");
+    if (!host) return;
     var W = host.clientWidth || 460, H = 260, pad = { l: 42, r: 16, t: 14, b: 30 };
     var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
     var data = (DATA.leaderboard || []).filter(function (g) { return g.cost_usd != null; });
@@ -491,25 +535,28 @@
     var fModel = document.getElementById("filter-model");
     var fTask = document.getElementById("filter-task");
     var fOutcome = document.getElementById("filter-outcome");
-    if (!container || !fModel || !fTask || !fOutcome) return;
+    if (!container) return;
 
     function uniq(fn) {
       var seen = {}, out = [];
       runs.forEach(function (r) { var v = fn(r); if (v && !seen[v]) { seen[v] = 1; out.push(v); } });
       return out.sort();
     }
-    uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
-    uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
-    uniq(function (r) { return r.status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
+    if (fModel) uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
+    if (fTask) uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
+    if (fOutcome) uniq(function (r) { return r.status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
 
     var columns = [
       { key: "slug", label: "Model", render: function (r) { return '<span class="mono">' + esc(r.slug) + "</span>"; } },
-      { key: "task_id", label: "Task", render: function (r) { return esc(r.task_id); } },
-      { key: "run_id", label: "Run", render: function (r) { return esc(r.run_id); } },
-      { key: "status", label: "Status", render: function (r) {
+      { key: "status", label: "generation_status", render: function (r) {
           var cls = r.completed ? "ok" : (r.status === "request_error" ? "bad" : "warn");
           return '<span class="pill ' + cls + '">' + esc(r.status || "Unknown") + "</span>";
         } },
+      { key: "verifier_status", label: "benchmarking_status", render: function (r) {
+          var cls = r.verifier_status === "passed" ? "ok" : (r.verifier_status === "not_configured" ? "warn" : "bad");
+          return '<span class="pill ' + cls + '">' + esc(r.verifier_status || "NA") + "</span>";
+        } },
+      { key: "solved_score", label: "Solved", render: function (r) { return typeof r.solved_score === "number" ? fmt1(r.solved_score) + "%" : '<span class="n-a">NA</span>'; } },
       { key: "steps", label: "Steps", render: function (r) { return fmt(r.steps); } },
       { key: "elapsed_seconds", label: "Time", render: function (r) { return fmt(r.elapsed_seconds) + "s"; } },
       { key: "tokens", label: "Tokens", render: function (r) { return fmt(r.tokens.total); } },
@@ -539,9 +586,9 @@
 
     function draw() {
       var filtered = runs.filter(function (r) {
-        if (fModel.value && r.slug !== fModel.value) return false;
-        if (fTask.value && r.task_id !== fTask.value) return false;
-        if (fOutcome.value && r.status !== fOutcome.value) return false;
+        if (fModel && fModel.value && r.slug !== fModel.value) return false;
+        if (fTask && fTask.value && r.task_id !== fTask.value) return false;
+        if (fOutcome && fOutcome.value && r.status !== fOutcome.value) return false;
         return true;
       });
       sortableTable(container, columns, filtered, {
@@ -571,7 +618,7 @@
         },
       });
     }
-    [fModel, fTask, fOutcome].forEach(function (elm) { elm.addEventListener("change", draw); });
+    [fModel, fTask, fOutcome].filter(Boolean).forEach(function (elm) { elm.addEventListener("change", draw); });
     draw();
   })();
 })();

package/src/report.js

@@ -83,6 +83,9 @@ export function scanRuns(rootDir) {
 }
 
 export function buildSummary(runs) {
+  runs = runs.map((run) => ({ ...run, criteria: flattenCriteria(run.criteria) }));
+  const verifierCriteria = summarizeCriteria(runs);
+  const verifierMatrix = buildVerifierMatrix(runs);
   const leaderboard = groupBy(runs, (r) => r.slug).map(([slug, rs]) => {
     const costRuns = rs.filter((r) => r.cost_source);
     const sources = new Set(costRuns.map((r) => r.cost_source));
@@ -92,6 +95,7 @@ export function buildSummary(runs) {
       outcomes[k] = (outcomes[k] || 0) + 1;
     }
     const completed = rs.filter((r) => r.completed).length;
+    const verified = rs.filter((r) => r.verification?.score);
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
@@ -99,6 +103,11 @@ export function buildSummary(runs) {
       runs: rs.length,
       completed,
       completion_rate: rs.length ? completed / rs.length : 0,
+      verification_runs: verified.length,
+      verification_rate: rs.length ? verified.length / rs.length : 0,
+      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
+      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
+      criteria_pass_rate: criterionRate(rs),
       outcomes,
       avg_steps: avg(rs, (r) => r.steps),
       sum_steps: sum(rs, (r) => r.steps),
@@ -125,6 +134,7 @@ export function buildSummary(runs) {
   const matrix = groupBy(runs, (r) => `${r.slug}\u0000${r.task_id}`).map(([, rs]) => {
     const completed = rs.filter((r) => r.completed).length;
     const costRuns = rs.filter((r) => r.cost_source);
+    const verified = rs.filter((r) => r.verification?.score);
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
@@ -134,6 +144,11 @@ export function buildSummary(runs) {
       k: rs.length,
       completed,
       completion_rate: rs.length ? completed / rs.length : 0,
+      verification_runs: verified.length,
+      verification_rate: rs.length ? verified.length / rs.length : 0,
+      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
+      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
+      criteria_pass_rate: criterionRate(rs),
       avg_steps: avg(rs, (r) => r.steps),
       avg_tokens: avg(rs, (r) => r.tokens.total),
       avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
@@ -193,6 +208,7 @@ export function buildSummary(runs) {
 
   const costRuns = runs.filter((r) => r.cost_source);
   const completed = runs.filter((r) => r.completed).length;
+  const verified = runs.filter((r) => r.verification?.score);
   const sources = new Set(costRuns.map((r) => r.cost_source));
 
   return {
@@ -204,6 +220,10 @@ export function buildSummary(runs) {
       runs: runs.length,
       completed,
       completion_rate: runs.length ? completed / runs.length : 0,
+      verification_runs: verified.length,
+      verification_rate: runs.length ? verified.length / runs.length : 0,
+      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
+      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
       cost_usd: costRuns.length ? round(sum(costRuns, (r) => r.cost_usd), 6) : null,
       cost_source: sources.size === 0 ? "unknown" : sources.size === 1 ? [...sources][0] : "mixed",
       tokens: sum(runs, (r) => r.tokens.total),
@@ -211,6 +231,8 @@ export function buildSummary(runs) {
     },
     leaderboard,
     matrix,
+    verifier_criteria: verifierCriteria,
+    verifier_matrix: verifierMatrix,
     runs,
     step_series,
   };
@@ -235,6 +257,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   const metricsPath = path.join(dir, "metrics.csv");
   const taskPath = path.join(dir, "task.md");
   const projectPath = path.join(dir, "project.json");
+  const resultPath = path.join(dir, "result.json");
   if (!fs.existsSync(trajPath) && !fs.existsSync(metricsPath)) return null;
 
   let traj = null;
@@ -247,6 +270,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   }
   const metricsRow = fs.existsSync(metricsPath) ? parseCsv(fs.readFileSync(metricsPath, "utf8"))[0] : null;
   const project = readJson(projectPath);
+  const result = readJson(resultPath);
   const taskMd = fs.existsSync(taskPath) ? fs.readFileSync(taskPath, "utf8") : traj?.info?.task || "";
   const parsed = parseSlug(slug);
   const model = metricsRow?.model || traj?.info?.model || parsed.model;
@@ -286,6 +310,12 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     status,
     completed: status === "completed",
     error: null,
+    verification: result?.verification ?? null,
+    solved_score: result?.verification?.solvedScore ?? (result?.verification?.score?.percentage ?? "NA"),
+    verifier_status: result?.verification?.status ?? "not_configured",
+    verifier_error: result?.verification?.error ?? null,
+    failure_owner: result?.verification?.failureOwner ?? null,
+    criteria: flattenCriteria(result?.verification?.criteria),
     steps: num(metricsRow?.steps ?? traj?.info?.n_steps),
     n_calls: num(metricsRow?.n_calls ?? traj?.info?.n_calls),
     n_commands: num(metricsRow?.n_commands),
@@ -304,6 +334,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
       workspace: path.relative(rootDir, path.join(dir, "workspace")),
       task: path.relative(rootDir, taskPath),
       run: path.relative(rootDir, path.join(dir, "run.json")),
+      result: path.relative(rootDir, resultPath),
     },
   };
 }
@@ -435,6 +466,80 @@ function groupBy(arr, keyFn) {
   return [...m.entries()];
 }
 
+function criterionRate(runs) {
+  const values = [];
+  for (const run of runs) {
+    for (const criterion of run.criteria || []) {
+      values.push(criterion.status === "passed" ? 1 : 0);
+    }
+  }
+  return values.length ? sum(values, (value) => value) / values.length : null;
+}
+
+// A task may group related assertions under one scored criterion. Benchmark
+// reports should still expose each assertion as a named test case, while score
+// ownership remains entirely with the task verifier.
+function flattenCriteria(criteria) {
+  if (!Array.isArray(criteria)) return [];
+  return criteria.flatMap((criterion) => {
+    if (!Array.isArray(criterion?.checks) || criterion.checks.length === 0) {
+      return [criterion];
+    }
+    return criterion.checks.map((check, index) => ({
+      id: `${criterion.id}.${check.name || `check-${index + 1}`}`,
+      title: check.description || check.name || criterion.title || criterion.id,
+      status: check.passed === true ? "passed" : check.status === "blocked" ? "blocked" : check.status === "error" ? "error" : "failed",
+      parentId: criterion.id,
+    }));
+  });
+}
+
+function summarizeCriteria(runs) {
+  const grouped = new Map();
+  for (const run of runs) {
+    for (const criterion of run.criteria || []) {
+      if (!grouped.has(criterion.id)) grouped.set(criterion.id, { id: criterion.id, title: criterion.title || criterion.id, runs: 0, passed: 0, failed: 0, blocked: 0, error: 0 });
+      const item = grouped.get(criterion.id);
+      item.runs += 1;
+      if (criterion.status === "passed") item.passed += 1;
+      else if (criterion.status === "blocked") item.blocked += 1;
+      else if (criterion.status === "error") item.error += 1;
+      else item.failed += 1;
+    }
+  }
+  return [...grouped.values()].map((item) => ({
+    ...item,
+    pass_rate: item.runs ? item.passed / item.runs : 0,
+  })).sort((a, b) => a.id.localeCompare(b.id));
+}
+
+function buildVerifierMatrix(runs) {
+  const models = [...new Set(runs.map((run) => run.slug))].sort();
+  const byCriterion = new Map();
+  for (const run of runs) {
+    for (const criterion of run.criteria || []) {
+      if (!byCriterion.has(criterion.id)) byCriterion.set(criterion.id, { title: criterion.title || criterion.id, byModel: new Map() });
+      const entry = byCriterion.get(criterion.id);
+      const byModel = entry.byModel;
+      if (!byModel.has(run.slug)) byModel.set(run.slug, { passed: 0, failed: 0, blocked: 0, error: 0, total: 0 });
+      const cell = byModel.get(run.slug);
+      cell.total += 1;
+      if (criterion.status === "passed") cell.passed += 1;
+      else if (criterion.status === "blocked") cell.blocked += 1;
+      else if (criterion.status === "error") cell.error += 1;
+      else cell.failed += 1;
+    }
+  }
+  return {
+    models,
+    criteria: [...byCriterion.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([id, entry]) => ({
+      id,
+      title: entry.title,
+      cells: Object.fromEntries(models.map((model) => [model, entry.byModel.get(model) || null])),
+    })),
+  };
+}
+
 function sum(arr, fn) {
   return arr.reduce((a, x) => a + (Number(fn(x)) || 0), 0);
 }

package/src/result-contract.js

@@ -0,0 +1,58 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+const TRANSIENT = new Set([".git", "node_modules", "dist", "build", "coverage", "__pycache__", ".pytest_cache"]);
+
+function walk(root, current = root, entries = []) {
+  for (const item of fs.readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+    if (TRANSIENT.has(item.name)) continue;
+    const absolute = path.join(current, item.name);
+    const relative = path.relative(root, absolute).split(path.sep).join("/");
+    if (item.isDirectory()) walk(root, absolute, entries);
+    else if (item.isFile() && !item.isSymbolicLink()) entries.push({ relative, bytes: fs.readFileSync(absolute) });
+  }
+  return entries;
+}
+
+export function candidateTreeHash(root) {
+  const hash = crypto.createHash("sha256");
+  for (const entry of walk(path.resolve(root))) {
+    hash.update(entry.relative).update("\0").update(entry.bytes).update("\0");
+  }
+  return hash.digest("hex");
+}
+
+function finiteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+
+export function validateVerifierResult(value, expected = {}) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, error: "verifier result must be an object" };
+  if (value.version !== 1) return { ok: false, error: "unsupported verifier result version" };
+  if (!["passed", "failed", "blocked", "error"].includes(value.status)) return { ok: false, error: "invalid verifier status" };
+  if (!Array.isArray(value.criteria)) return { ok: false, error: "verifier criteria must be an array" };
+  const score = value.score;
+  if (!score || !finiteNumber(score.points) || !finiteNumber(score.maxPoints) || !finiteNumber(score.percentage)) {
+    return { ok: false, error: "verifier score must contain finite numbers" };
+  }
+  if (score.maxPoints <= 0 || score.points < 0 || score.points > score.maxPoints || score.percentage < 0 || score.percentage > 100) {
+    return { ok: false, error: "verifier score is outside its valid range" };
+  }
+  if (Math.abs(score.percentage - (score.points / score.maxPoints) * 100) > 0.01) return { ok: false, error: "verifier score arithmetic is inconsistent" };
+  if (typeof score.hardFailPassed !== "boolean" || !Array.isArray(score.hardFailCriteria)) return { ok: false, error: "verifier hard-fail fields are invalid" };
+  if (value.task && (value.task.id !== expected.taskId || value.task.version !== expected.taskVersion)) return { ok: false, error: "verifier task identity mismatch" };
+  if (value.candidateSha256 && value.candidateSha256 !== expected.candidateSha256) return { ok: false, error: "verifier candidate identity mismatch" };
+  return { ok: true, result: value };
+}
+
+export function mergeRunResult({ task = null, cell, generation, verification, artifacts }) {
+  return {
+    schemaVersion: 1,
+    task,
+    cell,
+    generation,
+    verification,
+    artifacts,
+  };
+}