@hackerrank/astra-cli 0.1.5 → 0.1.7

package/src/project-bench.js

@@ -0,0 +1,154 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { matrixCells } from "./project.js";
+import { runCell as defaultRunCell, writeMetrics } from "./bench.js";
+import { candidateTreeHash, mergeRunResult } from "./result-contract.js";
+import { loadLedger, claimCell, selectWork, transitionCell } from "./ledger.js";
+import { runVerifier as defaultRunVerifier } from "./verifier-runner.js";
+import { newSessionId } from "./session.js";
+import { refreshReport } from "./report.js";
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(value, null, 2) + "\n");
+}
+
+export async function runProjectBench({ project, apiKey, baseUrl, quiet = false, resume = false, resumeSessionId = null, runCellFn = defaultRunCell, verifierFn = defaultRunVerifier, onCell = () => {} } = {}) {
+  const root = project.outputRoot;
+  const ledger = loadLedger(path.join(root, "benchmark.json"));
+  const desired = matrixCells({ project });
+  const work = selectWork(ledger, desired, Date.now(), { force: resume, sessionId: resumeSessionId });
+  const results = [];
+
+  for (const cell of work) {
+    const record = claimCell(ledger, cell);
+    const runDir = path.join(root, record.runPath);
+    const candidateDir = path.join(runDir, "workspace");
+    const reuseFrozen = ["candidate-frozen", "verifying"].includes(record.status);
+    const isResume = Boolean(record.sessionId && ["generating", "interrupted"].includes(record.status));
+    const sessionId = record.sessionId || newSessionId();
+
+    let generation;
+    if (reuseFrozen) {
+      const generationPath = path.join(runDir, "generation.json");
+      try {
+        generation = JSON.parse(fs.readFileSync(generationPath, "utf8"));
+        if (!fs.existsSync(candidateDir)) throw new Error("frozen candidate workspace is missing");
+        const actualHash = candidateTreeHash(candidateDir);
+        if (record.candidateSha256 && actualHash !== record.candidateSha256) {
+          throw new Error("frozen candidate workspace hash does not match the ledger");
+        }
+      } catch (error) {
+        const failure = { status: "error", failureOwner: "infrastructure", error: String(error?.message || error) };
+        transitionCell(ledger, cell.cellKey, "failed", { failureOwner: "infrastructure", error: failure.error });
+        results.push({ cell, generation: { status: "error" }, verification: failure });
+        continue;
+      }
+      onCell({ phase: "candidate-frozen", cell, record });
+    } else {
+      transitionCell(ledger, cell.cellKey, "generating", { sessionId, leaseAt: new Date().toISOString() });
+      onCell({ phase: "generating", cell, record });
+      try {
+        generation = await runCellFn({
+          model: cell.model,
+          reasoning: cell.reasoning,
+          apiKey,
+          baseUrl,
+          task: project.taskText,
+          taskPath: project.workspace,
+          projectMetadata: project.metadata,
+          root,
+          runDir,
+          sessionId,
+          resume: isResume,
+          cell,
+          steps: project.bench.steps,
+          wall: project.bench.wall,
+          timeout: project.bench.timeout,
+        });
+      } catch (error) {
+        const failure = { status: "error", failureOwner: "infrastructure", error: String(error?.message || error) };
+        transitionCell(ledger, cell.cellKey, "failed", { failureOwner: "infrastructure", error: failure.error });
+        results.push({ cell, generation: { status: "error" }, verification: failure });
+        continue;
+      }
+
+      const candidateSha256 = candidateTreeHash(candidateDir);
+      writeJson(path.join(runDir, "generation.json"), generation);
+      transitionCell(ledger, cell.cellKey, "candidate-frozen", { candidateSha256, sessionId });
+    }
+
+    const candidateSha256 = record.candidateSha256 || candidateTreeHash(candidateDir);
+
+    let verification = {
+      status: "not_configured",
+      failureOwner: null,
+      score: null,
+      solvedScore: "NA",
+      reason: "task verifier is not present or not configured",
+    };
+    if (project.verification && generation.status === "completed") {
+      transitionCell(ledger, cell.cellKey, "verifying", { candidateSha256 });
+      try {
+        verification = await verifierFn({
+          project,
+          cell,
+          runDir,
+          candidateDir,
+          config: project.verification,
+          expected: { taskId: project.id, taskVersion: project.version, candidateSha256 },
+        });
+      } catch (error) {
+        verification = {
+          status: "error",
+          failureOwner: "infrastructure",
+          score: null,
+          solvedScore: "NA",
+          error: String(error?.message || error),
+        };
+      }
+      if (verification.score && typeof verification.score.percentage === "number") {
+        verification.solvedScore = verification.score.percentage;
+      }
+      writeJson(path.join(runDir, "verifier.json"), verification);
+    } else if (generation.status !== "completed") {
+      verification = {
+        status: "not_run",
+        failureOwner: "candidate",
+        score: null,
+        solvedScore: "NA",
+        reason: "generation did not submit a candidate; verification was skipped",
+      };
+    }
+
+    const merged = mergeRunResult({
+      task: { id: project.id, version: project.version, type: project.type || "brownfield" },
+      cell,
+      generation: { ...generation, candidateSha256 },
+      verification,
+      artifacts: { runPath: record.runPath, candidateSha256 },
+    });
+    writeMetrics({
+      runDir,
+      root,
+      metrics: {
+        ...generation,
+        model: cell.model,
+        reasoning: cell.reasoning,
+        verification_status: verification.status,
+        verification_score: verification.score?.percentage ?? "",
+        solved_score: verification.solvedScore ?? "NA",
+        verification_hard_pass: verification.score?.hardFailPassed ?? "",
+        failure_owner: verification.failureOwner ?? "",
+      },
+    });
+    writeJson(path.join(runDir, "result.json"), merged);
+    transitionCell(ledger, cell.cellKey, "completed", { candidateSha256, resultPath: path.join(record.runPath, "result.json"), failureOwner: verification.failureOwner });
+    results.push(merged);
+    try { refreshReport(root); } catch { /* report regeneration is best-effort per cell */ }
+    onCell({ phase: "completed", cell, record, result: merged });
+  }
+  try { refreshReport(root); } catch { /* report regeneration is best-effort */ }
+  return results;
+}

package/src/project.js

@@ -19,6 +19,9 @@ const DEFAULT_BENCH = {
   command_timeout_seconds: 60,
 };
 
+const DEFAULT_VERIFICATION = null;
+const TASK_TYPES = new Set(["brownfield", "greenfield"]);
+
 function requireObject(value, name) {
   if (!value || typeof value !== "object" || Array.isArray(value)) {
     throw new ProjectConfigError(`${name} must be a TOML table`);
@@ -49,6 +52,42 @@ function stringArray(value, name, fallback) {
   return value;
 }
 
+function repeatOverrides(value) {
+  if (value === undefined) return {};
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new ProjectConfigError("bench.repeats must be a TOML table");
+  }
+  const result = {};
+  for (const [key, count] of Object.entries(value)) {
+    if (!key.trim()) throw new ProjectConfigError("bench.repeats keys must be non-empty");
+    result[key] = positiveInteger(count, `bench.repeats.${key}`, 1);
+  }
+  return result;
+}
+
+function verificationConfig(value) {
+  if (value === undefined) return DEFAULT_VERIFICATION;
+  const verification = requireObject(value, "verification");
+  return {
+    command: requireString(verification.command, "verification.command"),
+    startCommand: verification.start_command === undefined ? null : requireString(verification.start_command, "verification.start_command"),
+    stopCommand: verification.stop_command === undefined ? null : requireString(verification.stop_command, "verification.stop_command"),
+    baseUrl: verification.base_url === undefined ? null : requireString(verification.base_url, "verification.base_url"),
+    readinessUrl: verification.readiness_url === undefined ? null : requireString(verification.readiness_url, "verification.readiness_url"),
+    tokenEnv: verification.token_env === undefined ? null : requireString(verification.token_env, "verification.token_env"),
+    timeoutSeconds: positiveInteger(verification.timeout_seconds, "verification.timeout_seconds", 3600),
+    report: verification.report === undefined ? "verifier-report.json" : requireString(verification.report, "verification.report"),
+  };
+}
+
+function taskType(project) {
+  const value = project.type ?? project.profile ?? "brownfield";
+  if (typeof value !== "string" || !TASK_TYPES.has(value)) {
+    throw new ProjectConfigError("project.type must be either brownfield or greenfield");
+  }
+  return value;
+}
+
 function resolveInside(root, relative, name) {
   const candidate = path.resolve(root, requireString(relative, name));
   if (candidate !== root && !candidate.startsWith(root + path.sep)) {
@@ -114,12 +153,14 @@ export function loadProject(projectPath) {
 
   const project = requireObject(config.project, "project");
   const bench = requireObject(config.bench ?? {}, "bench");
+  let verification = verificationConfig(config.verification);
   const output = requireObject(config.output ?? {}, "output");
   const templates = requireObject(config.templates ?? {}, "templates");
   const extensionConfig = requireObject(config.extensions ?? {}, "extensions");
   const provenance = requireObject(config.provenance ?? {}, "provenance");
   const id = requireString(project.id, "project.id");
   const version = positiveInteger(project.version, "project.version", undefined);
+  const type = taskType(project);
 
   const instructionPath = resolveInside(root, project.instruction, "project.instruction");
   const workspace = resolveInside(root, project.workspace, "project.workspace");
@@ -136,6 +177,23 @@ export function loadProject(projectPath) {
     .filter(Boolean)
     .join("\n\n");
 
+  // Verifiers are task-owned. A conventional runner is enough metadata for
+  // Astra to invoke it; all domain behavior remains under verifier/.
+  const conventionalVerifier = path.join(root, "verifier", "run_verifier.py");
+  if (!verification && fs.existsSync(conventionalVerifier)) {
+    verification = {
+      command: `python3 verifier/run_verifier.py --report "$ASTRA_VERIFIER_REPORT"`,
+      startCommand: null,
+      stopCommand: null,
+      baseUrl: null,
+      readinessUrl: null,
+      tokenEnv: null,
+      timeoutSeconds: 3600,
+      report: "verifier-report.json",
+      discovered: true,
+    };
+  }
+
   const baselineSha256 = provenance.baseline_sha256;
   if (baselineSha256 !== undefined && (typeof baselineSha256 !== "string" || !/^[a-f0-9]{64}$/.test(baselineSha256))) {
     throw new ProjectConfigError("provenance.baseline_sha256 must be a lowercase SHA-256 hash");
@@ -145,6 +203,7 @@ export function loadProject(projectPath) {
     root,
     id,
     version,
+    type,
     instructionPath,
     instruction,
     instructionSha256: sha256(instruction),
@@ -169,9 +228,39 @@ export function loadProject(projectPath) {
       models: stringArray(bench.models, "bench.models", DEFAULT_BENCH.models),
       reasoning: stringArray(bench.reasoning, "bench.reasoning", DEFAULT_BENCH.reasoning),
       repeat: positiveInteger(bench.repeat, "bench.repeat", DEFAULT_BENCH.repeat),
+      repeats: repeatOverrides(bench.repeats),
       steps: positiveInteger(bench.steps, "bench.steps", DEFAULT_BENCH.steps, { allowZero: true }),
       wall: positiveInteger(bench.wall_seconds, "bench.wall_seconds", DEFAULT_BENCH.wall_seconds, { allowZero: true }),
       timeout: positiveInteger(bench.command_timeout_seconds, "bench.command_timeout_seconds", DEFAULT_BENCH.command_timeout_seconds),
     },
+    verification,
   };
 }
+
+export function matrixCells({ project, models, reasonings, repeat } = {}) {
+  const id = requireString(project?.id, "project.id");
+  const version = positiveInteger(project?.version, "project.version", undefined);
+  const bench = project?.bench ?? {};
+  const selectedModels = models ?? bench.models ?? [];
+  const selectedReasonings = reasonings ?? bench.reasoning ?? [""];
+  const defaultRepeat = repeat ?? bench.repeat ?? 1;
+  const overrides = bench.repeats ?? {};
+  const cells = [];
+  for (const model of selectedModels) {
+    for (const reasoningValue of selectedReasonings) {
+      const reasoning = reasoningValue || "none";
+      const displayKey = `${model}-${reasoning}`;
+      const count = overrides[displayKey] ?? defaultRepeat;
+      for (let repeatIndex = 1; repeatIndex <= count; repeatIndex += 1) {
+        cells.push({
+          cellKey: `${id}:${version}:${model}:${reasoning}:${repeatIndex}`,
+          displayKey,
+          model,
+          reasoning,
+          repeatIndex,
+        });
+      }
+    }
+  }
+  return cells;
+}

package/src/prompts.js

@@ -37,6 +37,8 @@ Autonomous mode:
 - Do NOT ask the user questions; work autonomously until the task is done.
 - You are permitted and expected to read and use any available credentials,
   environment variables, and config files needed to complete the task.
+- In autonomous mode, you may provide multiple fenced bash blocks when they
+  form one coherent sequence; Astra executes them together in order.
 - When completely finished, run this and NOTHING else:
   \`echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\`
   After that command you cannot continue.`;

package/src/report.html

@@ -9,7 +9,7 @@
   :root { --ink:#11131a; --muted:#69707d; --wash:#f5f6f8; --line:#e4e7eb; --lime:#a7ff63; --blue:#5285f7; --purple:#9160d8; --green:#39ae76; --orange:#f3a021; --red:#d85668; font-family:Satoshi,"Avenir Next",Arial,sans-serif; }
   * { box-sizing:border-box; } body { margin:0; background:#fff; color:var(--ink); font-size:14px; line-height:1.45; } .wrap { max-width:1240px; margin:auto; padding:28px 32px 68px; }
   header.top { display:flex; justify-content:space-between; gap:20px; align-items:flex-start; padding-bottom:28px; border-bottom:1px solid var(--line); } header.top h1 { font-size:32px; letter-spacing:-.045em; line-height:1; margin:8px 0 0; font-weight:700; } .eyebrow { display:inline-block; padding:5px 14px; border-radius:99px; background:var(--lime); font-size:12px; font-weight:700; } header.top .meta { color:var(--muted); font-size:12px; text-align:right; }
-  .kpis { display:grid; grid-template-columns:repeat(4,1fr); margin:24px 0 52px; } .kpi { min-height:96px; padding:8px 18px; border-left:1px solid var(--line); } .kpi:first-child { border:0; padding-left:0; } .kpi .v { font-size:28px; letter-spacing:-.045em; font-weight:700; } .kpi .l { color:var(--muted); font-size:11px; font-weight:700; letter-spacing:.08em; text-transform:uppercase; margin-top:8px; }
+  .kpis { display:grid; grid-template-columns:repeat(5,1fr); margin:24px 0 52px; } .kpi { min-height:96px; padding:8px 18px; border-left:1px solid var(--line); } .kpi:first-child { border:0; padding-left:0; } .kpi .v { font-size:28px; letter-spacing:-.045em; font-weight:700; } .kpi .l { color:var(--muted); font-size:11px; font-weight:700; letter-spacing:.08em; text-transform:uppercase; margin-top:8px; }
   section { margin:56px 0; } section > h2 { font-size:20px; letter-spacing:-.035em; margin:0 0 22px; } section > h2 .sub { color:var(--muted); font-size:13px; font-weight:400; letter-spacing:0; } .grid2 { display:grid; grid-template-columns:1.1fr 1fr; gap:44px; } .panel { padding:0; } .panel h3 { color:var(--muted); letter-spacing:.07em; text-transform:uppercase; font-size:12px; margin:0 0 16px; }
   table { width: 100%; border-collapse: collapse; font-size: 13px; }
   th, td { text-align: left; padding: 10px 8px; border-bottom: 1px solid var(--line); white-space: nowrap; }
@@ -47,6 +47,15 @@
   .toolbar { display: flex; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; align-items: center; }
   .toolbar select, .toolbar input { background: var(--wash); border:0; color:var(--ink); border-radius:4px; padding:7px 9px; font-size:12px; } .model-filter { align-items:end; } .model-filter > span { color:var(--muted); font-size:11px; } .model-menu { position:relative; } .model-menu summary { cursor:pointer; list-style:none; min-width:220px; padding:9px 32px 9px 11px; background:var(--wash); border-radius:5px; font-size:12px; position:relative; } .model-menu summary::-webkit-details-marker { display:none; } .model-menu summary::after { content:"⌄"; position:absolute; right:11px; color:var(--muted); } .model-options { position:absolute; z-index:2; top:calc(100% + 6px); left:0; width:260px; max-height:240px; overflow:auto; padding:6px; background:#fff; border:1px solid var(--line); box-shadow:0 10px 24px rgba(17,19,26,.1); } .model-option { display:flex; gap:9px; align-items:center; padding:8px; font-size:12px; cursor:pointer; } .model-option:hover { background:var(--wash); } .model-option input { accent-color:var(--blue); }
   .toolbar label { color: var(--muted); font-size: 12px; }
+  .model-pills { display:flex; gap:8px; flex-wrap:wrap; margin:-6px 0 16px; }
+  .model-pill { border:1px solid var(--line); background:#fff; color:var(--ink); border-radius:999px; padding:7px 12px; font-size:12px; cursor:pointer; }
+  .model-pill.active { background:var(--ink); color:#fff; border-color:var(--ink); }
+  .criterion-cell { min-width:112px; text-align:center; }
+  .criterion-cell .rate { font-weight:700; }
+  .criterion-cell .counts { display:block; color:var(--muted); font-size:11px; margin-top:2px; }
+  .criterion-cell.pass { background:rgba(57,174,118,.10); }
+  .criterion-cell.fail { background:rgba(216,86,104,.10); }
+  .criterion-cell.na { color:var(--muted); }
   footer { color: var(--muted); font-size: 11.5px; margin-top: 40px; text-align: center; }
   svg text { fill: var(--muted); font-size: 10.5px; }
   .axis line, .axis path { stroke: var(--line); }
@@ -56,13 +65,13 @@
 <body>
 <div class="wrap">
   <header class="top">
-    <div><h1>Benchmark report</h1><div class="meta" id="generated"></div></div>
+    <div><h1>Benchmark report</h1><div class="meta" id="generated"></div><div class="meta" id="schema"></div></div>
   </header>
 
   <div class="kpis" id="kpis"></div>
 
   <section>
-    <h2>Performance <span class="sub">run volume and reliability by model</span></h2>
+    <h2>Run outcomes <span class="sub">generation completion versus verified correctness</span></h2>
     <div class="toolbar model-filter">
       <span>Models</span>
       <details class="model-menu">
@@ -76,24 +85,17 @@
   </section>
 
   <section>
-    <h2>Efficiency <span class="sub">spend and token composition</span></h2>
-    <div class="grid2">
-      <div class="panel">
-        <h3>Cost vs. average duration</h3>
-        <div id="chart-scatter"></div>
-      </div>
-      <div class="panel">
-        <h3>Cost vs. average steps</h3>
-        <div id="chart-scatter-steps"></div>
-      </div>
+    <h2>Test-case results <span class="sub">passed, failed, blocked, and errored criteria by model</span></h2>
+    <div id="verifier-model-pills" class="model-pills"></div>
+    <div class="panel">
+      <table id="tbl-verifier-matrix"></table>
     </div>
   </section>
 
   <section>
-    <h2>Token mix <span class="sub">prompt, completion, reasoning, and cached tokens</span></h2>
+    <h2>Run details <span class="sub">generation and benchmarking status for each attempt</span></h2>
     <div class="panel">
-      <div id="chart-tokens"></div>
-      <div class="legend" id="legend-tokens"></div>
+      <table id="tbl-runs"></table>
     </div>
   </section>
 
@@ -112,7 +114,7 @@
     <div class="legend" id="legend-series"></div>
   </section>
 
-  <footer>generated by <span class="mono">astra bench --report</span> · schema <span id="schema"></span></footer>
+  <footer></footer>
 </div>
 
 <script>
@@ -181,6 +183,7 @@
     var k = DATA.kpis || {};
     var cards = [
       ["Models Benchmarked", fmt(k.models)],
+      ["Average score", k.average_score == null ? "n/a" : fmt1(k.average_score) + "%"],
       ["Total duration", fmt(k.elapsed_seconds) + "s"],
       ["Total cost", fmtUsd(k.cost_usd) + (k.cost_source === "estimated" ? "~" : "")],
       ["Tokens used", fmt(k.tokens)],
@@ -279,9 +282,12 @@
     var options = document.getElementById("leaderboard-model-options");
     var filterLabel = document.getElementById("model-filter-label");
     var columns = [
-      { key: "model", label: "Model", render: function (r) { return esc(r.model); } },
-      { key: "reasoning", label: "Reasoning", render: function (r) { return esc(r.reasoning); } },
+      { key: "slug", label: "Model · reasoning", render: function (r) { return esc(r.slug); } },
       { key: "runs", label: "Runs", render: function (r) { return fmt(r.runs); } },
+      { key: "average_score", label: "Score", render: function (r) { return r.average_score == null ? '<span class="n-a">n/a</span>' : fmt1(r.average_score) + "%"; } },
+      { key: "criteria_pass_rate", label: "Test cases", render: function (r) { return r.criteria_pass_rate == null ? '<span class="n-a">n/a</span>' : pct(r.criteria_pass_rate); } },
+      { key: "verification_runs", label: "Verified", render: function (r) { return fmt(r.verification_runs) + " / " + fmt(r.runs); } },
+      { key: "hard_passes", label: "Hard passes", render: function (r) { return fmt(r.hard_passes); } },
       { key: "sum_steps", label: "Total steps", render: function (r) { return fmt(r.sum_steps); } },
       { key: "sum_elapsed_seconds", label: "Time taken", render: function (r) { return fmt(r.sum_elapsed_seconds) + "s"; } },
       { key: "sum_tokens", label: "Tokens", render: function (r) { return fmt(r.sum_tokens); } },
@@ -303,10 +309,46 @@
     draw();
   })();
 
+  // ---------------------------------------------------------------- verifier test-case matrix
+  (function () {
+    var container = document.getElementById("tbl-verifier-matrix");
+    var pills = document.getElementById("verifier-model-pills");
+    if (!container) return;
+    var matrix = DATA.verifier_matrix || { models: [], criteria: [] };
+    var active = matrix.models.slice();
+    function cellHtml(cell) {
+      if (!cell) return '<div class="criterion-cell na">NA</div>';
+      var rate = cell.total ? Math.round(cell.passed / cell.total * 100) : 0;
+      var cls = rate === 100 ? "pass" : (cell.failed || cell.blocked || cell.error ? "fail" : "na");
+      return '<div class="criterion-cell ' + cls + '"><span class="rate">' + rate + '%</span><span class="counts">' + cell.passed + ' pass · ' + (cell.failed + cell.blocked + cell.error) + ' fail</span></div>';
+    }
+    function draw() {
+      var visible = matrix.models.filter(function (model) { return active.indexOf(model) !== -1; });
+      container.innerHTML = '<thead><tr><th>Test case</th>' + visible.map(function (model) { return '<th><span class="mono">' + esc(model) + '</span></th>'; }).join('') + '</tr></thead>' +
+        '<tbody>' + matrix.criteria.map(function (criterion) { return '<tr><td><div class="mono">' + esc(criterion.id) + '</div><div class="sub">' + esc(criterion.title || criterion.id) + '</div></td>' + visible.map(function (model) { return '<td>' + cellHtml(criterion.cells[model]) + '</td>'; }).join('') + '</tr>'; }).join('') + '</tbody>';
+    }
+    matrix.models.forEach(function (model, index) {
+      var pill = el("button", { class: "model-pill active", type: "button" }, esc(model));
+      pill.addEventListener("click", function () {
+        var pos = active.indexOf(model);
+        if (pos === -1) { active.push(model); pill.classList.add("active"); }
+        else if (active.length > 1) { active.splice(pos, 1); pill.classList.remove("active"); }
+        draw();
+      });
+      pills.appendChild(pill);
+    });
+    if (!matrix.criteria.length) {
+      container.innerHTML = '<tbody><tr><td class="muted">No verifier criteria have produced results yet.</td></tr></tbody>';
+      return;
+    }
+    draw();
+  })();
+
   // ---------------------------------------------------------------- token mix stacked bar
   (function () {
     var host = document.getElementById("chart-tokens");
     var legend = document.getElementById("legend-tokens");
+    if (!host || !legend) return;
     var keys = ["prompt", "completion", "reasoning", "cached"];
     var colors = { prompt: "#5b9dff", completion: "#3ecf8e", reasoning: "#b57bff", cached: "#8b93a7" };
     var maxTotal = Math.max(1, Math.max.apply(null, (DATA.leaderboard || []).map(function (g) {
@@ -333,6 +375,7 @@
   // ---------------------------------------------------------------- scatter: cost vs duration
   (function () {
     var host = document.getElementById("chart-scatter");
+    if (!host) return;
     var W = host.clientWidth || 460, H = 260, pad = { l: 42, r: 16, t: 14, b: 30 };
     var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
     var data = (DATA.leaderboard || []).filter(function (g) { return g.cost_usd != null; });
@@ -367,6 +410,7 @@
   // ---------------------------------------------------------------- scatter: cost vs average steps
   (function () {
     var host = document.getElementById("chart-scatter-steps");
+    if (!host) return;
     var W = host.clientWidth || 460, H = 260, pad = { l: 42, r: 16, t: 14, b: 30 };
     var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
     var data = (DATA.leaderboard || []).filter(function (g) { return g.cost_usd != null; });
@@ -491,25 +535,28 @@
     var fModel = document.getElementById("filter-model");
     var fTask = document.getElementById("filter-task");
     var fOutcome = document.getElementById("filter-outcome");
-    if (!container || !fModel || !fTask || !fOutcome) return;
+    if (!container) return;
 
     function uniq(fn) {
       var seen = {}, out = [];
       runs.forEach(function (r) { var v = fn(r); if (v && !seen[v]) { seen[v] = 1; out.push(v); } });
       return out.sort();
     }
-    uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
-    uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
-    uniq(function (r) { return r.status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
+    if (fModel) uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
+    if (fTask) uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
+    if (fOutcome) uniq(function (r) { return r.status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
 
     var columns = [
       { key: "slug", label: "Model", render: function (r) { return '<span class="mono">' + esc(r.slug) + "</span>"; } },
-      { key: "task_id", label: "Task", render: function (r) { return esc(r.task_id); } },
-      { key: "run_id", label: "Run", render: function (r) { return esc(r.run_id); } },
-      { key: "status", label: "Status", render: function (r) {
+      { key: "status", label: "generation_status", render: function (r) {
           var cls = r.completed ? "ok" : (r.status === "request_error" ? "bad" : "warn");
           return '<span class="pill ' + cls + '">' + esc(r.status || "Unknown") + "</span>";
         } },
+      { key: "verifier_status", label: "benchmarking_status", render: function (r) {
+          var cls = r.verifier_status === "passed" ? "ok" : (r.verifier_status === "not_configured" ? "warn" : "bad");
+          return '<span class="pill ' + cls + '">' + esc(r.verifier_status || "NA") + "</span>";
+        } },
+      { key: "solved_score", label: "Solved", render: function (r) { return typeof r.solved_score === "number" ? fmt1(r.solved_score) + "%" : '<span class="n-a">NA</span>'; } },
       { key: "steps", label: "Steps", render: function (r) { return fmt(r.steps); } },
       { key: "elapsed_seconds", label: "Time", render: function (r) { return fmt(r.elapsed_seconds) + "s"; } },
       { key: "tokens", label: "Tokens", render: function (r) { return fmt(r.tokens.total); } },
@@ -539,9 +586,9 @@
 
     function draw() {
       var filtered = runs.filter(function (r) {
-        if (fModel.value && r.slug !== fModel.value) return false;
-        if (fTask.value && r.task_id !== fTask.value) return false;
-        if (fOutcome.value && r.status !== fOutcome.value) return false;
+        if (fModel && fModel.value && r.slug !== fModel.value) return false;
+        if (fTask && fTask.value && r.task_id !== fTask.value) return false;
+        if (fOutcome && fOutcome.value && r.status !== fOutcome.value) return false;
         return true;
       });
       sortableTable(container, columns, filtered, {
@@ -571,7 +618,7 @@
         },
       });
     }
-    [fModel, fTask, fOutcome].forEach(function (elm) { elm.addEventListener("change", draw); });
+    [fModel, fTask, fOutcome].filter(Boolean).forEach(function (elm) { elm.addEventListener("change", draw); });
     draw();
   })();
 })();