@hackerrank/astra-cli 0.1.30 → 0.1.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.30",
+  "version": "0.1.31",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/bench.js

@@ -51,11 +51,6 @@ export const METRIC_COLUMNS = [
   "last_context_tokens",
   "cost_usd",
   "cost_source",
-  "verification_status",
-  "verification_score",
-  "solved_score",
-  "verification_hard_pass",
-  "failure_owner",
 ];
 
 /**

package/src/ledger.js

@@ -90,7 +90,7 @@ export function transitionCell(ledger, cellKey, status, patch = {}) {
     const cell = ledger.cells[cellKey] || localCell;
     if (!cell) throw new Error(`unknown benchmark cell: ${cellKey}`);
     const next = { ...cell, ...patch, status, updatedAt: new Date().toISOString() };
-    if (["generating", "verifying"].includes(status) && patch.leaseAt === undefined) next.leaseAt = next.updatedAt;
+    if (["generating"].includes(status) && patch.leaseAt === undefined) next.leaseAt = next.updatedAt;
     ledger.cells[cellKey] = next;
     atomicWrite(ledger.path, { version: LEDGER_VERSION, cells: ledger.cells });
     return next;
@@ -103,7 +103,7 @@ export function selectWork(ledger, desiredCells, now = Date.now(), { force = fal
     if (!existing) return true;
     if (["completed", "failed"].includes(existing.status)) return false;
     if (sessionId && existing.sessionId !== sessionId) return false;
-    if (["generating", "verifying"].includes(existing.status)) {
+    if (["generating"].includes(existing.status)) {
       return force || !existing.leaseAt || now - Date.parse(existing.leaseAt) > STALE_AFTER_MS;
     }
     return !sessionId || existing.sessionId === sessionId;

package/src/project-bench.js

@@ -5,7 +5,6 @@ import { matrixCells } from "./project.js";
 import { runCell as defaultRunCell, writeMetrics } from "./bench.js";
 import { candidateTreeHash, mergeRunResult } from "./result-contract.js";
 import { loadLedger, claimCell, selectWork, transitionCell } from "./ledger.js";
-import { runVerifier as defaultRunVerifier } from "./verifier-runner.js";
 import { newSessionId } from "./session.js";
 import { refreshReport } from "./report.js";
 
@@ -14,7 +13,7 @@ function writeJson(file, value) {
   fs.writeFileSync(file, JSON.stringify(value, null, 2) + "\n");
 }
 
-export async function runProjectBench({ project, apiKey, baseUrl, quiet = false, resume = false, resumeSessionId = null, runCellFn = defaultRunCell, verifierFn = defaultRunVerifier, onCell = () => {} } = {}) {
+export async function runProjectBench({ project, apiKey, baseUrl, quiet = false, resume = false, resumeSessionId = null, runCellFn = defaultRunCell, onCell = () => {} } = {}) {
   const root = project.outputRoot;
   const ledger = loadLedger(path.join(root, "benchmark.json"));
   const desired = matrixCells({ project });
@@ -25,7 +24,7 @@ export async function runProjectBench({ project, apiKey, baseUrl, quiet = false,
     const record = claimCell(ledger, cell);
     const runDir = path.join(root, record.runPath);
     const candidateDir = path.join(runDir, "workspace");
-    const reuseFrozen = ["candidate-frozen", "verifying"].includes(record.status);
+    const reuseFrozen = ["candidate-frozen"].includes(record.status);
     const isResume = Boolean(record.sessionId && ["generating", "interrupted"].includes(record.status));
     const sessionId = record.sessionId || newSessionId();
 
@@ -42,7 +41,7 @@ export async function runProjectBench({ project, apiKey, baseUrl, quiet = false,
       } catch (error) {
         const failure = { status: "error", failureOwner: "infrastructure", error: String(error?.message || error) };
         transitionCell(ledger, cell.cellKey, "failed", { failureOwner: "infrastructure", error: failure.error });
-        results.push({ cell, generation: { status: "error" }, verification: failure });
+        results.push({ cell, generation: { status: "error" } });
         continue;
       }
       onCell({ phase: "candidate-frozen", cell, record });
@@ -70,7 +69,7 @@ export async function runProjectBench({ project, apiKey, baseUrl, quiet = false,
       } catch (error) {
         const failure = { status: "error", failureOwner: "infrastructure", error: String(error?.message || error) };
         transitionCell(ledger, cell.cellKey, "failed", { failureOwner: "infrastructure", error: failure.error });
-        results.push({ cell, generation: { status: "error" }, verification: failure });
+        results.push({ cell, generation: { status: "error" } });
         continue;
       }
 
@@ -81,53 +80,10 @@ export async function runProjectBench({ project, apiKey, baseUrl, quiet = false,
 
     const candidateSha256 = record.candidateSha256 || candidateTreeHash(candidateDir);
 
-    let verification = {
-      status: "not_configured",
-      failureOwner: null,
-      score: null,
-      solvedScore: "NA",
-      reason: "task verifier is not present or not configured",
-    };
-    if (project.verification && generation.status === "completed") {
-      transitionCell(ledger, cell.cellKey, "verifying", { candidateSha256 });
-      try {
-        verification = await verifierFn({
-          project,
-          cell,
-          runDir,
-          candidateDir,
-          config: project.verification,
-          gateway: { apiKey, baseUrl },
-          expected: { taskId: project.id, taskVersion: project.version, candidateSha256 },
-        });
-      } catch (error) {
-        verification = {
-          status: "error",
-          failureOwner: "infrastructure",
-          score: null,
-          solvedScore: "NA",
-          error: String(error?.message || error),
-        };
-      }
-      if (verification.score && typeof verification.score.percentage === "number") {
-        verification.solvedScore = verification.score.percentage;
-      }
-      writeJson(path.join(runDir, "verifier.json"), verification);
-    } else if (generation.status !== "completed") {
-      verification = {
-        status: "not_run",
-        failureOwner: "candidate",
-        score: null,
-        solvedScore: "NA",
-        reason: "generation did not submit a candidate; verification was skipped",
-      };
-    }
-
     const merged = mergeRunResult({
       task: { id: project.id, version: project.version, type: project.type || "brownfield" },
       cell,
       generation: { ...generation, candidateSha256 },
-      verification,
       artifacts: { runPath: record.runPath, candidateSha256 },
     });
     writeMetrics({
@@ -137,15 +93,10 @@ export async function runProjectBench({ project, apiKey, baseUrl, quiet = false,
         ...generation,
         model: cell.model,
         reasoning: cell.reasoning,
-        verification_status: verification.status,
-        verification_score: verification.score?.percentage ?? "",
-        solved_score: verification.solvedScore ?? "NA",
-        verification_hard_pass: verification.score?.hardFailPassed ?? "",
-        failure_owner: verification.failureOwner ?? "",
       },
     });
     writeJson(path.join(runDir, "result.json"), merged);
-    transitionCell(ledger, cell.cellKey, "completed", { candidateSha256, resultPath: path.join(record.runPath, "result.json"), failureOwner: verification.failureOwner });
+    transitionCell(ledger, cell.cellKey, "completed", { candidateSha256, resultPath: path.join(record.runPath, "result.json") });
     results.push(merged);
     try { refreshReport(root); } catch { /* report regeneration is best-effort per cell */ }
     onCell({ phase: "completed", cell, record, result: merged });

package/src/project.js

@@ -19,7 +19,6 @@ const DEFAULT_BENCH = {
   command_timeout_seconds: 60,
 };
 
-const DEFAULT_VERIFICATION = null;
 const TASK_TYPES = new Set(["brownfield", "greenfield"]);
 
 function requireObject(value, name) {
@@ -65,25 +64,6 @@ function repeatOverrides(value) {
   return result;
 }
 
-function verificationConfig(value) {
-  if (value === undefined) return DEFAULT_VERIFICATION;
-  const verification = requireObject(value, "verification");
-  return {
-    command: requireString(verification.command, "verification.command"),
-    startCommand: verification.start_command === undefined ? null : requireString(verification.start_command, "verification.start_command"),
-    stopCommand: verification.stop_command === undefined ? null : requireString(verification.stop_command, "verification.stop_command"),
-    baseUrl: verification.base_url === undefined ? null : requireString(verification.base_url, "verification.base_url"),
-    readinessUrl: verification.readiness_url === undefined ? null : requireString(verification.readiness_url, "verification.readiness_url"),
-    tokenEnv: verification.token_env === undefined ? null : requireString(verification.token_env, "verification.token_env"),
-    evaluatorModel: verification.evaluator_model === undefined ? null : requireString(verification.evaluator_model, "verification.evaluator_model"),
-    evaluatorReasoning: verification.evaluator_reasoning === undefined ? null : requireString(verification.evaluator_reasoning, "verification.evaluator_reasoning"),
-    // Zero means no verifier deadline. Tasks with agentic verification choose
-    // completion over an arbitrary wall-clock cutoff.
-    timeoutSeconds: positiveInteger(verification.timeout_seconds, "verification.timeout_seconds", 0, { allowZero: true }),
-    report: verification.report === undefined ? "verifier-report.json" : requireString(verification.report, "verification.report"),
-  };
-}
-
 function taskType(project) {
   const value = project.type ?? project.profile ?? "brownfield";
   if (typeof value !== "string" || !TASK_TYPES.has(value)) {
@@ -157,7 +137,6 @@ export function loadProject(projectPath) {
 
   const project = requireObject(config.project, "project");
   const bench = requireObject(config.bench ?? {}, "bench");
-  let verification = verificationConfig(config.verification);
   const output = requireObject(config.output ?? {}, "output");
   const templates = requireObject(config.templates ?? {}, "templates");
   const extensionConfig = requireObject(config.extensions ?? {}, "extensions");
@@ -181,23 +160,6 @@ export function loadProject(projectPath) {
     .filter(Boolean)
     .join("\n\n");
 
-  // Verifiers are task-owned. A conventional runner is enough metadata for
-  // Astra to invoke it; all domain behavior remains under verifier/.
-  const conventionalVerifier = path.join(root, "verifier", "run_verifier.py");
-  if (!verification && fs.existsSync(conventionalVerifier)) {
-    verification = {
-      command: `python3 verifier/run_verifier.py --report "$ASTRA_VERIFIER_REPORT"`,
-      startCommand: null,
-      stopCommand: null,
-      baseUrl: null,
-      readinessUrl: null,
-      tokenEnv: null,
-      timeoutSeconds: 3600,
-      report: "verifier-report.json",
-      discovered: true,
-    };
-  }
-
   const baselineSha256 = provenance.baseline_sha256;
   if (baselineSha256 !== undefined && (typeof baselineSha256 !== "string" || !/^[a-f0-9]{64}$/.test(baselineSha256))) {
     throw new ProjectConfigError("provenance.baseline_sha256 must be a lowercase SHA-256 hash");
@@ -237,7 +199,6 @@ export function loadProject(projectPath) {
       wall: positiveInteger(bench.wall_seconds, "bench.wall_seconds", DEFAULT_BENCH.wall_seconds, { allowZero: true }),
       timeout: positiveInteger(bench.command_timeout_seconds, "bench.command_timeout_seconds", DEFAULT_BENCH.command_timeout_seconds),
     },
-    verification,
   };
 }
 

package/src/report.html

@@ -74,10 +74,6 @@
 
   <div class="kpis" id="kpis"></div>
 
-  <section id="section-score-distribution">
-    <h2>Score distribution</h2>
-    <div class="score-chart-scroll"><div id="chart-score-distribution" role="img" aria-label="Verified score by model and reasoning configuration"></div></div>
-  </section>
 
   <section>
     <h2>Run outcomes</h2>
@@ -92,13 +88,6 @@
     </div>
   </section>
 
-  <section>
-    <h2>Test-case results</h2>
-    <div class="toolbar"><label for="verifier-model-select">Model</label><select id="verifier-model-select"></select></div>
-    <div class="panel">
-      <table id="tbl-verifier-matrix"></table>
-    </div>
-  </section>
 
   <section id="section-series">
     <h2>Trajectory</h2>
@@ -197,8 +186,7 @@
     var k = DATA.kpis || {};
     var cards = [
       ["Solutions", fmt(k.models)],
-      ["Average score", k.average_score == null ? "n/a" : fmt1(k.average_score) + "%"],
-      ["Average duration", fmtDuration(k.runs ? k.elapsed_seconds / k.runs : 0)],
+            ["Average duration", fmtDuration(k.runs ? k.elapsed_seconds / k.runs : 0)],
       ["Total cost", fmtUsd(k.cost_usd) + (k.cost_source === "estimated" ? "~" : "")],
       ["Tokens used", fmtTokens(k.tokens)],
     ];
@@ -208,37 +196,6 @@
     });
   })();
 
-  // ---------------------------------------------------------------- score distribution
-  (function renderScoreDistribution() {
-    var host = document.getElementById("chart-score-distribution");
-    if (!host) return;
-    var rows = (DATA.leaderboard || []).map(function (row) {
-      var verified = typeof row.average_score === "number";
-      return { slug: row.slug, score: verified ? Math.max(0, Math.min(100, row.average_score)) : null, verified: verified };
-    }).sort(function (a, b) {
-      if (a.verified !== b.verified) return a.verified ? -1 : 1;
-      return (b.score || 0) - (a.score || 0) || a.slug.localeCompare(b.slug);
-    });
-    if (!rows.length) {
-      document.getElementById("section-score-distribution").style.display = "none";
-      return;
-    }
-    host.className = "score-chart";
-    host.style.minWidth = Math.max(760, rows.length * 45) + "px";
-    [["top", "100"], ["mid", "50"], ["zero", "0"]].forEach(function (axis) {
-      host.appendChild(el("span", { class: "score-axis " + axis[0] }, axis[1]));
-    });
-    rows.forEach(function (row) {
-      var item = el("div", { class: "score-item" });
-      var bar = el("div", { class: "score-bar", title: row.slug + ": " + (row.verified ? fmt1(row.score) + "%" : "n/a") });
-      if (row.verified) bar.style.height = Math.max(2, row.score) + "%";
-      bar.appendChild(el("span", { class: "score-value" }, row.verified ? fmt1(row.score) + "%" : "n/a"));
-      item.appendChild(bar);
-      item.appendChild(el("span", { class: "score-label" }, esc(row.slug)));
-      host.appendChild(item);
-    });
-  })();
-
   // ---------------------------------------------------------------- sortable table helper
   function sortableTable(container, columns, rows, opts) {
     opts = opts || {};
@@ -333,7 +290,6 @@
     var columns = [
       { key: "slug", label: "Model · reasoning", render: function (r) { return esc(r.slug); } },
       { key: "runs", label: "Runs", render: function (r) { return fmt(r.runs); } },
-      { key: "average_score", label: "Score", render: function (r) { return r.average_score == null ? '<span class="n-a">n/a</span>' : fmt1(r.average_score) + "%"; } },
       { key: "sum_steps", label: "Total steps", render: function (r) { return fmt(r.sum_steps); } },
       { key: "sum_elapsed_seconds", label: "Time taken", render: function (r) { return fmtDuration(r.sum_elapsed_seconds); } },
       { key: "sum_tokens", label: "Tokens", render: function (r) { return fmtTokens(r.sum_tokens); } },
@@ -349,70 +305,12 @@
       var selected = Array.prototype.slice.call(options.querySelectorAll("input:checked")).map(function (o) { return o.value; });
       var rows = selected.length ? (DATA.leaderboard || []).filter(function (r) { return selected.indexOf(r.slug) !== -1; }) : DATA.leaderboard || [];
       filterLabel.textContent = selected.length ? selected.length + " model" + (selected.length === 1 ? "" : "s") + " selected" : "All models";
-      sortableTable(container, columns, rows, { defaultKey: "average_score" });
+      sortableTable(container, columns, rows, { defaultKey: "runs" });
     }
     options.addEventListener("change", draw);
     draw();
   })();
 
-  // ---------------------------------------------------------------- verifier test-case matrix
-  (function () {
-    var container = document.getElementById("tbl-verifier-matrix");
-    var selector = document.getElementById("verifier-model-select");
-    if (!container) return;
-    var matrix = DATA.verifier_matrix || { models: [], criteria: [] };
-    // Prefer a model with actual verifier evidence. Alphabetical-first often
-    // selects an unscored gateway failure and makes a healthy report appear
-    // empty on first open.
-    var selected = matrix.models.find(function (model) {
-      return matrix.criteria.some(function (criterion) { return criterion.cells[model]; });
-    }) || matrix.models[0] || "";
-    var GROUP_TITLES = {
-      "scim.discovery-auth": "Discovery & authentication",
-      "scim.read-existing": "Reading existing resources",
-      "scim.create-user": "User creation",
-      "scim.user-lifecycle": "User lifecycle",
-      "scim.group-lifecycle": "Group lifecycle",
-      "scim.membership": "Group membership",
-      "scim.errors-atomicity": "Errors & atomicity"
-    };
-    function groupTitle(parentId) {
-      if (GROUP_TITLES[parentId]) return GROUP_TITLES[parentId];
-      return String(parentId || "Other checks")
-        .replace(/[._-]+/g, " ")
-        .replace(/\b\w/g, function (letter) { return letter.toUpperCase(); });
-    }
-    function draw() {
-      var groups = {};
-      matrix.criteria.forEach(function (criterion) {
-        var parentId = criterion.parentId || criterion.id;
-        (groups[parentId] || (groups[parentId] = [])).push(criterion);
-      });
-      var bodies = Object.keys(groups).sort().map(function (parentId) {
-        var rows = groups[parentId].map(function (criterion) {
-          var cell = criterion.cells[selected];
-          if (!cell) return '<tr><td>' + esc(criterion.title || criterion.id) + '</td><td class="n-a">NA</td><td><span class="result-badge unscored">Unscored</span></td></tr>';
-          var max = cell.maxPoints || 0, earned = cell.earnedPoints || 0;
-          var passed = cell.passed === cell.total;
-          return '<tr><td>' + esc(criterion.title || criterion.id) + '</td><td>' + fmt1(earned) + ' / ' + fmt1(max) + '</td><td><span class="result-badge ' + (passed ? 'passed' : 'failed') + '">' + (passed ? 'Passed' : 'Failed') + '</span></td></tr>';
-        }).join('');
-        return '<tbody><tr class="criterion-group"><th colspan="3">' + esc(groupTitle(parentId)) + '</th></tr>' + rows + '</tbody>';
-      }).join('');
-      container.innerHTML = '<thead><tr><th>Test case</th><th>Points</th><th>Result</th></tr></thead>' + bodies;
-    }
-    matrix.models.forEach(function (model) {
-      var option = el("option", { value: model }, esc(model));
-      if (model === selected) option.selected = true;
-      selector.appendChild(option);
-    });
-    selector.addEventListener("change", function () { selected = selector.value; draw(); });
-    if (!matrix.criteria.length) {
-      container.innerHTML = '<tbody><tr><td class="muted">No verifier criteria have produced results yet.</td></tr></tbody>';
-      return;
-    }
-    draw();
-  })();
-
   // ---------------------------------------------------------------- token mix stacked bar
   (function () {
     var host = document.getElementById("chart-tokens");

package/src/report.js

@@ -102,9 +102,6 @@ function latestTaskRuns(runs) {
 }
 
 export function buildSummary(runs) {
-  runs = runs.map((run) => ({ ...run, criteria: flattenCriteria(run.criteria) }));
-  const verifierCriteria = summarizeCriteria(runs);
-  const verifierMatrix = buildVerifierMatrix(runs);
   const leaderboard = groupBy(runs, (r) => r.slug).map(([slug, rs]) => {
     const costRuns = rs.filter((r) => r.cost_source);
     const sources = new Set(costRuns.map((r) => r.cost_source));
@@ -114,7 +111,6 @@ export function buildSummary(runs) {
       outcomes[k] = (outcomes[k] || 0) + 1;
     }
     const completed = rs.filter((r) => r.completed).length;
-    const verified = rs.filter((r) => r.verification?.score);
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
@@ -122,11 +118,6 @@ export function buildSummary(runs) {
       runs: rs.length,
       completed,
       completion_rate: rs.length ? completed / rs.length : 0,
-      verification_runs: verified.length,
-      verification_rate: rs.length ? verified.length / rs.length : 0,
-      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
-      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
-      criteria_pass_rate: criterionRate(rs),
       outcomes,
       avg_steps: avg(rs, (r) => r.steps),
       sum_steps: sum(rs, (r) => r.steps),
@@ -148,18 +139,14 @@ export function buildSummary(runs) {
       avg_n_retries: avg(rs, (r) => r.n_retries),
     };
   });
-  leaderboard.sort((a, b) => {
-    const aScored = a.average_score != null;
-    const bScored = b.average_score != null;
-    if (aScored !== bScored) return aScored ? -1 : 1;
-    return (b.average_score ?? 0) - (a.average_score ?? 0)
-      || a.slug.localeCompare(b.slug);
-  });
+  leaderboard.sort((a, b) =>
+    (b.completion_rate ?? 0) - (a.completion_rate ?? 0)
+      || a.slug.localeCompare(b.slug)
+  );
 
   const matrix = groupBy(runs, (r) => `${r.slug}\u0000${r.task_id}`).map(([, rs]) => {
     const completed = rs.filter((r) => r.completed).length;
     const costRuns = rs.filter((r) => r.cost_source);
-    const verified = rs.filter((r) => r.verification?.score);
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
@@ -169,11 +156,6 @@ export function buildSummary(runs) {
       k: rs.length,
       completed,
       completion_rate: rs.length ? completed / rs.length : 0,
-      verification_runs: verified.length,
-      verification_rate: rs.length ? verified.length / rs.length : 0,
-      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
-      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
-      criteria_pass_rate: criterionRate(rs),
       avg_steps: avg(rs, (r) => r.steps),
       avg_tokens: avg(rs, (r) => r.tokens.total),
       avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
@@ -233,7 +215,6 @@ export function buildSummary(runs) {
 
   const costRuns = runs.filter((r) => r.cost_source);
   const completed = runs.filter((r) => r.completed).length;
-  const verified = runs.filter((r) => r.verification?.score);
   const sources = new Set(costRuns.map((r) => r.cost_source));
 
   return {
@@ -245,10 +226,6 @@ export function buildSummary(runs) {
       runs: runs.length,
       completed,
       completion_rate: runs.length ? completed / runs.length : 0,
-      verification_runs: verified.length,
-      verification_rate: runs.length ? verified.length / runs.length : 0,
-      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
-      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
       cost_usd: costRuns.length ? round(sum(costRuns, (r) => r.cost_usd), 6) : null,
       cost_source: sources.size === 0 ? "unknown" : sources.size === 1 ? [...sources][0] : "mixed",
       tokens: sum(runs, (r) => r.tokens.total),
@@ -256,8 +233,6 @@ export function buildSummary(runs) {
     },
     leaderboard,
     matrix,
-    verifier_criteria: verifierCriteria,
-    verifier_matrix: verifierMatrix,
     runs,
     step_series,
   };
@@ -347,12 +322,6 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     status,
     completed: status === "completed",
     error: null,
-    verification: result?.verification ?? null,
-    solved_score: result?.verification?.solvedScore ?? (result?.verification?.score?.percentage ?? "NA"),
-    verifier_status: result?.verification?.status ?? "not_configured",
-    verifier_error: result?.verification?.error ?? null,
-    failure_owner: result?.verification?.failureOwner ?? null,
-    criteria: flattenCriteria(result?.verification?.criteria),
     steps: num(metricsRow?.steps ?? traj?.info?.n_steps),
     n_calls: num(metricsRow?.n_calls ?? traj?.info?.n_calls),
     n_commands: num(metricsRow?.n_commands),
@@ -503,90 +472,6 @@ function groupBy(arr, keyFn) {
   return [...m.entries()];
 }
 
-function criterionRate(runs) {
-  const values = [];
-  for (const run of runs) {
-    for (const criterion of run.criteria || []) {
-      values.push(criterion.status === "passed" ? 1 : 0);
-    }
-  }
-  return values.length ? sum(values, (value) => value) / values.length : null;
-}
-
-// A task may group related assertions under one scored criterion. Benchmark
-// reports should still expose each assertion as a named test case, while score
-// ownership remains entirely with the task verifier.
-function flattenCriteria(criteria) {
-  if (!Array.isArray(criteria)) return [];
-  return criteria.flatMap((criterion) => {
-    if (!Array.isArray(criterion?.checks) || criterion.checks.length === 0) {
-      return [criterion];
-    }
-    const leafWeight = Math.max(0, Number(criterion.weight) || 0) / criterion.checks.length;
-    return criterion.checks.map((check, index) => ({
-      id: `${criterion.id}.${check.name || `check-${index + 1}`}`,
-      title: check.description || check.name || criterion.title || criterion.id,
-      status: check.passed === true ? "passed" : check.status === "blocked" ? "blocked" : check.status === "error" ? "error" : "failed",
-      parentId: criterion.id,
-      weight: leafWeight,
-    }));
-  });
-}
-
-function summarizeCriteria(runs) {
-  const grouped = new Map();
-  for (const run of runs) {
-    for (const criterion of run.criteria || []) {
-      if (!grouped.has(criterion.id)) grouped.set(criterion.id, { id: criterion.id, title: criterion.title || criterion.id, runs: 0, passed: 0, failed: 0, blocked: 0, error: 0 });
-      const item = grouped.get(criterion.id);
-      item.runs += 1;
-      if (criterion.status === "passed") item.passed += 1;
-      else if (criterion.status === "blocked") item.blocked += 1;
-      else if (criterion.status === "error") item.error += 1;
-      else item.failed += 1;
-    }
-  }
-  return [...grouped.values()].map((item) => ({
-    ...item,
-    pass_rate: item.runs ? item.passed / item.runs : 0,
-  })).sort((a, b) => a.id.localeCompare(b.id));
-}
-
-function buildVerifierMatrix(runs) {
-  const models = [...new Set(runs.map((run) => run.slug))].sort();
-  const byCriterion = new Map();
-  for (const run of runs) {
-    for (const criterion of run.criteria || []) {
-      if (!byCriterion.has(criterion.id)) byCriterion.set(criterion.id, {
-        title: criterion.title || criterion.id,
-        parentId: criterion.parentId || criterion.id,
-        byModel: new Map(),
-      });
-      const entry = byCriterion.get(criterion.id);
-      const byModel = entry.byModel;
-      if (!byModel.has(run.slug)) byModel.set(run.slug, { passed: 0, failed: 0, blocked: 0, error: 0, total: 0, maxPoints: 0, earnedPoints: 0 });
-      const cell = byModel.get(run.slug);
-      cell.total += 1;
-      const weight = Math.max(0, Number(criterion.weight) || 0);
-      cell.maxPoints += weight;
-      if (criterion.status === "passed") cell.earnedPoints += weight;
-      if (criterion.status === "passed") cell.passed += 1;
-      else if (criterion.status === "blocked") cell.blocked += 1;
-      else if (criterion.status === "error") cell.error += 1;
-      else cell.failed += 1;
-    }
-  }
-  return {
-    models,
-    criteria: [...byCriterion.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([id, entry]) => ({
-      id,
-      title: entry.title,
-      parentId: entry.parentId,
-      cells: Object.fromEntries(models.map((model) => [model, entry.byModel.get(model) || null])),
-    })),
-  };
-}
-
 function sum(arr, fn) {
   return arr.reduce((a, x) => a + (Number(fn(x)) || 0), 0);
 }

package/src/result-contract.js

@@ -27,32 +27,12 @@ function finiteNumber(value) {
   return typeof value === "number" && Number.isFinite(value);
 }
 
-export function validateVerifierResult(value, expected = {}) {
-  if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, error: "verifier result must be an object" };
-  if (value.version !== 1) return { ok: false, error: "unsupported verifier result version" };
-  if (!["passed", "failed", "blocked", "error"].includes(value.status)) return { ok: false, error: "invalid verifier status" };
-  if (!Array.isArray(value.criteria)) return { ok: false, error: "verifier criteria must be an array" };
-  const score = value.score;
-  if (!score || !finiteNumber(score.points) || !finiteNumber(score.maxPoints) || !finiteNumber(score.percentage)) {
-    return { ok: false, error: "verifier score must contain finite numbers" };
-  }
-  if (score.maxPoints <= 0 || score.points < 0 || score.points > score.maxPoints || score.percentage < 0 || score.percentage > 100) {
-    return { ok: false, error: "verifier score is outside its valid range" };
-  }
-  if (Math.abs(score.percentage - (score.points / score.maxPoints) * 100) > 0.01) return { ok: false, error: "verifier score arithmetic is inconsistent" };
-  if (typeof score.hardFailPassed !== "boolean" || !Array.isArray(score.hardFailCriteria)) return { ok: false, error: "verifier hard-fail fields are invalid" };
-  if (value.task && (value.task.id !== expected.taskId || value.task.version !== expected.taskVersion)) return { ok: false, error: "verifier task identity mismatch" };
-  if (value.candidateSha256 && value.candidateSha256 !== expected.candidateSha256) return { ok: false, error: "verifier candidate identity mismatch" };
-  return { ok: true, result: value };
-}
-
-export function mergeRunResult({ task = null, cell, generation, verification, artifacts }) {
+export function mergeRunResult({ task = null, cell, generation, artifacts }) {
   return {
     schemaVersion: 1,
     task,
     cell,
     generation,
-    verification,
     artifacts,
   };
 }

package/src/verifier-runner.js

@@ -1,222 +0,0 @@
-import fs from "node:fs";
-import path from "node:path";
-import { spawn } from "node:child_process";
-
-import { validateVerifierResult } from "./result-contract.js";
-
-function redactOutput(value) {
-  return String(value || "")
-    .replace(/(authorization\s*:\s*bearer\s+)[^\s,;]+/gi, "$1[REDACTED]")
-    .replace(/(--token\s+)[^\s,;]+/gi, "$1[REDACTED]")
-    .slice(-2000);
-}
-
-function runCommand(command, { cwd, env, timeoutSeconds }) {
-  return new Promise((resolve) => {
-    const child = spawn(command, { cwd, env, shell: true, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] });
-    let stdout = "";
-    let stderr = "";
-    child.stdout.on("data", (chunk) => { stdout += chunk; });
-    child.stderr.on("data", (chunk) => { stderr += chunk; });
-    let timedOut = false;
-    const limit = Number(timeoutSeconds);
-    const timer = limit > 0 ? setTimeout(() => {
-      timedOut = true;
-      stopProcessTree(child, "SIGTERM");
-      setTimeout(() => stopProcessTree(child, "SIGKILL"), 250);
-    }, limit * 1000) : null;
-    child.on("close", (code, signal) => {
-      if (timer) clearTimeout(timer);
-      resolve({ code, signal, timedOut, stdout, stderr });
-    });
-    child.on("error", (error) => {
-      if (timer) clearTimeout(timer);
-      resolve({ code: null, signal: null, timedOut: false, stdout, stderr: `${stderr}${error.message}` });
-    });
-  });
-}
-
-function stopProcessTree(child, signal) {
-  try {
-    if (process.platform !== "win32" && child.pid) process.kill(-child.pid, signal);
-    else child.kill(signal);
-  } catch {
-    // The command may already have exited between the timer and the signal.
-  }
-}
-
-function startProcess(command, { cwd, env }) {
-  const child = spawn(command, { cwd, env, shell: true, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"] });
-  let output = "";
-  child.stdout.on("data", (chunk) => { output += chunk; });
-  child.stderr.on("data", (chunk) => { output += chunk; });
-  child.startupOutput = () => redactOutput(output);
-  return child;
-}
-
-async function acquireVerifierLock(projectRoot, timeoutSeconds) {
-  const lockPath = path.join(projectRoot, ".astra", "verifier.lock");
-  fs.mkdirSync(path.dirname(lockPath), { recursive: true });
-  const limit = Number(timeoutSeconds);
-  const deadline = limit > 0 ? Date.now() + limit * 1000 : Infinity;
-  while (Date.now() < deadline) {
-    try {
-      const descriptor = fs.openSync(lockPath, "wx");
-      fs.writeFileSync(descriptor, `${process.pid}\n`);
-      return () => {
-        try { fs.closeSync(descriptor); } catch {}
-        try { fs.rmSync(lockPath, { force: true }); } catch {}
-      };
-    } catch (error) {
-      if (error?.code !== "EEXIST") throw error;
-      // A killed benchmark worker cannot run its finally block. Reclaim only
-      // a lock whose recorded owner is no longer alive; active workers keep
-      // exclusive ownership of fixed Docker ports.
-      try {
-        const owner = Number(fs.readFileSync(lockPath, "utf8").trim());
-        process.kill(owner, 0);
-      } catch (ownerError) {
-        if (ownerError?.code === "ESRCH" || !Number.isInteger(Number(fs.readFileSync(lockPath, "utf8").trim()))) {
-          try { fs.rmSync(lockPath, { force: true }); } catch {}
-          continue;
-        }
-      }
-      await new Promise((resolve) => setTimeout(resolve, 100));
-    }
-  }
-  throw new Error("timed out waiting for the task verifier lock");
-}
-
-async function waitReady(url, timeoutSeconds, candidateProcess) {
-  const limit = Number(timeoutSeconds);
-  const deadline = limit > 0 ? Date.now() + limit * 1000 : Infinity;
-  while (Date.now() < deadline) {
-    if (candidateProcess && candidateProcess.exitCode !== null) {
-      return { ready: false, exited: true, output: candidateProcess.startupOutput?.() || "" };
-    }
-    try {
-      const response = await fetch(url, { signal: AbortSignal.timeout(1000) });
-      if (response.ok) return { ready: true, exited: false, output: "" };
-    } catch {
-      // Candidate is still starting.
-    }
-    await new Promise((resolve) => setTimeout(resolve, 250));
-  }
-  return { ready: false, exited: false, output: candidateProcess?.startupOutput?.() || "" };
-}
-
-export async function runVerifier({ project, cell, runDir, candidateDir, config, expected = {}, gateway = {} }) {
-  const started = Date.now();
-  fs.mkdirSync(runDir, { recursive: true });
-  const reportPath = path.resolve(runDir, config.report || "verifier.json");
-  if (reportPath !== path.resolve(runDir) && !reportPath.startsWith(`${path.resolve(runDir)}${path.sep}`)) {
-    return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds: 0, error: "verifier report must stay inside the run directory" };
-  }
-  // Never accept a report left by an earlier interrupted attempt.
-  try { fs.rmSync(reportPath, { force: true }); } catch (error) {
-    return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds: 0, error: `cannot clear previous verifier report: ${error.message}` };
-  }
-  const env = {
-    PATH: process.env.PATH,
-    HOME: process.env.HOME,
-    ...(process.env.DOCKER_HOST ? { DOCKER_HOST: process.env.DOCKER_HOST } : {}),
-    ASTRA_VERIFIER_REPORT: reportPath,
-    ASTRA_CANDIDATE_WORKSPACE: candidateDir,
-    ASTRA_TASK_ID: project.id,
-    ASTRA_TASK_VERSION: String(project.version),
-    ASTRA_TASK_TYPE: project.type || "brownfield",
-    ASTRA_CELL_KEY: cell.cellKey,
-    ASTRA_PROJECT_ROOT: project.root,
-    // The task verifier invokes ASTRA's evaluator bridge for dynamic discovery
-    // and source judgment. Keep this explicit and task-scoped: the candidate
-    // command environment is separately scrubbed by the verifier.
-    ASTRA_EVALUATOR_BIN: new URL("./evaluator-cli.js", import.meta.url).pathname,
-    ...(gateway.apiKey ? { ASTRA_GATEWAY_API_KEY: gateway.apiKey } : {}),
-    ...(gateway.baseUrl ? { ASTRA_GATEWAY_BASE_URL: gateway.baseUrl } : {}),
-    ASTRA_EVALUATOR_MODEL: config.evaluatorModel || cell.model,
-    ASTRA_EVALUATOR_REASONING: config.evaluatorReasoning || cell.reasoning || "high",
-    ...(config.baseUrl ? { SCIM_BASE_URL: config.baseUrl } : {}),
-    ...(config.tokenEnv && process.env[config.tokenEnv] ? { [config.tokenEnv]: process.env[config.tokenEnv] } : {}),
-  };
-  let candidateProcess = null;
-  let releaseLock = null;
-  try {
-    // Candidate Docker Compose files commonly claim fixed host ports. Serialize
-    // verification per task while allowing costly model generation to run in
-    // parallel safely.
-    releaseLock = await acquireVerifierLock(project.root, config.timeoutSeconds);
-    if (config.startCommand) {
-      candidateProcess = startProcess(config.startCommand, { cwd: candidateDir, env });
-      // A cold Docker build legitimately exceeds a minute. Give the candidate
-      // a bounded five-minute startup window, still capped by task timeout.
-      if (config.readinessUrl) {
-        const readiness = await waitReady(config.readinessUrl, config.timeoutSeconds > 0 ? Math.min(config.timeoutSeconds, 300) : 0, candidateProcess);
-        if (!readiness.ready) {
-          const durationSeconds = (Date.now() - started) / 1000;
-          const detail = readiness.output;
-          if (readiness.exited) {
-            const portCollision = /EADDRINUSE|address already in use|port is already allocated/i.test(detail);
-            return {
-              status: "error",
-              failureOwner: portCollision ? "infrastructure" : "candidate",
-              solvedScore: "NA",
-              durationSeconds,
-              error: detail
-                ? `${portCollision ? "candidate startup blocked by host-port collision" : "candidate startup failed"}: ${detail}`
-                : "candidate startup process exited before readiness",
-            };
-          }
-          return {
-            status: "error",
-            failureOwner: "infrastructure",
-            solvedScore: "NA",
-            durationSeconds,
-            error: detail ? `candidate readiness timed out: ${detail}` : "candidate readiness timed out",
-          };
-        }
-      }
-    }
-    const commandResult = await runCommand(config.command, {
-      cwd: project.root,
-      env,
-      timeoutSeconds: config.timeoutSeconds,
-    });
-    const durationSeconds = (Date.now() - started) / 1000;
-    if (commandResult.timedOut) return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds, error: "verifier timed out" };
-    if (!fs.existsSync(reportPath)) {
-      const detail = redactOutput(commandResult.stderr);
-      return {
-        status: "error",
-        failureOwner: "infrastructure",
-        solvedScore: "NA",
-        durationSeconds,
-        error: detail ? `verifier report was not produced: ${detail}` : "verifier report was not produced",
-        exitCode: commandResult.code,
-      };
-    }
-    let raw;
-    try {
-      raw = JSON.parse(fs.readFileSync(reportPath, "utf8"));
-    } catch (error) {
-      return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds, error: `invalid verifier JSON: ${error.message}`, exitCode: commandResult.code };
-    }
-    const validation = validateVerifierResult(raw, expected);
-    if (!validation.ok) return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds, error: validation.error, exitCode: commandResult.code };
-    return {
-      status: raw.status,
-      failureOwner: raw.status === "failed" || raw.status === "blocked" ? "candidate" : raw.status === "passed" ? null : "infrastructure",
-      score: raw.score,
-      solvedScore: raw.score?.percentage ?? "NA",
-      criteria: raw.criteria,
-      reportPath,
-      durationSeconds,
-      exitCode: commandResult.code,
-    };
-  } finally {
-    if (config.stopCommand) {
-      await runCommand(config.stopCommand, { cwd: candidateDir, env, timeoutSeconds: config.timeoutSeconds > 0 ? Math.min(Number(config.timeoutSeconds), 30) : 0 });
-    }
-    if (candidateProcess && !candidateProcess.killed) stopProcessTree(candidateProcess, "SIGTERM");
-    if (releaseLock) releaseLock();
-  }
-}