@hackerrank/astra-cli 0.1.5 → 0.1.7

package/src/report.js

@@ -83,6 +83,9 @@ export function scanRuns(rootDir) {
 }
 
 export function buildSummary(runs) {
+  runs = runs.map((run) => ({ ...run, criteria: flattenCriteria(run.criteria) }));
+  const verifierCriteria = summarizeCriteria(runs);
+  const verifierMatrix = buildVerifierMatrix(runs);
   const leaderboard = groupBy(runs, (r) => r.slug).map(([slug, rs]) => {
     const costRuns = rs.filter((r) => r.cost_source);
     const sources = new Set(costRuns.map((r) => r.cost_source));
@@ -92,6 +95,7 @@ export function buildSummary(runs) {
       outcomes[k] = (outcomes[k] || 0) + 1;
     }
     const completed = rs.filter((r) => r.completed).length;
+    const verified = rs.filter((r) => r.verification?.score);
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
@@ -99,6 +103,11 @@ export function buildSummary(runs) {
       runs: rs.length,
       completed,
       completion_rate: rs.length ? completed / rs.length : 0,
+      verification_runs: verified.length,
+      verification_rate: rs.length ? verified.length / rs.length : 0,
+      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
+      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
+      criteria_pass_rate: criterionRate(rs),
       outcomes,
       avg_steps: avg(rs, (r) => r.steps),
       sum_steps: sum(rs, (r) => r.steps),
@@ -125,6 +134,7 @@ export function buildSummary(runs) {
   const matrix = groupBy(runs, (r) => `${r.slug}\u0000${r.task_id}`).map(([, rs]) => {
     const completed = rs.filter((r) => r.completed).length;
     const costRuns = rs.filter((r) => r.cost_source);
+    const verified = rs.filter((r) => r.verification?.score);
     return {
       model: rs[0].model,
       reasoning: rs[0].reasoning,
@@ -134,6 +144,11 @@ export function buildSummary(runs) {
       k: rs.length,
       completed,
       completion_rate: rs.length ? completed / rs.length : 0,
+      verification_runs: verified.length,
+      verification_rate: rs.length ? verified.length / rs.length : 0,
+      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
+      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
+      criteria_pass_rate: criterionRate(rs),
       avg_steps: avg(rs, (r) => r.steps),
       avg_tokens: avg(rs, (r) => r.tokens.total),
       avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
@@ -193,6 +208,7 @@ export function buildSummary(runs) {
 
   const costRuns = runs.filter((r) => r.cost_source);
   const completed = runs.filter((r) => r.completed).length;
+  const verified = runs.filter((r) => r.verification?.score);
   const sources = new Set(costRuns.map((r) => r.cost_source));
 
   return {
@@ -204,6 +220,10 @@ export function buildSummary(runs) {
       runs: runs.length,
       completed,
       completion_rate: runs.length ? completed / runs.length : 0,
+      verification_runs: verified.length,
+      verification_rate: runs.length ? verified.length / runs.length : 0,
+      average_score: verified.length ? avg(verified, (r) => r.verification.score.percentage) : null,
+      hard_passes: verified.filter((r) => r.verification.score.hardFailPassed === true).length,
       cost_usd: costRuns.length ? round(sum(costRuns, (r) => r.cost_usd), 6) : null,
       cost_source: sources.size === 0 ? "unknown" : sources.size === 1 ? [...sources][0] : "mixed",
       tokens: sum(runs, (r) => r.tokens.total),
@@ -211,6 +231,8 @@ export function buildSummary(runs) {
     },
     leaderboard,
     matrix,
+    verifier_criteria: verifierCriteria,
+    verifier_matrix: verifierMatrix,
     runs,
     step_series,
   };
@@ -235,6 +257,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   const metricsPath = path.join(dir, "metrics.csv");
   const taskPath = path.join(dir, "task.md");
   const projectPath = path.join(dir, "project.json");
+  const resultPath = path.join(dir, "result.json");
   if (!fs.existsSync(trajPath) && !fs.existsSync(metricsPath)) return null;
 
   let traj = null;
@@ -247,6 +270,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
   }
   const metricsRow = fs.existsSync(metricsPath) ? parseCsv(fs.readFileSync(metricsPath, "utf8"))[0] : null;
   const project = readJson(projectPath);
+  const result = readJson(resultPath);
   const taskMd = fs.existsSync(taskPath) ? fs.readFileSync(taskPath, "utf8") : traj?.info?.task || "";
   const parsed = parseSlug(slug);
   const model = metricsRow?.model || traj?.info?.model || parsed.model;
@@ -286,6 +310,12 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     status,
     completed: status === "completed",
     error: null,
+    verification: result?.verification ?? null,
+    solved_score: result?.verification?.solvedScore ?? (result?.verification?.score?.percentage ?? "NA"),
+    verifier_status: result?.verification?.status ?? "not_configured",
+    verifier_error: result?.verification?.error ?? null,
+    failure_owner: result?.verification?.failureOwner ?? null,
+    criteria: flattenCriteria(result?.verification?.criteria),
     steps: num(metricsRow?.steps ?? traj?.info?.n_steps),
     n_calls: num(metricsRow?.n_calls ?? traj?.info?.n_calls),
     n_commands: num(metricsRow?.n_commands),
@@ -296,6 +326,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     tokens,
     cost_usd: costSource ? num(metricsRow?.cost_usd ?? traj?.info?.cost?.usd) : null,
     cost_source: costSource || "",
+    resume_segments: Array.isArray(traj?.bench?.segments) ? traj.bench.segments : [],
     timeline,
     paths: {
       dir: path.relative(rootDir, dir),
@@ -303,6 +334,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
       workspace: path.relative(rootDir, path.join(dir, "workspace")),
       task: path.relative(rootDir, taskPath),
       run: path.relative(rootDir, path.join(dir, "run.json")),
+      result: path.relative(rootDir, resultPath),
     },
   };
 }
@@ -434,6 +466,80 @@ function groupBy(arr, keyFn) {
   return [...m.entries()];
 }
 
+function criterionRate(runs) {
+  const values = [];
+  for (const run of runs) {
+    for (const criterion of run.criteria || []) {
+      values.push(criterion.status === "passed" ? 1 : 0);
+    }
+  }
+  return values.length ? sum(values, (value) => value) / values.length : null;
+}
+
+// A task may group related assertions under one scored criterion. Benchmark
+// reports should still expose each assertion as a named test case, while score
+// ownership remains entirely with the task verifier.
+function flattenCriteria(criteria) {
+  if (!Array.isArray(criteria)) return [];
+  return criteria.flatMap((criterion) => {
+    if (!Array.isArray(criterion?.checks) || criterion.checks.length === 0) {
+      return [criterion];
+    }
+    return criterion.checks.map((check, index) => ({
+      id: `${criterion.id}.${check.name || `check-${index + 1}`}`,
+      title: check.description || check.name || criterion.title || criterion.id,
+      status: check.passed === true ? "passed" : check.status === "blocked" ? "blocked" : check.status === "error" ? "error" : "failed",
+      parentId: criterion.id,
+    }));
+  });
+}
+
+function summarizeCriteria(runs) {
+  const grouped = new Map();
+  for (const run of runs) {
+    for (const criterion of run.criteria || []) {
+      if (!grouped.has(criterion.id)) grouped.set(criterion.id, { id: criterion.id, title: criterion.title || criterion.id, runs: 0, passed: 0, failed: 0, blocked: 0, error: 0 });
+      const item = grouped.get(criterion.id);
+      item.runs += 1;
+      if (criterion.status === "passed") item.passed += 1;
+      else if (criterion.status === "blocked") item.blocked += 1;
+      else if (criterion.status === "error") item.error += 1;
+      else item.failed += 1;
+    }
+  }
+  return [...grouped.values()].map((item) => ({
+    ...item,
+    pass_rate: item.runs ? item.passed / item.runs : 0,
+  })).sort((a, b) => a.id.localeCompare(b.id));
+}
+
+function buildVerifierMatrix(runs) {
+  const models = [...new Set(runs.map((run) => run.slug))].sort();
+  const byCriterion = new Map();
+  for (const run of runs) {
+    for (const criterion of run.criteria || []) {
+      if (!byCriterion.has(criterion.id)) byCriterion.set(criterion.id, { title: criterion.title || criterion.id, byModel: new Map() });
+      const entry = byCriterion.get(criterion.id);
+      const byModel = entry.byModel;
+      if (!byModel.has(run.slug)) byModel.set(run.slug, { passed: 0, failed: 0, blocked: 0, error: 0, total: 0 });
+      const cell = byModel.get(run.slug);
+      cell.total += 1;
+      if (criterion.status === "passed") cell.passed += 1;
+      else if (criterion.status === "blocked") cell.blocked += 1;
+      else if (criterion.status === "error") cell.error += 1;
+      else cell.failed += 1;
+    }
+  }
+  return {
+    models,
+    criteria: [...byCriterion.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([id, entry]) => ({
+      id,
+      title: entry.title,
+      cells: Object.fromEntries(models.map((model) => [model, entry.byModel.get(model) || null])),
+    })),
+  };
+}
+
 function sum(arr, fn) {
   return arr.reduce((a, x) => a + (Number(fn(x)) || 0), 0);
 }

package/src/result-contract.js

@@ -0,0 +1,58 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+const TRANSIENT = new Set([".git", "node_modules", "dist", "build", "coverage", "__pycache__", ".pytest_cache"]);
+
+function walk(root, current = root, entries = []) {
+  for (const item of fs.readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+    if (TRANSIENT.has(item.name)) continue;
+    const absolute = path.join(current, item.name);
+    const relative = path.relative(root, absolute).split(path.sep).join("/");
+    if (item.isDirectory()) walk(root, absolute, entries);
+    else if (item.isFile() && !item.isSymbolicLink()) entries.push({ relative, bytes: fs.readFileSync(absolute) });
+  }
+  return entries;
+}
+
+export function candidateTreeHash(root) {
+  const hash = crypto.createHash("sha256");
+  for (const entry of walk(path.resolve(root))) {
+    hash.update(entry.relative).update("\0").update(entry.bytes).update("\0");
+  }
+  return hash.digest("hex");
+}
+
+function finiteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+
+export function validateVerifierResult(value, expected = {}) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return { ok: false, error: "verifier result must be an object" };
+  if (value.version !== 1) return { ok: false, error: "unsupported verifier result version" };
+  if (!["passed", "failed", "blocked", "error"].includes(value.status)) return { ok: false, error: "invalid verifier status" };
+  if (!Array.isArray(value.criteria)) return { ok: false, error: "verifier criteria must be an array" };
+  const score = value.score;
+  if (!score || !finiteNumber(score.points) || !finiteNumber(score.maxPoints) || !finiteNumber(score.percentage)) {
+    return { ok: false, error: "verifier score must contain finite numbers" };
+  }
+  if (score.maxPoints <= 0 || score.points < 0 || score.points > score.maxPoints || score.percentage < 0 || score.percentage > 100) {
+    return { ok: false, error: "verifier score is outside its valid range" };
+  }
+  if (Math.abs(score.percentage - (score.points / score.maxPoints) * 100) > 0.01) return { ok: false, error: "verifier score arithmetic is inconsistent" };
+  if (typeof score.hardFailPassed !== "boolean" || !Array.isArray(score.hardFailCriteria)) return { ok: false, error: "verifier hard-fail fields are invalid" };
+  if (value.task && (value.task.id !== expected.taskId || value.task.version !== expected.taskVersion)) return { ok: false, error: "verifier task identity mismatch" };
+  if (value.candidateSha256 && value.candidateSha256 !== expected.candidateSha256) return { ok: false, error: "verifier candidate identity mismatch" };
+  return { ok: true, result: value };
+}
+
+export function mergeRunResult({ task = null, cell, generation, verification, artifacts }) {
+  return {
+    schemaVersion: 1,
+    task,
+    cell,
+    generation,
+    verification,
+    artifacts,
+  };
+}

package/src/verifier-runner.js

@@ -0,0 +1,159 @@
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+
+import { validateVerifierResult } from "./result-contract.js";
+
+function redactOutput(value) {
+  return String(value || "")
+    .replace(/(authorization\s*:\s*bearer\s+)[^\s,;]+/gi, "$1[REDACTED]")
+    .replace(/(--token\s+)[^\s,;]+/gi, "$1[REDACTED]")
+    .slice(-2000);
+}
+
+function runCommand(command, { cwd, env, timeoutSeconds }) {
+  return new Promise((resolve) => {
+    const child = spawn(command, { cwd, env, shell: true, stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (chunk) => { stdout += chunk; });
+    child.stderr.on("data", (chunk) => { stderr += chunk; });
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGTERM");
+      setTimeout(() => child.kill("SIGKILL"), 250);
+    }, Math.max(1, Number(timeoutSeconds) || 3600) * 1000);
+    child.on("close", (code, signal) => {
+      clearTimeout(timer);
+      resolve({ code, signal, timedOut, stdout, stderr });
+    });
+    child.on("error", (error) => {
+      clearTimeout(timer);
+      resolve({ code: null, signal: null, timedOut: false, stdout, stderr: `${stderr}${error.message}` });
+    });
+  });
+}
+
+function startProcess(command, { cwd, env }) {
+  return spawn(command, { cwd, env, shell: true, stdio: ["ignore", "ignore", "ignore"] });
+}
+
+async function acquireVerifierLock(projectRoot, timeoutSeconds) {
+  const lockPath = path.join(projectRoot, ".astra", "verifier.lock");
+  fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+  const deadline = Date.now() + Math.max(1, Number(timeoutSeconds) || 3600) * 1000;
+  while (Date.now() < deadline) {
+    try {
+      const descriptor = fs.openSync(lockPath, "wx");
+      fs.writeFileSync(descriptor, `${process.pid}\n`);
+      return () => {
+        try { fs.closeSync(descriptor); } catch {}
+        try { fs.rmSync(lockPath, { force: true }); } catch {}
+      };
+    } catch (error) {
+      if (error?.code !== "EEXIST") throw error;
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+  }
+  throw new Error("timed out waiting for the task verifier lock");
+}
+
+async function waitReady(url, timeoutSeconds) {
+  const deadline = Date.now() + Math.max(1, Number(timeoutSeconds) || 30) * 1000;
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(1000) });
+      if (response.ok) return true;
+    } catch {
+      // Candidate is still starting.
+    }
+    await new Promise((resolve) => setTimeout(resolve, 250));
+  }
+  return false;
+}
+
+export async function runVerifier({ project, cell, runDir, candidateDir, config, expected = {} }) {
+  const started = Date.now();
+  fs.mkdirSync(runDir, { recursive: true });
+  const reportPath = path.resolve(runDir, config.report || "verifier.json");
+  if (reportPath !== path.resolve(runDir) && !reportPath.startsWith(`${path.resolve(runDir)}${path.sep}`)) {
+    return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds: 0, error: "verifier report must stay inside the run directory" };
+  }
+  // Never accept a report left by an earlier interrupted attempt.
+  try { fs.rmSync(reportPath, { force: true }); } catch (error) {
+    return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds: 0, error: `cannot clear previous verifier report: ${error.message}` };
+  }
+  const env = {
+    PATH: process.env.PATH,
+    HOME: process.env.HOME,
+    ...(process.env.DOCKER_HOST ? { DOCKER_HOST: process.env.DOCKER_HOST } : {}),
+    ASTRA_VERIFIER_REPORT: reportPath,
+    ASTRA_CANDIDATE_WORKSPACE: candidateDir,
+    ASTRA_TASK_ID: project.id,
+    ASTRA_TASK_VERSION: String(project.version),
+    ASTRA_TASK_TYPE: project.type || "brownfield",
+    ASTRA_CELL_KEY: cell.cellKey,
+    ASTRA_PROJECT_ROOT: project.root,
+    ...(config.baseUrl ? { SCIM_BASE_URL: config.baseUrl } : {}),
+    ...(config.tokenEnv && process.env[config.tokenEnv] ? { [config.tokenEnv]: process.env[config.tokenEnv] } : {}),
+  };
+  let candidateProcess = null;
+  let releaseLock = null;
+  try {
+    // Candidate Docker Compose files commonly claim fixed host ports. Serialize
+    // verification per task while allowing costly model generation to run in
+    // parallel safely.
+    releaseLock = await acquireVerifierLock(project.root, config.timeoutSeconds);
+    if (config.startCommand) {
+      candidateProcess = startProcess(config.startCommand, { cwd: candidateDir, env });
+      // A cold Docker build legitimately exceeds a minute. Give the candidate
+      // a bounded five-minute startup window, still capped by task timeout.
+      if (config.readinessUrl && !(await waitReady(config.readinessUrl, Math.min(config.timeoutSeconds, 300)))) {
+        return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds: (Date.now() - started) / 1000, error: "candidate readiness timed out" };
+      }
+    }
+    const commandResult = await runCommand(config.command, {
+      cwd: project.root,
+      env,
+      timeoutSeconds: config.timeoutSeconds,
+    });
+    const durationSeconds = (Date.now() - started) / 1000;
+    if (commandResult.timedOut) return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds, error: "verifier timed out" };
+    if (!fs.existsSync(reportPath)) {
+      const detail = redactOutput(commandResult.stderr);
+      return {
+        status: "error",
+        failureOwner: "infrastructure",
+        solvedScore: "NA",
+        durationSeconds,
+        error: detail ? `verifier report was not produced: ${detail}` : "verifier report was not produced",
+        exitCode: commandResult.code,
+      };
+    }
+    let raw;
+    try {
+      raw = JSON.parse(fs.readFileSync(reportPath, "utf8"));
+    } catch (error) {
+      return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds, error: `invalid verifier JSON: ${error.message}`, exitCode: commandResult.code };
+    }
+    const validation = validateVerifierResult(raw, expected);
+    if (!validation.ok) return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds, error: validation.error, exitCode: commandResult.code };
+    return {
+      status: raw.status,
+      failureOwner: raw.status === "failed" ? "candidate" : raw.status === "passed" ? null : "infrastructure",
+      score: raw.score,
+      solvedScore: raw.score?.percentage ?? "NA",
+      criteria: raw.criteria,
+      reportPath,
+      durationSeconds,
+      exitCode: commandResult.code,
+    };
+  } finally {
+    if (config.stopCommand) {
+      await runCommand(config.stopCommand, { cwd: candidateDir, env, timeoutSeconds: Math.min(Number(config.timeoutSeconds) || 30, 30) });
+    }
+    if (candidateProcess && !candidateProcess.killed) candidateProcess.kill("SIGTERM");
+    if (releaseLock) releaseLock();
+  }
+}