@hackerrank/astra-cli 0.1.0 → 0.1.1

package/src/report.js

@@ -0,0 +1,459 @@
+/**
+ * Bench report: scan run folders into a tidy `astra-bench-1` summary and a
+ * self-contained HTML dashboard (charts + tables).
+ *
+ * Artifacts written under the bench root:
+ *   summary.json   data contract (kpis, leaderboard, matrix, runs, step_series)
+ *   report.html    standalone dashboard (embeds summary.json)
+ *   <slug>/run-NN/run.json   per-attempt record with command timeline
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+const SCHEMA = "astra-bench-1";
+const RUN_SCHEMA = "astra-run-1";
+
+/** Rebuild summary.json + report.html from whatever is already on disk. */
+export function refreshReport(root) {
+  const rootDir = path.resolve(root || path.join(process.cwd(), "bench"));
+  if (!fs.existsSync(rootDir)) {
+    throw new Error(`bench root not found: ${rootDir}`);
+  }
+  const runs = scanRuns(rootDir);
+  const summary = buildSummary(runs);
+
+  for (const run of runs) {
+    const dir = path.join(rootDir, run.paths.dir);
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(path.join(dir, "run.json"), JSON.stringify(toRunDoc(run), null, 2) + "\n");
+  }
+
+  const summaryPath = path.join(rootDir, "summary.json");
+  fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2) + "\n");
+
+  const htmlPath = path.join(rootDir, "report.html");
+  fs.writeFileSync(htmlPath, renderReportHtml(summary));
+
+  return {
+    root: rootDir,
+    summary: summaryPath,
+    html: htmlPath,
+    runs: runs.length,
+    models: summary.leaderboard.length,
+    tasks: summary.kpis.tasks,
+  };
+}
+
+export function inferTaskTitle(text) {
+  const heading = String(text || "").match(/^#\s+(.+)$/m);
+  if (heading) return heading[1].replace(/`/g, "").trim();
+  const first = String(text || "").trim().split("\n")[0] || "";
+  return first.slice(0, 80) || "untitled";
+}
+
+export function inferTaskId({ taskPath, taskFile, taskText, taskMd } = {}) {
+  if (taskPath) {
+    try {
+      const resolved = path.resolve(taskPath);
+      return path.basename(fs.statSync(resolved).isDirectory() ? resolved : path.dirname(resolved));
+    } catch {
+      return path.basename(String(taskPath).replace(/\/+$/, ""));
+    }
+  }
+  if (taskFile) return path.basename(path.dirname(path.resolve(taskFile)));
+  return slugify(inferTaskTitle(taskMd || taskText)) || "inline";
+}
+
+/** Walk bench/<slug>/run-NN/ and reconstruct one record per attempt. */
+export function scanRuns(rootDir) {
+  const runs = [];
+  for (const slug of safeReaddir(rootDir)) {
+    const slugDir = path.join(rootDir, slug);
+    if (!isDir(slugDir)) continue;
+    for (const runId of safeReaddir(slugDir).sort()) {
+      if (!/^run-\d+$/.test(runId)) continue;
+      const dir = path.join(slugDir, runId);
+      const rec = readRunDir({ rootDir, slug, runId, dir });
+      if (rec) runs.push(rec);
+    }
+  }
+  runs.sort((a, b) => String(a.timestamp).localeCompare(String(b.timestamp)));
+  return runs;
+}
+
+export function buildSummary(runs) {
+  const leaderboard = groupBy(runs, (r) => r.slug).map(([slug, rs]) => {
+    const costRuns = rs.filter((r) => r.cost_source);
+    const sources = new Set(costRuns.map((r) => r.cost_source));
+    const outcomes = {};
+    for (const r of rs) {
+      const k = r.exit_status || "Unknown";
+      outcomes[k] = (outcomes[k] || 0) + 1;
+    }
+    const solved = rs.filter((r) => r.resolved).length;
+    return {
+      model: rs[0].model,
+      reasoning: rs[0].reasoning,
+      slug,
+      runs: rs.length,
+      solved,
+      solved_rate: rs.length ? solved / rs.length : 0,
+      outcomes,
+      avg_steps: avg(rs, (r) => r.steps),
+      avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
+      avg_tokens: avg(rs, (r) => r.tokens.total),
+      sum_tokens: sum(rs, (r) => r.tokens.total),
+      tokens: {
+        prompt: sum(rs, (r) => r.tokens.prompt),
+        completion: sum(rs, (r) => r.tokens.completion),
+        reasoning: sum(rs, (r) => r.tokens.reasoning),
+        cached: sum(rs, (r) => r.tokens.cached),
+        cache_write: sum(rs, (r) => r.tokens.cache_write),
+      },
+      cost_usd: costRuns.length ? sum(costRuns, (r) => r.cost_usd) : null,
+      cost_source: sources.size === 0 ? "unknown" : sources.size === 1 ? [...sources][0] : "mixed",
+      avg_n_failed_commands: avg(rs, (r) => r.n_failed_commands),
+      avg_n_format_errors: avg(rs, (r) => r.n_format_errors),
+      avg_n_retries: avg(rs, (r) => r.n_retries),
+    };
+  });
+  leaderboard.sort((a, b) => b.solved_rate - a.solved_rate || a.avg_steps - b.avg_steps);
+
+  const matrix = groupBy(runs, (r) => `${r.slug}\u0000${r.task_id}`).map(([, rs]) => {
+    const passed = rs.filter((r) => r.resolved).length;
+    const costRuns = rs.filter((r) => r.cost_source);
+    return {
+      model: rs[0].model,
+      reasoning: rs[0].reasoning,
+      slug: rs[0].slug,
+      task_id: rs[0].task_id,
+      task_title: rs[0].task_title,
+      k: rs.length,
+      passed,
+      pass_at_k: rs.length ? passed / rs.length : 0,
+      avg_steps: avg(rs, (r) => r.steps),
+      avg_tokens: avg(rs, (r) => r.tokens.total),
+      avg_elapsed_seconds: avg(rs, (r) => r.elapsed_seconds),
+      cost_usd: costRuns.length ? sum(costRuns, (r) => r.cost_usd) : null,
+    };
+  });
+
+  const buckets = new Map();
+  for (const run of runs) {
+    let cum = 0;
+    for (const step of run.timeline || []) {
+      cum += Number(step.cost_usd) || 0;
+      const key = `${run.slug}\u0000${step.step}`;
+      if (!buckets.has(key)) {
+        buckets.set(key, {
+          model: run.model,
+          reasoning: run.reasoning,
+          slug: run.slug,
+          step: step.step,
+          prompt: [],
+          completion: [],
+          reasoningTok: [],
+          cached: [],
+          cost: [],
+          cum: [],
+          fails: 0,
+          n: 0,
+        });
+      }
+      const b = buckets.get(key);
+      b.n++;
+      b.prompt.push(step.tokens?.prompt || 0);
+      b.completion.push(step.tokens?.completion || 0);
+      b.reasoningTok.push(step.tokens?.reasoning || 0);
+      b.cached.push(step.tokens?.cached || 0);
+      b.cost.push(Number(step.cost_usd) || 0);
+      b.cum.push(cum);
+      if (step.returncode != null && Number(step.returncode) !== 0) b.fails++;
+    }
+  }
+  const step_series = [...buckets.values()]
+    .map((b) => ({
+      model: b.model,
+      reasoning: b.reasoning,
+      slug: b.slug,
+      step: b.step,
+      n: b.n,
+      prompt_tokens: dist(b.prompt),
+      completion_tokens: dist(b.completion),
+      reasoning_tokens: dist(b.reasoningTok),
+      cached_tokens: dist(b.cached),
+      cost_usd: dist(b.cost),
+      cum_cost_usd: dist(b.cum),
+      fail_rate: b.n ? b.fails / b.n : 0,
+    }))
+    .sort((a, b) => a.slug.localeCompare(b.slug) || a.step - b.step);
+
+  const costRuns = runs.filter((r) => r.cost_source);
+  const solved = runs.filter((r) => r.resolved).length;
+  const sources = new Set(costRuns.map((r) => r.cost_source));
+
+  return {
+    schema: SCHEMA,
+    generated_at: new Date().toISOString(),
+    kpis: {
+      models: new Set(runs.map((r) => r.slug)).size,
+      tasks: new Set(runs.map((r) => r.task_id)).size,
+      runs: runs.length,
+      solved,
+      solved_rate: runs.length ? solved / runs.length : 0,
+      cost_usd: costRuns.length ? round(sum(costRuns, (r) => r.cost_usd), 6) : null,
+      cost_source: sources.size === 0 ? "unknown" : sources.size === 1 ? [...sources][0] : "mixed",
+      tokens: sum(runs, (r) => r.tokens.total),
+      elapsed_seconds: sum(runs, (r) => r.elapsed_seconds),
+    },
+    leaderboard,
+    matrix,
+    runs,
+    step_series,
+  };
+}
+
+export function renderReportHtml(summary) {
+  const tplPath = new URL("./report.html", import.meta.url);
+  const tpl = fs.readFileSync(tplPath, "utf8");
+  const data = JSON.stringify(summary).replace(/</g, "\\u003c");
+  if (!tpl.includes("/*__ASTRA_DATA__*/")) {
+    throw new Error("src/report.html is missing the /*__ASTRA_DATA__*/ injection marker");
+  }
+  return tpl.replace("/*__ASTRA_DATA__*/", `window.DATA = ${data};`);
+}
+
+function toRunDoc(run) {
+  return { schema: RUN_SCHEMA, ...run };
+}
+
+function readRunDir({ rootDir, slug, runId, dir }) {
+  const trajPath = path.join(dir, "trajectory.json");
+  const metricsPath = path.join(dir, "metrics.csv");
+  const taskPath = path.join(dir, "task.md");
+  if (!fs.existsSync(trajPath) && !fs.existsSync(metricsPath)) return null;
+
+  let traj = null;
+  if (fs.existsSync(trajPath)) {
+    try {
+      traj = JSON.parse(fs.readFileSync(trajPath, "utf8"));
+    } catch {
+      traj = null;
+    }
+  }
+  const metricsRow = fs.existsSync(metricsPath) ? parseCsv(fs.readFileSync(metricsPath, "utf8"))[0] : null;
+  const taskMd = fs.existsSync(taskPath) ? fs.readFileSync(taskPath, "utf8") : traj?.info?.task || "";
+  const parsed = parseSlug(slug);
+  const model = metricsRow?.model || traj?.info?.model || parsed.model;
+  const reasoning = metricsRow?.reasoning || parsed.reasoning;
+  const runNum = Number((/^run-(\d+)$/.exec(runId) || [])[1] || 0);
+  const timeline = extractTimeline(traj);
+  const tokens = {
+    prompt: num(metricsRow?.prompt_tokens ?? traj?.info?.tokens?.prompt),
+    completion: num(metricsRow?.completion_tokens ?? traj?.info?.tokens?.completion),
+    reasoning: num(metricsRow?.reasoning_tokens),
+    cached: num(metricsRow?.cached_tokens),
+    cache_write: num(metricsRow?.cache_write_tokens),
+    total: num(metricsRow?.total_tokens ?? traj?.info?.tokens?.total),
+    last_context: num(metricsRow?.last_context_tokens ?? traj?.info?.tokens?.last_context),
+  };
+  const costSource = metricsRow?.cost_source || traj?.info?.cost?.source || "";
+  const resolvedRaw = metricsRow?.resolved;
+  const resolved =
+    resolvedRaw == null
+      ? traj?.info?.exit_status === "Submitted"
+      : resolvedRaw === true || resolvedRaw === "1" || resolvedRaw === 1;
+
+  return {
+    id: `${slug}/${runId}`,
+    timestamp: metricsRow?.timestamp || traj?.created || "",
+    model,
+    reasoning,
+    slug,
+    task_id: inferTaskId({ taskMd }),
+    task_title: inferTaskTitle(taskMd),
+    run: runNum,
+    run_id: runId,
+    exit_status: metricsRow?.exit_status || traj?.info?.exit_status || "",
+    resolved: !!resolved,
+    error: null,
+    steps: num(metricsRow?.steps ?? traj?.info?.n_steps),
+    n_calls: num(metricsRow?.n_calls ?? traj?.info?.n_calls),
+    n_commands: num(metricsRow?.n_commands),
+    n_failed_commands: num(metricsRow?.n_failed_commands),
+    n_format_errors: num(metricsRow?.n_format_errors),
+    n_retries: num(metricsRow?.n_retries),
+    elapsed_seconds: num(metricsRow?.elapsed_seconds ?? traj?.info?.elapsed_seconds),
+    tokens,
+    cost_usd: costSource ? num(metricsRow?.cost_usd ?? traj?.info?.cost?.usd) : null,
+    cost_source: costSource || "",
+    timeline,
+    paths: {
+      dir: path.relative(rootDir, dir),
+      trajectory: path.relative(rootDir, trajPath),
+      workspace: path.relative(rootDir, path.join(dir, "workspace")),
+      task: path.relative(rootDir, taskPath),
+      run: path.relative(rootDir, path.join(dir, "run.json")),
+    },
+  };
+}
+
+export function extractTimeline(doc) {
+  const messages = doc?.messages || [];
+  const steps = [];
+  let step = 0;
+  for (let i = 0; i < messages.length; i++) {
+    const m = messages[i];
+    if (m.role !== "assistant") continue;
+    step++;
+    const next = messages[i + 1];
+    const command = commandFrom(m, next);
+    const usage = m.extra?.usage || {};
+    steps.push({
+      step,
+      thought: thoughtFrom(m.content),
+      command,
+      returncode: next?.extra?.returncode ?? null,
+      output_preview: preview(stripObs(next?.content), 360),
+      tokens: {
+        prompt: num(usage.prompt_tokens),
+        completion: num(usage.completion_tokens),
+        total: num(usage.total_tokens),
+        cached: num(usage.cached_tokens),
+        cache_write: num(usage.cache_write_tokens),
+        reasoning: num(usage.reasoning_tokens),
+      },
+      cost_usd: usage.cost_usd == null ? null : num(usage.cost_usd),
+      cost_kind: usage.cost_kind || "",
+    });
+  }
+  return steps;
+}
+
+function commandFrom(assistant, next) {
+  if (next?.extra?.command) return String(next.extra.command);
+  const m = /```(?:bash|sh)?\s*\n([\s\S]*?)```/.exec(assistant?.content || "");
+  return m ? m[1].trim() : "";
+}
+
+function thoughtFrom(content) {
+  return String(content || "")
+    .replace(/```[\s\S]*?```/g, "")
+    .trim()
+    .replace(/\s+/g, " ")
+    .slice(0, 240);
+}
+
+function stripObs(content) {
+  return String(content || "")
+    .replace(/<\/?returncode>/g, "")
+    .replace(/<\/?output>/g, "")
+    .trim();
+}
+
+function preview(s, n) {
+  const t = String(s || "");
+  return t.length > n ? t.slice(0, n) + "…" : t;
+}
+
+function parseSlug(slug) {
+  const m = /^(.*)-(off|none|low|medium|high|xhigh|disabled)$/.exec(slug || "");
+  if (m) return { model: m[1], reasoning: m[2] };
+  return { model: slug, reasoning: "none" };
+}
+
+function parseCsv(text) {
+  const lines = String(text || "").trim().split(/\r?\n/);
+  if (lines.length < 2) return [];
+  const headers = splitCsvLine(lines[0]);
+  return lines.slice(1).filter(Boolean).map((line) => {
+    const cells = splitCsvLine(line);
+    const o = {};
+    headers.forEach((h, i) => {
+      o[h] = cells[i] ?? "";
+    });
+    return o;
+  });
+}
+
+function splitCsvLine(line) {
+  const out = [];
+  let cur = "";
+  let q = false;
+  for (let i = 0; i < line.length; i++) {
+    const c = line[i];
+    if (q) {
+      if (c === '"' && line[i + 1] === '"') {
+        cur += '"';
+        i++;
+      } else if (c === '"') q = false;
+      else cur += c;
+    } else if (c === '"') q = true;
+    else if (c === ",") {
+      out.push(cur);
+      cur = "";
+    } else cur += c;
+  }
+  out.push(cur);
+  return out;
+}
+
+function slugify(s) {
+  return String(s || "")
+    .toLowerCase()
+    .replace(/[`'"]/g, "")
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+}
+
+function groupBy(arr, keyFn) {
+  const m = new Map();
+  for (const x of arr) {
+    const k = keyFn(x);
+    if (!m.has(k)) m.set(k, []);
+    m.get(k).push(x);
+  }
+  return [...m.entries()];
+}
+
+function sum(arr, fn) {
+  return arr.reduce((a, x) => a + (Number(fn(x)) || 0), 0);
+}
+
+function avg(arr, fn) {
+  return arr.length ? sum(arr, fn) / arr.length : 0;
+}
+
+function dist(values) {
+  const xs = values.map((v) => Number(v) || 0).sort((a, b) => a - b);
+  const mean = xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0;
+  const p50 = xs.length ? xs[Math.floor((xs.length - 1) / 2)] : 0;
+  return { mean: round(mean, 6), p50: round(p50, 6) };
+}
+
+function num(v) {
+  const n = Number(v);
+  return Number.isFinite(n) ? n : 0;
+}
+
+function round(n, dp) {
+  const f = 10 ** dp;
+  return Math.round((Number(n) || 0) * f) / f;
+}
+
+function safeReaddir(dir) {
+  try {
+    return fs.readdirSync(dir);
+  } catch {
+    return [];
+  }
+}
+
+function isDir(p) {
+  try {
+    return fs.statSync(p).isDirectory();
+  } catch {
+    return false;
+  }
+}