@hackerrank/astra-cli 0.1.1 → 0.1.2

package/README.md

@@ -126,6 +126,29 @@ price entry shows **`n/a`** (never `$0`), and estimated costs (native routes) ar
 marked `~`. `--push` requires an `s3://` URI and the `aws` CLI on PATH; if the
 upload fails the local tarball is kept.
 
+### HTML report
+
+Every matrix run (and every single `-p`/`-t`/`-f` bench run) rebuilds
+`bench/summary.json` and a self-contained `bench/report.html` dashboard —
+no server, no build step, no external JS: open it straight from `file://` or
+from inside the `bench-*.tgz` tarball. It has a leaderboard, a model × task
+pass-rate heatmap, cost/token charts, per-step trend lines, and an
+expandable command timeline for every run.
+
+Regenerate it on demand (e.g. after manually editing/pruning `bench/`) with:
+
+```bash
+astra --report                      # rebuild ./bench/{summary.json,report.html}
+astra --report --bench-root ./other-bench
+```
+
+`summary.json` follows the `astra-bench-1` schema: `kpis` (totals), a tidy
+`leaderboard[]` (one row per model×reasoning), `matrix[]` (model×reasoning×task
+pass@k), `runs[]` (every attempt, with a compact per-step `timeline`), and
+`step_series[]` (token/cost distributions by step, for the trend charts). Each
+run folder also gets a standalone `run.json` for drill-down without loading
+the full `trajectory.json`.
+
 ### Sessions
 
 Every run (either mode) is saved under `~/.astra/sessions/<id>.json`.
@@ -153,6 +176,8 @@ credits — that's a quota issue, not a bug.
     --bench-root <dir>  Root folder for bench runs (default: ./bench)
     --tar               After a matrix, write a bench/ tarball locally
     --push <s3-uri>     After a matrix, tar bench/ and `aws s3 cp` to the URI
+    --report            Rebuild bench/summary.json + bench/report.html from
+                        whatever runs already exist on disk, then exit
 -C, --cwd <path>        Working directory for commands (default: cwd)
 -o, --output <path>     Also write trajectory JSON here (autonomous mode)
 -s, --steps <n>         Step limit (default: 40)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/cli.js

@@ -32,6 +32,9 @@
  *                           s3://astra-bench-results/<timestamp>/
  *       --push-uri <s3-uri> Override the S3 push destination
  *       --tar               After a matrix, write a bench/ tarball locally
+ *       --report            Rebuild bench/summary.json + bench/report.html
+ *                           from whatever runs already exist on disk, then
+ *                           exit (no model/API key needed).
  *   -t, --task <text>       Task text -> bench mode (run to completion)
  *   -f, --task-file <path>  Read task text from a file -> bench mode
  *   -p, --path <dir>        Task directory (copied into an isolated bench
@@ -94,6 +97,7 @@ import {
   defaultPushUri,
   benchRoot,
 } from "./bench.js";
+import { refreshReport } from "./report.js";
 
 /** Map a reasoning level to gateway modelKwargs. Empty for off/none/unset. */
 function reasoningKwargs(level) {
@@ -115,6 +119,7 @@ function parseArgs(argv) {
     "--push": "push",
     "--push-uri": "push-uri",
     "--tar": "tar",
+    "--report": "report",
     "--bench-root": "bench-root",
     "-C": "cwd", "--cwd": "cwd",
     "-o": "output", "--output": "output",
@@ -130,7 +135,7 @@ function parseArgs(argv) {
     "-y": "yolo", "--yolo": "yolo",
     "-h": "help", "--help": "help",
   };
-  const flags = new Set(["quiet", "yolo", "help", "sessions", "tar", "push"]);
+  const flags = new Set(["quiet", "yolo", "help", "sessions", "tar", "push", "report"]);
   for (let i = 2; i < argv.length; i++) {
     const key = alias[argv[i]];
     if (!key) { console.error(`Unknown option: ${argv[i]}`); process.exit(2); }
@@ -154,6 +159,18 @@ async function main() {
 
   if (args.sessions) { printSessions(); process.exit(0); }
   if (args.help) { console.log(HELP); process.exit(0); }
+  if (args.report) {
+    try {
+      const res = refreshReport(args["bench-root"]);
+      console.error(`\x1b[1m[astra] report · ${res.runs} run(s) · ${res.models} model(s) · ${res.tasks} task(s)\x1b[0m`);
+      console.error(`\x1b[1m  summary\x1b[0m  ${res.summary}`);
+      console.error(`\x1b[1m  html    \x1b[0m  ${res.html}`);
+    } catch (err) {
+      console.error(`\x1b[31m[astra] report failed: ${err.message}\x1b[0m`);
+      process.exit(1);
+    }
+    process.exit(0);
+  }
 
   // Load a session to resume (if any) to infer defaults.
   let resumeDoc = null;
@@ -384,6 +401,12 @@ async function main() {
       console.error(`\x1b[2m[astra] metrics -> ${paths.runMetrics}\x1b[0m`);
       console.error(`\x1b[2m[astra] index   -> ${paths.index}\x1b[0m`);
     }
+    try {
+      const report = refreshReport(benchRun.root);
+      if (!quiet) console.error(`\x1b[2m[astra] report  -> ${report.html}\x1b[0m`);
+    } catch {
+      // Non-fatal: report regeneration is best-effort for single runs.
+    }
   }
 
   if (!quiet) {
@@ -408,9 +431,50 @@ async function main() {
     console.error(`\x1b[2m[astra] session -> ${sessionId}\x1b[0m`);
   }
   if (result.submission) console.log(result.submission);
+
+  // For a single-run bench, tar + push the bench/ folder if requested.
+  // (Matrix runs handle this inside runBenchMatrix.)
+  if (benchRun) {
+    const links = [];
+    archiveAndPush(args, benchRun.root, links);
+    if (links.length) {
+      const w = Math.max(...links.map(([k]) => k.length));
+      console.error("");
+      for (const [k, v] of links) console.error(`\x1b[1m  ${k.padEnd(w)}\x1b[0m  ${v}`);
+    }
+  }
+
   process.exit(result.exit_status === "Submitted" ? 0 : 1);
 }
 
+/**
+ * Tar the bench/ folder and (optionally) upload it to S3. Mutates `links`
+ * with the resulting artifact paths/URLs and prints any failures.
+ * Works for both single-run and matrix bench runs.
+ */
+function archiveAndPush(args, root, links) {
+  if (!(args.tar || args.push)) return;
+  try {
+    const tarball = packBench({ root });
+    links.push(["tarball", tarball]);
+    if (args.push) {
+      // Bare --push uses the default bucket + timestamp prefix; --push-uri
+      // overrides the destination.
+      const dest = args["push-uri"] || defaultPushUri();
+      try {
+        const pushed = pushToS3(tarball, dest);
+        links.push(["s3 uri", pushed.uri]);
+        links.push(["s3 url", pushed.url]);
+      } catch (err) {
+        console.error(`\x1b[31m[astra] S3 push failed: ${err.message}\x1b[0m`);
+        console.error(`\x1b[33m[astra] tarball kept locally: ${tarball}\x1b[0m`);
+      }
+    }
+  } catch (err) {
+    console.error(`\x1b[31m[astra] tar failed: ${err.message}\x1b[0m`);
+  }
+}
+
 /**
  * Run a multi-model / multi-reasoning / repeated bench matrix, print a
  * leaderboard, optionally tar + upload the bench/ folder, then exit.
@@ -463,29 +527,18 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
   // Collect the important artifact locations to print together at the end.
   const links = [["metrics", indexPath]];
 
-  // Tar + optional S3 push of the whole bench/ folder.
-  if (args.tar || args.push) {
-    try {
-      const tarball = packBench({ root });
-      links.push(["tarball", tarball]);
-      if (args.push) {
-        // Bare --push uses the default bucket + timestamp prefix; --push-uri
-        // overrides the destination.
-        const dest = args["push-uri"] || defaultPushUri();
-        try {
-          const pushed = pushToS3(tarball, dest);
-          links.push(["s3 uri", pushed.uri]);
-          links.push(["s3 url", pushed.url]);
-        } catch (err) {
-          console.error(`\x1b[31m[astra] S3 push failed: ${err.message}\x1b[0m`);
-          console.error(`\x1b[33m[astra] tarball kept locally: ${tarball}\x1b[0m`);
-        }
-      }
-    } catch (err) {
-      console.error(`\x1b[31m[astra] tar failed: ${err.message}\x1b[0m`);
-    }
+  // Rebuild the dashboard (summary.json + report.html) from every run on
+  // disk under this root, including runs from earlier matrix invocations.
+  try {
+    const report = refreshReport(root);
+    links.push(["report", report.html]);
+  } catch (err) {
+    console.error(`\x1b[33m[astra] report generation skipped: ${err.message}\x1b[0m`);
   }
 
+  // Tar + optional S3 push of the whole bench/ folder.
+  archiveAndPush(args, root, links);
+
   const solved = rows.filter((r) => r.resolved).length;
   console.error(`\n\x1b[1m[astra] matrix done · ${solved}/${rows.length} resolved\x1b[0m`);
 

package/src/model.js

@@ -67,6 +67,9 @@ export class GatewayModel {
     this.costSource = null; // "reported" | "estimated" | "mixed" | null
     // Preferred token-cap parameter; swapped automatically on 400 if needed.
     this._tokenParam = "max_tokens";
+    // Optional AbortSignal; when set and aborted, in-flight requests are
+    // cancelled (used to force-quit a long model call on a second Ctrl+C).
+    this.signal = null;
     if (!this.apiKey) {
       throw new Error(
         "GatewayModel: no API key. Set ASTRA_GATEWAY_API_KEY or configure ~/.astra/config.json."
@@ -100,6 +103,7 @@ export class GatewayModel {
             Authorization: `Bearer ${this.apiKey}`,
           },
           body: JSON.stringify(body),
+          signal: this.signal || undefined,
         });
 
         if (!res.ok) {
@@ -228,8 +232,10 @@ export class GatewayModel {
 
 function isRetryable(err) {
   // Never retry classified terminal errors (auth/quota/bad-request) or context
-  // overflow — those won't succeed on retry.
+  // overflow — those won't succeed on retry. Also never retry an aborted
+  // request (user force-quit via Ctrl+C).
   if (err instanceof GatewayError || err instanceof ContextWindowError) return false;
+  if (err?.name === "AbortError") return false;
   const msg = String(err?.message || err);
   return /HTTP 5\d\d|HTTP 429|ECONNRESET|ETIMEDOUT|fetch failed|network/i.test(msg);
 }

package/src/repl.js

@@ -134,7 +134,9 @@ function turnSummary({ turn, upTok, downTok, costUsd, costKind, seconds }) {
  */
 function footerLines(agent, model, yolo, mode = "agent") {
   const cols = process.stdout.columns || 100;
-  const rule = "─".repeat(cols);
+  // Leave one cell of slack so a full-width rule never wraps to a new row
+  // (which pushes the whole footer up and looks broken on some terminals).
+  const rule = "─".repeat(Math.max(1, cols - 1));
 
   const reasoning = model.modelKwargs?.reasoning_effort || model.modelKwargs?.reasoning?.effort || "off";
   const modeTag = mode === "bench" ? C.yellow("◆ bench") : C.cyan("◆ agent");
@@ -154,12 +156,13 @@ function footerLines(agent, model, yolo, mode = "agent") {
 
   return [
     C.dim(rule),
-    modeTag + "  " + C.cyan(repoLabel()) + "    " + C.dim(modelBits.join("  ")) + "    " + C.dim("opt+←→ model · opt+↑↓ reason · tab: mode"),
+    modeTag + "  " + C.cyan(repoLabel()) + "    " + C.dim(modelBits.join("  ")),
     C.dim(usage),
+    C.dim("opt+left/right: model  ·  opt+up/down: reasoning  ·  shift+tab: mode"),
   ];
 }
 
-const FOOTER_LINES = 3; // divider + repo/model + usage
+const FOOTER_LINES = 4; // divider + repo/model + usage + shortcuts
 const PROMPT_PREFIX = C.green("› ");        // live input row (minimal)
 const HISTORY_PREFIX = C.green("you › ");   // echoed into transcript/history
 const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
@@ -336,6 +339,11 @@ class Screen {
     this.drawPrompt();
   }
 
+  /** True while the agent is working (a turn is in progress). */
+  isBusy() {
+    return this._busy || this._busyResume;
+  }
+
   /**
    * Resolve with the next full line the user types. An optional prompt label
    * replaces the default "you › " (used by the approval gate so the question
@@ -486,8 +494,12 @@ class Screen {
         this.buf = this.buf.slice(0, -1);
         this.drawPrompt();
       } else if (ch === "\x03") {                  // Ctrl+C
+        // If a prompt is pending (idle), resolve it so the loop can react.
+        // Also always notify the interrupt hook so a *running* agent turn can
+        // be aborted even when no readLine is pending.
         const r = this.resolve; this.resolve = null;
         if (r) r("__SIGINT__");
+        if (this.onInterrupt) this.onInterrupt();
       } else if (ch === "\x04") {                  // Ctrl+D
         const r = this.resolve; this.resolve = null;
         if (r) r("__EOF__");
@@ -576,6 +588,31 @@ async function runStickyRepl(agent, model, yolo, intro = [], { runBench = null }
   // extra needed here, but expose a hook for symmetry / future use).
   screen.onClearInput = () => {};
 
+  // Ctrl+C handling. In raw mode Ctrl+C does NOT raise SIGINT — it arrives as
+  // a byte, so we handle it explicitly. The screen's key handler already
+  // resolves any pending readLine with "__SIGINT__" (idle case); here we only
+  // deal with the *busy* case where no readLine is pending:
+  //   - first Ctrl+C while working -> request an abort after the current step
+  //   - second Ctrl+C while working -> force an immediate quit
+  const interrupt = { aborted: false, quit: false };
+  screen.onInterrupt = () => {
+    if (!screen.isBusy()) {
+      // Idle: the readLine resolver was already fired with "__SIGINT__", which
+      // breaks the loop and quits gracefully. Nothing more to do here.
+      return;
+    }
+    if (interrupt.aborted) {
+      // Second Ctrl+C while working -> force quit: abort the in-flight model
+      // request so a long call unblocks immediately.
+      interrupt.quit = true;
+      log(C.red("[astra] force quitting…"));
+      if (model._abort) { try { model._abort.abort(); } catch {} }
+      return;
+    }
+    interrupt.aborted = true;
+    log(C.yellow("[astra] interrupting… (press Ctrl+C again to force quit)"));
+  };
+
   // Approval gate: show the command, then ask for y/n/a on the prompt row.
   agent.confirm = async (command) => {
     log(C.yellow("$ " + command));
@@ -621,6 +658,7 @@ async function runStickyRepl(agent, model, yolo, intro = [], { runBench = null }
           mode = "agent"; refresh(); continue;
         }
         screen.startBusy("benching");
+        interrupt.aborted = false;
         try {
           await runBench(line, log);
         } catch (err) {
@@ -628,18 +666,33 @@ async function runStickyRepl(agent, model, yolo, intro = [], { runBench = null }
         } finally {
           screen.stopBusy();
         }
+        if (interrupt.quit) break;
         mode = "agent";
         refresh();
         continue;
       }
 
       agent.addUserMessage(line);
+      interrupt.aborted = false; // reset per turn
+      // Fresh AbortController for this turn so a force-quit can cancel an
+      // in-flight model request. Cleared in `finally`.
+      model._abort = new AbortController();
+      model.signal = model._abort.signal;
       screen.startBusy("working");
       try {
-        await driveUntilChat(agent, log);
+        await driveUntilChat(agent, log, interrupt);
+      } catch (err) {
+        if (err?.name === "AbortError" || interrupt.quit) {
+          // Force-quit path: swallow the abort and fall through to break.
+        } else {
+          throw err;
+        }
       } finally {
         screen.stopBusy();
+        model.signal = null;
+        model._abort = null;
       }
+      if (interrupt.quit) break; // force-quit requested via a second Ctrl+C
       const after = usageSnapshot(model);
       log(
         turnSummary({
@@ -733,10 +786,16 @@ function usageSnapshot(model) {
 
 /**
  * Take turns until the agent produces a chat reply (or terminates). Commands
- * run in between; their output goes back to the model automatically.
+ * run in between; their output goes back to the model automatically. If an
+ * `interrupt` signal is provided, the loop stops cleanly between turns when
+ * `interrupt.aborted` becomes true (Ctrl+C while working).
  */
-async function driveUntilChat(agent, log) {
+async function driveUntilChat(agent, log, interrupt = null) {
   while (true) {
+    if (interrupt && interrupt.aborted) {
+      log(C.yellow("[astra] interrupted — returning to prompt."));
+      return;
+    }
     let turn;
     try {
       turn = await agent.runTurn();
@@ -748,6 +807,10 @@ async function driveUntilChat(agent, log) {
       throw err;
     }
 
+    if (interrupt && interrupt.aborted) {
+      log(C.yellow("[astra] interrupted — returning to prompt."));
+      return;
+    }
     if (turn.kind === "chat") {
       log("\n" + C.cyan("astra › ") + turn.content.trim());
       return;

package/src/report.html

@@ -0,0 +1,569 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>astra bench report</title>
+<style>
+  :root {
+    --bg: #0f1115;
+    --panel: #171a21;
+    --panel-2: #1d212b;
+    --border: #262b36;
+    --text: #e6e8ee;
+    --muted: #8b93a7;
+    --accent: #5b9dff;
+    --good: #3ecf8e;
+    --bad: #ef5a6f;
+    --warn: #e8b339;
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
+  }
+  * { box-sizing: border-box; }
+  body { margin: 0; background: var(--bg); color: var(--text); font-size: 14px; line-height: 1.45; }
+  a { color: var(--accent); }
+  .wrap { max-width: 1180px; margin: 0 auto; padding: 28px 20px 64px; }
+  header.top { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 4px; flex-wrap: wrap; gap: 8px; }
+  header.top h1 { font-size: 20px; margin: 0; font-weight: 650; letter-spacing: .2px; }
+  header.top .meta { color: var(--muted); font-size: 12.5px; }
+  .kpis { display: grid; grid-template-columns: repeat(auto-fit, minmax(130px, 1fr)); gap: 10px; margin: 20px 0 28px; }
+  .kpi { background: var(--panel); border: 1px solid var(--border); border-radius: 10px; padding: 12px 14px; }
+  .kpi .v { font-size: 22px; font-weight: 700; }
+  .kpi .l { color: var(--muted); font-size: 11.5px; text-transform: uppercase; letter-spacing: .06em; margin-top: 2px; }
+  section { margin: 34px 0; }
+  section > h2 { font-size: 15px; margin: 0 0 12px; font-weight: 650; color: var(--text); display: flex; align-items: center; gap: 8px; }
+  section > h2 .sub { color: var(--muted); font-weight: 400; font-size: 12px; }
+  .grid2 { display: grid; grid-template-columns: 1.1fr 1fr; gap: 16px; }
+  .grid3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; }
+  @media (max-width: 900px) { .grid2, .grid3 { grid-template-columns: 1fr; } }
+  .panel { background: var(--panel); border: 1px solid var(--border); border-radius: 10px; padding: 14px 16px; }
+  .panel h3 { margin: 0 0 10px; font-size: 12.5px; color: var(--muted); font-weight: 600; text-transform: uppercase; letter-spacing: .05em; }
+  table { width: 100%; border-collapse: collapse; font-size: 13px; }
+  th, td { text-align: left; padding: 7px 10px; border-bottom: 1px solid var(--border); white-space: nowrap; }
+  th { color: var(--muted); font-weight: 600; font-size: 11.5px; text-transform: uppercase; letter-spacing: .03em; cursor: pointer; user-select: none; }
+  th:hover { color: var(--text); }
+  th .arrow { opacity: .5; font-size: 10px; margin-left: 3px; }
+  tbody tr:hover { background: rgba(255,255,255,0.02); }
+  tbody tr.run-row { cursor: pointer; }
+  .pill { display: inline-block; padding: 1px 8px; border-radius: 100px; font-size: 11px; font-weight: 600; }
+  .pill.ok { background: rgba(62,207,142,.15); color: var(--good); }
+  .pill.bad { background: rgba(239,90,111,.15); color: var(--bad); }
+  .pill.warn { background: rgba(232,179,57,.15); color: var(--warn); }
+  .mono { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12px; }
+  .muted { color: var(--muted); }
+  .bar-row { display: flex; align-items: center; gap: 8px; margin: 6px 0; }
+  .bar-row .lbl { width: 150px; flex: 0 0 150px; font-size: 12px; overflow: hidden; text-overflow: ellipsis; }
+  .bar-track { flex: 1; height: 14px; background: var(--panel-2); border-radius: 6px; overflow: hidden; display: flex; }
+  .bar-seg { height: 100%; }
+  .bar-val { width: 64px; flex: 0 0 64px; text-align: right; font-size: 12px; font-variant-numeric: tabular-nums; }
+  .legend { display: flex; gap: 14px; flex-wrap: wrap; margin-top: 10px; font-size: 11.5px; color: var(--muted); }
+  .legend .sw { display: inline-block; width: 9px; height: 9px; border-radius: 2px; margin-right: 5px; vertical-align: -1px; }
+  .heat-table td { text-align: center; font-variant-numeric: tabular-nums; }
+  .heat-cell { display: inline-block; min-width: 44px; padding: 3px 6px; border-radius: 5px; font-size: 12px; }
+  details.run-detail { margin: 0; }
+  details.run-detail > summary { display: none; }
+  .timeline { margin: 0 0 18px; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
+  .timeline .step { padding: 10px 14px; border-bottom: 1px solid var(--border); }
+  .timeline .step:last-child { border-bottom: none; }
+  .timeline .step .head { display: flex; justify-content: space-between; gap: 10px; align-items: baseline; margin-bottom: 4px; }
+  .timeline .step .n { color: var(--accent); font-weight: 700; font-size: 12px; }
+  .timeline .step .stats { color: var(--muted); font-size: 11.5px; }
+  .timeline .step .thought { color: var(--muted); font-size: 12.5px; margin-bottom: 6px; }
+  .timeline .step pre { background: var(--panel-2); border-radius: 6px; padding: 8px 10px; margin: 4px 0; font-size: 12px; overflow-x: auto; white-space: pre-wrap; word-break: break-word; }
+  .rc0 { color: var(--good); } .rcN { color: var(--bad); }
+  .toolbar { display: flex; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; align-items: center; }
+  .toolbar select, .toolbar input { background: var(--panel-2); border: 1px solid var(--border); color: var(--text); border-radius: 6px; padding: 5px 9px; font-size: 12.5px; }
+  .toolbar label { color: var(--muted); font-size: 12px; }
+  footer { color: var(--muted); font-size: 11.5px; margin-top: 40px; text-align: center; }
+  svg text { fill: var(--muted); font-size: 10.5px; }
+  .axis line, .axis path { stroke: var(--border); }
+  .n-a { color: var(--muted); }
+</style>
+</head>
+<body>
+<div class="wrap">
+  <header class="top">
+    <h1>astra bench report</h1>
+    <div class="meta" id="generated"></div>
+  </header>
+
+  <div class="kpis" id="kpis"></div>
+
+  <section>
+    <h2>Leaderboard <span class="sub">one row per model · reasoning</span></h2>
+    <div class="grid2">
+      <div class="panel">
+        <h3>Solve rate</h3>
+        <div id="chart-solve"></div>
+      </div>
+      <div class="panel">
+        <h3>Outcomes</h3>
+        <div id="chart-outcomes"></div>
+        <div class="legend" id="legend-outcomes"></div>
+      </div>
+    </div>
+    <div class="panel" style="margin-top:16px;">
+      <table id="tbl-leaderboard"></table>
+    </div>
+  </section>
+
+  <section>
+    <h2>Cost &amp; tokens</h2>
+    <div class="grid2">
+      <div class="panel">
+        <h3>Cost vs. solve rate</h3>
+        <div id="chart-scatter"></div>
+      </div>
+      <div class="panel">
+        <h3>Token mix (prompt / completion / reasoning / cached)</h3>
+        <div id="chart-tokens"></div>
+        <div class="legend" id="legend-tokens"></div>
+      </div>
+    </div>
+  </section>
+
+  <section id="section-series">
+    <h2>Per-step trend <span class="sub">median tokens &amp; cumulative cost across the trajectory</span></h2>
+    <div class="grid2">
+      <div class="panel">
+        <h3>Prompt tokens (context) vs. step</h3>
+        <div id="chart-line-tokens"></div>
+      </div>
+      <div class="panel">
+        <h3>Cumulative cost vs. step</h3>
+        <div id="chart-line-cost"></div>
+      </div>
+    </div>
+    <div class="legend" id="legend-series"></div>
+  </section>
+
+  <section id="section-matrix">
+    <h2>Model &times; task <span class="sub">pass@k</span></h2>
+    <div class="panel">
+      <table id="tbl-matrix" class="heat-table"></table>
+    </div>
+  </section>
+
+  <section>
+    <h2>Runs <span class="sub">every attempt · click a row to expand the command timeline</span></h2>
+    <div class="toolbar">
+      <label>Model
+        <select id="filter-model"><option value="">all</option></select>
+      </label>
+      <label>Task
+        <select id="filter-task"><option value="">all</option></select>
+      </label>
+      <label>Outcome
+        <select id="filter-outcome"><option value="">all</option></select>
+      </label>
+    </div>
+    <div class="panel">
+      <table id="tbl-runs"></table>
+    </div>
+  </section>
+
+  <footer>generated by <span class="mono">astra bench --report</span> · schema <span id="schema"></span></footer>
+</div>
+
+<script>
+/*__ASTRA_DATA__*/
+</script>
+<script>
+(function () {
+  var DATA = window.DATA || { kpis: {}, leaderboard: [], matrix: [], runs: [], step_series: [] };
+  var PALETTE = ["#5b9dff", "#3ecf8e", "#e8b339", "#ef5a6f", "#b57bff", "#33c2c2", "#ff8a5b", "#a3d977"];
+  var colorCache = {};
+  function colorFor(key) {
+    if (!colorCache[key]) {
+      var n = Object.keys(colorCache).length;
+      colorCache[key] = PALETTE[n % PALETTE.length];
+    }
+    return colorCache[key];
+  }
+
+  function fmt(n) { return Number(n || 0).toLocaleString("en-US"); }
+  function fmt1(n) { return (Number(n) || 0).toFixed(1); }
+  function pct(n) { return Math.round((Number(n) || 0) * 100) + "%"; }
+  function fmtUsd(n) {
+    if (n == null) return '<span class="n-a">n/a</span>';
+    var v = Number(n) || 0;
+    if (v === 0) return "$0";
+    if (v < 0.01) return "$" + v.toFixed(5);
+    if (v < 1) return "$" + v.toFixed(4);
+    return "$" + v.toFixed(2);
+  }
+  function esc(s) {
+    return String(s == null ? "" : s).replace(/[&<>"']/g, function (c) {
+      return { "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c];
+    });
+  }
+  function el(tag, attrs, html) {
+    var e = document.createElement(tag);
+    for (var k in attrs || {}) e.setAttribute(k, attrs[k]);
+    if (html != null) e.innerHTML = html;
+    return e;
+  }
+  function svgEl(tag, attrs) {
+    var e = document.createElementNS("http://www.w3.org/2000/svg", tag);
+    for (var k in attrs || {}) e.setAttribute(k, attrs[k]);
+    return e;
+  }
+
+  // ---------------------------------------------------------------- header
+  document.getElementById("generated").textContent = "generated " + (DATA.generated_at || "");
+  document.getElementById("schema").textContent = DATA.schema || "";
+
+  // ---------------------------------------------------------------- KPIs
+  (function renderKpis() {
+    var k = DATA.kpis || {};
+    var cards = [
+      ["Runs", fmt(k.runs)],
+      ["Models", fmt(k.models)],
+      ["Tasks", fmt(k.tasks)],
+      ["Solved", fmt(k.solved) + " / " + fmt(k.runs)],
+      ["Solve rate", pct(k.solved_rate)],
+      ["Total tokens", fmt(k.tokens)],
+      ["Total cost", fmtUsd(k.cost_usd) + (k.cost_source === "estimated" ? "~" : "")],
+      ["Elapsed", fmt(k.elapsed_seconds) + "s"],
+    ];
+    var host = document.getElementById("kpis");
+    cards.forEach(function (c) {
+      host.appendChild(el("div", { class: "kpi" }, '<div class="v">' + c[1] + '</div><div class="l">' + esc(c[0]) + "</div>"));
+    });
+  })();
+
+  // ---------------------------------------------------------------- sortable table helper
+  function sortableTable(container, columns, rows, opts) {
+    opts = opts || {};
+    var state = { key: opts.defaultKey || columns[0].key, dir: opts.defaultDir || -1 };
+    function draw() {
+      var sorted = rows.slice().sort(function (a, b) {
+        var va = a[state.key], vb = b[state.key];
+        if (typeof va === "string" || typeof vb === "string") {
+          return state.dir * String(va).localeCompare(String(vb));
+        }
+        return state.dir * ((Number(va) || 0) - (Number(vb) || 0));
+      });
+      var thead = "<thead><tr>" + columns.map(function (c) {
+        var arrow = c.key === state.key ? (state.dir === 1 ? "\u2191" : "\u2193") : "";
+        return '<th data-key="' + c.key + '">' + esc(c.label) + (arrow ? ' <span class="arrow">' + arrow + "</span>" : "") + "</th>";
+      }).join("") + "</tr></thead>";
+      var tbody = "<tbody>" + sorted.map(function (row, i) {
+        var cells = columns.map(function (c) { return "<td>" + c.render(row) + "</td>"; }).join("");
+        var extra = opts.rowAttrs ? opts.rowAttrs(row, i) : "";
+        return "<tr " + extra + ">" + cells + "</tr>";
+      }).join("") + "</tbody>";
+      container.innerHTML = thead + tbody;
+      container.querySelectorAll("th").forEach(function (th) {
+        th.addEventListener("click", function () {
+          var key = th.getAttribute("data-key");
+          if (state.key === key) state.dir *= -1;
+          else { state.key = key; state.dir = -1; }
+          draw();
+        });
+      });
+      if (opts.afterDraw) opts.afterDraw(container, sorted);
+    }
+    draw();
+  }
+
+  // ---------------------------------------------------------------- leaderboard chart: solve rate bars
+  (function () {
+    var host = document.getElementById("chart-solve");
+    (DATA.leaderboard || []).forEach(function (g) {
+      var row = el("div", { class: "bar-row" });
+      row.appendChild(el("div", { class: "lbl mono" }, esc(g.slug)));
+      var track = el("div", { class: "bar-track" });
+      var seg = el("div", {
+        class: "bar-seg",
+        style: "width:" + Math.round((g.solved_rate || 0) * 100) + "%;background:" + colorFor(g.slug),
+      });
+      track.appendChild(seg);
+      row.appendChild(track);
+      row.appendChild(el("div", { class: "bar-val" }, pct(g.solved_rate) + " (" + g.solved + "/" + g.runs + ")"));
+      host.appendChild(row);
+    });
+  })();
+
+  // ---------------------------------------------------------------- outcomes stacked bar
+  (function () {
+    var host = document.getElementById("chart-outcomes");
+    var legend = document.getElementById("legend-outcomes");
+    var outcomeColors = { Submitted: "#3ecf8e", LimitsExceeded: "#e8b339", TimeExceeded: "#e8b339", RepeatedFormatError: "#ef5a6f", ContextWindowExceeded: "#ef5a6f", Error: "#ef5a6f", Unknown: "#8b93a7" };
+    function colorForOutcome(k) { return outcomeColors[k] || "#8b93a7"; }
+    var allOutcomes = {};
+    (DATA.leaderboard || []).forEach(function (g) {
+      var row = el("div", { class: "bar-row" });
+      row.appendChild(el("div", { class: "lbl mono" }, esc(g.slug)));
+      var track = el("div", { class: "bar-track" });
+      var total = g.runs || 1;
+      Object.keys(g.outcomes || {}).forEach(function (k) {
+        allOutcomes[k] = true;
+        var w = (g.outcomes[k] / total) * 100;
+        track.appendChild(el("div", { class: "bar-seg", style: "width:" + w + "%;background:" + colorForOutcome(k) }));
+      });
+      row.appendChild(track);
+      row.appendChild(el("div", { class: "bar-val" }, g.runs + " runs"));
+      host.appendChild(row);
+    });
+    Object.keys(allOutcomes).forEach(function (k) {
+      legend.appendChild(el("span", {}, '<span class="sw" style="background:' + colorForOutcome(k) + '"></span>' + esc(k)));
+    });
+  })();
+
+  // ---------------------------------------------------------------- leaderboard table
+  (function () {
+    var container = document.getElementById("tbl-leaderboard");
+    var columns = [
+      { key: "model", label: "Model", render: function (r) { return esc(r.model); } },
+      { key: "reasoning", label: "Reasoning", render: function (r) { return esc(r.reasoning); } },
+      { key: "runs", label: "Runs", render: function (r) { return fmt(r.runs); } },
+      { key: "solved_rate", label: "Solved", render: function (r) { return r.solved + "/" + r.runs + " (" + pct(r.solved_rate) + ")"; } },
+      { key: "avg_steps", label: "Avg steps", render: function (r) { return fmt1(r.avg_steps); } },
+      { key: "avg_elapsed_seconds", label: "Avg time", render: function (r) { return fmt1(r.avg_elapsed_seconds) + "s"; } },
+      { key: "avg_tokens", label: "Avg tokens", render: function (r) { return fmt(Math.round(r.avg_tokens)); } },
+      { key: "cost_usd", label: "Cost", render: function (r) { return fmtUsd(r.cost_usd) + (r.cost_source === "estimated" ? "~" : ""); } },
+      { key: "avg_n_failed_commands", label: "Fail/run", render: function (r) { return fmt1(r.avg_n_failed_commands); } },
+    ];
+    sortableTable(container, columns, DATA.leaderboard || [], { defaultKey: "solved_rate" });
+  })();
+
+  // ---------------------------------------------------------------- token mix stacked bar
+  (function () {
+    var host = document.getElementById("chart-tokens");
+    var legend = document.getElementById("legend-tokens");
+    var keys = ["prompt", "completion", "reasoning", "cached"];
+    var colors = { prompt: "#5b9dff", completion: "#3ecf8e", reasoning: "#b57bff", cached: "#8b93a7" };
+    var maxTotal = Math.max(1, Math.max.apply(null, (DATA.leaderboard || []).map(function (g) {
+      return keys.reduce(function (s, k) { return s + (g.tokens[k] || 0); }, 0);
+    })));
+    (DATA.leaderboard || []).forEach(function (g) {
+      var row = el("div", { class: "bar-row" });
+      row.appendChild(el("div", { class: "lbl mono" }, esc(g.slug)));
+      var track = el("div", { class: "bar-track" });
+      var total = keys.reduce(function (s, k) { return s + (g.tokens[k] || 0); }, 0);
+      keys.forEach(function (k) {
+        var w = (g.tokens[k] || 0) / maxTotal * 100;
+        if (w > 0) track.appendChild(el("div", { class: "bar-seg", style: "width:" + w + "%;background:" + colors[k] }));
+      });
+      row.appendChild(track);
+      row.appendChild(el("div", { class: "bar-val" }, fmt(total)));
+      host.appendChild(row);
+    });
+    keys.forEach(function (k) {
+      legend.appendChild(el("span", {}, '<span class="sw" style="background:' + colors[k] + '"></span>' + k));
+    });
+  })();
+
+  // ---------------------------------------------------------------- scatter: cost vs solve rate
+  (function () {
+    var host = document.getElementById("chart-scatter");
+    var W = host.clientWidth || 460, H = 260, pad = { l: 42, r: 16, t: 14, b: 30 };
+    var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
+    var data = (DATA.leaderboard || []).filter(function (g) { return g.cost_usd != null; });
+    var maxCost = Math.max(0.001, Math.max.apply(null, data.map(function (g) { return g.cost_usd; }).concat([0.001])));
+    function x(v) { return pad.l + (v / maxCost) * (W - pad.l - pad.r); }
+    function y(v) { return H - pad.b - v * (H - pad.t - pad.b); }
+    // axes
+    svg.appendChild(svgEl("line", { x1: pad.l, x2: W - pad.r, y1: H - pad.b, y2: H - pad.b, stroke: "#262b36" }));
+    svg.appendChild(svgEl("line", { x1: pad.l, x2: pad.l, y1: pad.t, y2: H - pad.b, stroke: "#262b36" }));
+    [0, 0.25, 0.5, 0.75, 1].forEach(function (t) {
+      var ty = y(t);
+      var lab = svgEl("text", { x: pad.l - 8, y: ty + 3, "text-anchor": "end" });
+      lab.textContent = Math.round(t * 100) + "%";
+      svg.appendChild(lab);
+    });
+    var lab2 = svgEl("text", { x: W - pad.r, y: H - pad.b + 20, "text-anchor": "end" });
+    lab2.textContent = "cost \u2192 (max " + fmtUsd(maxCost) + ")";
+    svg.appendChild(lab2);
+    data.forEach(function (g) {
+      var cx = x(g.cost_usd), cy = y(g.solved_rate);
+      var r = 6 + Math.min(10, (g.avg_tokens || 0) / 4000);
+      var c = svgEl("circle", { cx: cx, cy: cy, r: r, fill: colorFor(g.slug), "fill-opacity": 0.85 });
+      svg.appendChild(c);
+      var t = svgEl("text", { x: cx + r + 4, y: cy + 3 });
+      t.textContent = g.slug;
+      svg.appendChild(t);
+    });
+    host.appendChild(svg);
+  })();
+
+  // ---------------------------------------------------------------- per-step line charts
+  function lineChart(host, series, valueFn, opts) {
+    opts = opts || {};
+    var W = host.clientWidth || 460, H = 260, pad = { l: 46, r: 16, t: 14, b: 26 };
+    var svg = svgEl("svg", { width: "100%", height: H, viewBox: "0 0 " + W + " " + H });
+    var maxStep = 1, maxVal = 0.0001;
+    series.forEach(function (s) {
+      s.points.forEach(function (p) {
+        if (p.step > maxStep) maxStep = p.step;
+        var v = valueFn(p);
+        if (v > maxVal) maxVal = v;
+      });
+    });
+    function x(v) { return pad.l + (v - 1) / Math.max(1, maxStep - 1) * (W - pad.l - pad.r); }
+    function y(v) { return H - pad.b - (v / maxVal) * (H - pad.t - pad.b); }
+    svg.appendChild(svgEl("line", { x1: pad.l, x2: W - pad.r, y1: H - pad.b, y2: H - pad.b, stroke: "#262b36" }));
+    svg.appendChild(svgEl("line", { x1: pad.l, x2: pad.l, y1: pad.t, y2: H - pad.b, stroke: "#262b36" }));
+    [0, 0.5, 1].forEach(function (t) {
+      var ty = y(maxVal * t);
+      var lab = svgEl("text", { x: pad.l - 6, y: ty + 3, "text-anchor": "end" });
+      lab.textContent = opts.fmt ? opts.fmt(maxVal * t) : Math.round(maxVal * t);
+      svg.appendChild(lab);
+    });
+    series.forEach(function (s) {
+      var pts = s.points.slice().sort(function (a, b) { return a.step - b.step; });
+      var d = pts.map(function (p, i) { return (i === 0 ? "M" : "L") + x(p.step) + "," + y(valueFn(p)); }).join(" ");
+      svg.appendChild(svgEl("path", { d: d, fill: "none", stroke: colorFor(s.slug), "stroke-width": 2 }));
+      pts.forEach(function (p) {
+        svg.appendChild(svgEl("circle", { cx: x(p.step), cy: y(valueFn(p)), r: 2.5, fill: colorFor(s.slug) }));
+      });
+    });
+    host.appendChild(svg);
+  }
+
+  (function () {
+    var bySlug = {};
+    (DATA.step_series || []).forEach(function (r) {
+      if (!bySlug[r.slug]) bySlug[r.slug] = [];
+      bySlug[r.slug].push(r);
+    });
+    var series = Object.keys(bySlug).map(function (slug) { return { slug: slug, points: bySlug[slug] }; });
+    if (series.length === 0) {
+      document.getElementById("section-series").style.display = "none";
+      return;
+    }
+    lineChart(document.getElementById("chart-line-tokens"), series, function (p) { return p.prompt_tokens.p50; }, { fmt: fmt });
+    lineChart(document.getElementById("chart-line-cost"), series, function (p) { return p.cum_cost_usd.mean; }, { fmt: fmtUsd });
+    var legend = document.getElementById("legend-series");
+    series.forEach(function (s) {
+      legend.appendChild(el("span", {}, '<span class="sw" style="background:' + colorFor(s.slug) + '"></span>' + esc(s.slug)));
+    });
+  })();
+
+  // ---------------------------------------------------------------- model x task matrix (heatmap table)
+  (function () {
+    var matrix = DATA.matrix || [];
+    var container = document.getElementById("tbl-matrix");
+    if (matrix.length === 0) {
+      document.getElementById("section-matrix").style.display = "none";
+      return;
+    }
+    var tasks = [];
+    var slugs = [];
+    var cell = {};
+    matrix.forEach(function (m) {
+      if (tasks.indexOf(m.task_id) === -1) tasks.push(m.task_id);
+      if (slugs.indexOf(m.slug) === -1) slugs.push(m.slug);
+      cell[m.slug + "\u0000" + m.task_id] = m;
+    });
+    function heatColor(rate) {
+      if (rate == null) return "transparent";
+      var g = Math.round(60 + rate * 140);
+      var r = Math.round(220 - rate * 160);
+      return "rgba(" + r + "," + g + ",120,0.35)";
+    }
+    var thead = "<thead><tr><th>Model</th>" + tasks.map(function (t) { return "<th>" + esc(t) + "</th>"; }).join("") + "</tr></thead>";
+    var tbody = "<tbody>" + slugs.map(function (slug) {
+      var cells = tasks.map(function (t) {
+        var m = cell[slug + "\u0000" + t];
+        if (!m) return "<td>&mdash;</td>";
+        return '<td><span class="heat-cell" style="background:' + heatColor(m.pass_at_k) + '">' + m.passed + "/" + m.k + "<br>" + pct(m.pass_at_k) + "</span></td>";
+      }).join("");
+      return "<tr><td class=\"mono\">" + esc(slug) + "</td>" + cells + "</tr>";
+    }).join("") + "</tbody>";
+    container.innerHTML = thead + tbody;
+  })();
+
+  // ---------------------------------------------------------------- runs table + filters + expandable timeline
+  (function () {
+    var runs = DATA.runs || [];
+    var container = document.getElementById("tbl-runs");
+    var fModel = document.getElementById("filter-model");
+    var fTask = document.getElementById("filter-task");
+    var fOutcome = document.getElementById("filter-outcome");
+
+    function uniq(fn) {
+      var seen = {}, out = [];
+      runs.forEach(function (r) { var v = fn(r); if (v && !seen[v]) { seen[v] = 1; out.push(v); } });
+      return out.sort();
+    }
+    uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
+    uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
+    uniq(function (r) { return r.exit_status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
+
+    var columns = [
+      { key: "slug", label: "Model", render: function (r) { return '<span class="mono">' + esc(r.slug) + "</span>"; } },
+      { key: "task_id", label: "Task", render: function (r) { return esc(r.task_id); } },
+      { key: "run_id", label: "Run", render: function (r) { return esc(r.run_id); } },
+      { key: "exit_status", label: "Outcome", render: function (r) {
+          var cls = r.resolved ? "ok" : (r.exit_status === "Error" ? "bad" : "warn");
+          return '<span class="pill ' + cls + '">' + esc(r.exit_status || "Unknown") + "</span>";
+        } },
+      { key: "steps", label: "Steps", render: function (r) { return fmt(r.steps); } },
+      { key: "elapsed_seconds", label: "Time", render: function (r) { return fmt(r.elapsed_seconds) + "s"; } },
+      { key: "tokens", label: "Tokens", render: function (r) { return fmt(r.tokens.total); } },
+      { key: "cost_usd", label: "Cost", render: function (r) { return fmtUsd(r.cost_usd) + (r.cost_source === "estimated" ? "~" : ""); } },
+      { key: "n_failed_commands", label: "Fails", render: function (r) { return fmt(r.n_failed_commands); } },
+    ];
+
+    function renderTimeline(run) {
+      var box = el("div", { class: "timeline" });
+      (run.timeline || []).forEach(function (s) {
+        var rcClass = s.returncode === 0 ? "rc0" : (s.returncode == null ? "muted" : "rcN");
+        var step = el("div", { class: "step" });
+        step.innerHTML =
+          '<div class="head"><span class="n">step ' + s.step + '</span>' +
+          '<span class="stats">' +
+            fmt(s.tokens.total) + " tok" +
+            (s.cost_usd != null ? " \u00b7 " + fmtUsd(s.cost_usd) : "") +
+            (s.returncode != null ? ' \u00b7 rc=<span class="' + rcClass + '">' + s.returncode + "</span>" : "") +
+          "</span></div>" +
+          (s.thought ? '<div class="thought">' + esc(s.thought) + "</div>" : "") +
+          (s.command ? "<pre>$ " + esc(s.command) + "</pre>" : "") +
+          (s.output_preview ? "<pre>" + esc(s.output_preview) + "</pre>" : "");
+        box.appendChild(step);
+      });
+      return box;
+    }
+
+    function draw() {
+      var filtered = runs.filter(function (r) {
+        if (fModel.value && r.slug !== fModel.value) return false;
+        if (fTask.value && r.task_id !== fTask.value) return false;
+        if (fOutcome.value && r.exit_status !== fOutcome.value) return false;
+        return true;
+      });
+      sortableTable(container, columns, filtered, {
+        defaultKey: "slug",
+        defaultDir: 1,
+        rowAttrs: function (r, i) { return 'class="run-row" data-idx="' + i + '"'; },
+        afterDraw: function (containerEl, sorted) {
+          containerEl.querySelectorAll("tbody tr").forEach(function (tr) {
+            var idx = Number(tr.getAttribute("data-idx"));
+            var run = sorted[idx];
+            tr.addEventListener("click", function () {
+              var next = tr.nextElementSibling;
+              if (next && next.classList.contains("detail-row")) {
+                next.remove();
+                return;
+              }
+              containerEl.querySelectorAll(".detail-row").forEach(function (d) { d.remove(); });
+              var detailRow = document.createElement("tr");
+              detailRow.className = "detail-row";
+              var td = document.createElement("td");
+              td.colSpan = columns.length;
+              td.appendChild(renderTimeline(run));
+              detailRow.appendChild(td);
+              tr.parentNode.insertBefore(detailRow, tr.nextSibling);
+            });
+          });
+        },
+      });
+    }
+    [fModel, fTask, fOutcome].forEach(function (elm) { elm.addEventListener("change", draw); });
+    draw();
+  })();
+})();
+</script>
+</body>
+</html>