@hackerrank/astra-cli 0.1.0 → 0.1.1

package/README.md

@@ -100,6 +100,32 @@ astra \
   -o ./runs/run-01.traj.json
 ```
 
+### Multi-model bench matrix
+
+Pass a **comma-separated** model list (and/or `--repeat`, and/or a comma-separated
+`-r` reasoning list) to run a matrix. Each cell runs sequentially in its own
+isolated workspace under `bench/<model-reasoning>/run-NN/`, and per-run metrics
+are appended to a rolled-up `bench/metrics.csv` index. A leaderboard prints at
+the end.
+
+```bash
+astra \
+  -m claude-sonnet-5,gpt-5.6-sol,gemini-3.7-flash \
+  -p tasks/dummy-slugify \
+  --repeat 3 \
+  --tar                       # write a bench/ tarball locally
+
+# push results to S3 (tars bench/ then `aws s3 cp` to the URI)
+astra -m m1,m2 -p tasks/dummy-slugify --repeat 3 \
+  --push s3://hackerrank-astra-bench-results-dev/runs/2025-09-02/
+```
+
+Leaderboard columns: `solved` (pass rate), average `steps` / `tokens`, and
+`cost`. Cost is per-run summed; a model whose route reports no cost and has no
+price entry shows **`n/a`** (never `$0`), and estimated costs (native routes) are
+marked `~`. `--push` requires an `s3://` URI and the `aws` CLI on PATH; if the
+upload fails the local tarball is kept.
+
 ### Sessions
 
 Every run (either mode) is saved under `~/.astra/sessions/<id>.json`.
@@ -118,9 +144,15 @@ credits — that's a quota issue, not a bug.
 ### Options
 
 ```
--m, --model <id>        Model id on the gateway (required unless --sessions)
+-m, --model <id>        Model id (comma-separated -> multi-model matrix)
+-r, --reasoning <lvl>   Reasoning effort (comma-separated -> sweep levels)
 -t, --task <text>       Task text -> autonomous mode
 -f, --task-file <path>  Read task text from a file -> autonomous mode
+-p, --path <dir>        Task directory -> isolated bench workspace
+    --repeat <n>        Attempts per (model,reasoning) in a matrix (default: 1)
+    --bench-root <dir>  Root folder for bench runs (default: ./bench)
+    --tar               After a matrix, write a bench/ tarball locally
+    --push <s3-uri>     After a matrix, tar bench/ and `aws s3 cp` to the URI
 -C, --cwd <path>        Working directory for commands (default: cwd)
 -o, --output <path>     Also write trajectory JSON here (autonomous mode)
 -s, --steps <n>         Step limit (default: 40)
@@ -202,4 +234,6 @@ Tokens are exact (from each response's `usage`). Cost is **hybrid**, tracking-on
 A run that mixes both is tagged `source: "mixed"` with the split preserved in
 `reported_usd` / `estimated_usd`.
 
-Remote / Jenkins benchmarking lives in the sibling repo `astra-bench`, not here.
+Multi-model benchmarking runs in this CLI (`-m a,b,c --repeat N`, with `--tar` /
+`--push s3://...` to archive results). The sibling repo `astra-bench` only hosts
+the Docker image + Jenkins job that invoke this CLI.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/agent.js

@@ -50,6 +50,11 @@ export class Agent {
     this.messages = [];
     this.nSteps = 0;
     this.formatErrorStreak = 0;
+    // Counters surfaced as benchmark metrics.
+    this.nCommands = 0;        // commands actually executed
+    this.nFailedCommands = 0;  // executed commands with non-zero return code
+    this.nFormatErrors = 0;    // unparseable model replies
+    this.nDeclined = 0;        // commands rejected by the approval gate
     this.startTime = Date.now();
     // Exact context size (prompt tokens) reported by the most recent call.
     this.lastContextTokens = 0;
@@ -153,6 +158,7 @@ export class Agent {
         return { kind: "chat", content };
       }
       this.formatErrorStreak++;
+      this.nFormatErrors++;
       if (
         this.maxConsecutiveFormatErrors > 0 &&
         this.formatErrorStreak >= this.maxConsecutiveFormatErrors
@@ -166,6 +172,7 @@ export class Agent {
     // --- 3. Approval gate ---
     const allowed = await this.confirm(command);
     if (!allowed) {
+      this.nDeclined++;
       this.add("user", "The user declined to run that command. Suggest an alternative or ask what to do instead.", {
         command,
         declined: true,
@@ -175,6 +182,8 @@ export class Agent {
 
     // --- 4. Execute ---
     const output = await this.env.execute(command);
+    this.nCommands++;
+    if (output.returncode !== 0) this.nFailedCommands++;
 
     // --- 5. Submission sentinel ---
     const submitted = checkSubmitted(output);

package/src/bench.js

@@ -0,0 +1,478 @@
+/**
+ * Benchmark run management.
+ *
+ * A "bench" run executes a task autonomously in an isolated workspace and
+ * records metrics for later comparison across models / reasoning levels.
+ *
+ * Layout (created under the invocation cwd unless --bench-root overrides it):
+ *
+ *   bench/<model-name-reasoning>/run-NN/
+ *     ├── workspace/        copy of the task files; the agent works here
+ *     ├── trajectory.json   full astra-1 trajectory for the run
+ *     ├── metrics.csv       one-row metrics summary (also appended to the
+ *     │                     rolled-up bench/metrics.csv index)
+ *     └── task.md           the task text the agent was given
+ *
+ * The per-run metrics are also appended to a top-level `bench/metrics.csv`
+ * index so every run across every model lands in one comparable table.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { GatewayModel } from "./model.js";
+import { LocalEnvironment } from "./environment.js";
+import { Agent } from "./agent.js";
+
+/** CSV columns, in order, for the metrics table. */
+export const METRIC_COLUMNS = [
+  "timestamp",
+  "model",
+  "reasoning",
+  "exit_status",
+  "resolved",
+  "steps",
+  "n_commands",
+  "n_failed_commands",
+  "n_format_errors",
+  "n_retries",
+  "n_calls",
+  "elapsed_seconds",
+  "prompt_tokens",
+  "cached_tokens",
+  "cache_write_tokens",
+  "completion_tokens",
+  "reasoning_tokens",
+  "total_tokens",
+  "last_context_tokens",
+  "cost_usd",
+  "cost_source",
+];
+
+/**
+ * Turn a model id + reasoning level into a filesystem-safe folder name like
+ * `claude-sonnet-5-high` or `grok-4.6-none`.
+ */
+export function benchSlug(model, reasoning) {
+  const clean = (s) =>
+    String(s ?? "")
+      .trim()
+      .replace(/[^a-zA-Z0-9._-]+/g, "-")
+      .replace(/^-+|-+$/g, "");
+  const r = reasoning ? clean(reasoning) : "none";
+  return `${clean(model)}-${r}`;
+}
+
+/** Absolute path to the bench root (default: <cwd>/bench). */
+export function benchRoot(root) {
+  return path.resolve(root || path.join(process.cwd(), "bench"));
+}
+
+/**
+ * Allocate the next `run-NN` directory for a (model, reasoning) slug, creating
+ * parent folders. Returns { dir, run, slug, workspace, root }.
+ */
+export function allocateRun({ model, reasoning, root } = {}) {
+  const rootDir = benchRoot(root);
+  const slug = benchSlug(model, reasoning);
+  const slugDir = path.join(rootDir, slug);
+  fs.mkdirSync(slugDir, { recursive: true });
+
+  // Find the next run index by scanning existing run-NN folders.
+  let max = 0;
+  for (const name of safeReaddir(slugDir)) {
+    const m = /^run-(\d+)$/.exec(name);
+    if (m) max = Math.max(max, Number(m[1]));
+  }
+  const run = max + 1;
+  const runId = `run-${String(run).padStart(2, "0")}`;
+  const dir = path.join(slugDir, runId);
+  const workspace = path.join(dir, "workspace");
+  fs.mkdirSync(workspace, { recursive: true });
+
+  return { dir, run, runId, slug, workspace, root: rootDir };
+}
+
+/**
+ * Seed a run workspace from a task source.
+ * @param {string} workspace  destination directory (already created)
+ * @param {object} src        { taskPath?, taskFile?, taskText? }
+ *   - taskPath: a directory whose contents are copied into the workspace
+ *   - taskFile: a single instruction file; its dir contents are copied too
+ *   - taskText: raw task text (no files to copy)
+ * Returns the resolved task text.
+ */
+export function seedWorkspace(workspace, { taskPath, taskFile, taskText } = {}) {
+  let text = taskText ?? "";
+  let copyFrom = null;
+  let instructionName = null;
+
+  if (taskPath) {
+    const resolved = path.resolve(taskPath);
+    const stat = fs.statSync(resolved);
+    if (stat.isDirectory()) {
+      copyFrom = resolved;
+      // Prefer a conventional instruction file for the task text.
+      for (const name of ["instruction.md", "INSTRUCTION.md", "task.md", "README.md"]) {
+        if (fs.existsSync(path.join(resolved, name))) {
+          text = fs.readFileSync(path.join(resolved, name), "utf8");
+          break;
+        }
+      }
+    } else {
+      copyFrom = path.dirname(resolved);
+      instructionName = path.basename(resolved);
+      text = fs.readFileSync(resolved, "utf8");
+    }
+  } else if (taskFile) {
+    const resolved = path.resolve(taskFile);
+    copyFrom = path.dirname(resolved);
+    instructionName = path.basename(resolved);
+    text = fs.readFileSync(resolved, "utf8");
+  }
+
+  if (copyFrom) copyDir(copyFrom, workspace);
+  return { taskText: text, instructionName };
+}
+
+/** Directory / file names never copied into a fresh workspace (kept clean so
+ * result tarballs don't carry build cruft). */
+const COPY_SKIP = new Set([
+  "bench",
+  ".git",
+  "node_modules",
+  "__pycache__",
+  ".pytest_cache",
+  ".mypy_cache",
+  ".ruff_cache",
+]);
+
+/** Recursively copy a directory (skips build cruft — see COPY_SKIP). */
+function copyDir(from, to) {
+  fs.mkdirSync(to, { recursive: true });
+  for (const entry of fs.readdirSync(from, { withFileTypes: true })) {
+    if (COPY_SKIP.has(entry.name)) continue;
+    const s = path.join(from, entry.name);
+    const d = path.join(to, entry.name);
+    if (entry.isDirectory()) copyDir(s, d);
+    else if (entry.isFile()) fs.copyFileSync(s, d);
+  }
+}
+
+/** Build a metrics row object from a finished agent + run info. */
+export function collectMetrics({ agent, model, reasoning }) {
+  const info = agent.serialize().info;
+  const resolved = info.exit_status === "Submitted" ? 1 : 0;
+  return {
+    timestamp: new Date().toISOString(),
+    model: model.model,
+    reasoning: reasoning || "none",
+    exit_status: info.exit_status || "",
+    resolved,
+    steps: info.n_steps ?? 0,
+    n_commands: agent.nCommands ?? 0,
+    n_failed_commands: agent.nFailedCommands ?? 0,
+    n_format_errors: agent.nFormatErrors ?? 0,
+    n_retries: model.nRetries ?? 0,
+    n_calls: info.n_calls ?? 0,
+    elapsed_seconds: info.elapsed_seconds ?? 0,
+    prompt_tokens: info.tokens?.prompt ?? 0,
+    cached_tokens: model.totalCachedTokens ?? 0,
+    cache_write_tokens: model.totalCacheWriteTokens ?? 0,
+    completion_tokens: info.tokens?.completion ?? 0,
+    reasoning_tokens: model.totalReasoningTokens ?? 0,
+    total_tokens: info.tokens?.total ?? 0,
+    last_context_tokens: info.tokens?.last_context ?? 0,
+    cost_usd: round(info.cost?.usd ?? 0, 6),
+    cost_source: info.cost?.source ?? "",
+  };
+}
+
+/** Serialize a metrics row to a single CSV line (columns in METRIC_COLUMNS order). */
+export function metricsRow(m) {
+  return METRIC_COLUMNS.map((c) => csvCell(m[c])).join(",");
+}
+
+/** The CSV header line. */
+export function metricsHeader() {
+  return METRIC_COLUMNS.join(",");
+}
+
+/**
+ * Write a per-run metrics.csv (header + single row) and append the same row to
+ * the top-level rolled-up index at <root>/metrics.csv.
+ */
+export function writeMetrics({ runDir, root, metrics }) {
+  const header = metricsHeader();
+  const row = metricsRow(metrics);
+
+  // Per-run file.
+  fs.writeFileSync(path.join(runDir, "metrics.csv"), header + "\n" + row + "\n");
+
+  // Rolled-up index (create header once).
+  const indexPath = path.join(benchRoot(root), "metrics.csv");
+  if (!fs.existsSync(indexPath)) {
+    fs.writeFileSync(indexPath, header + "\n" + row + "\n");
+  } else {
+    fs.appendFileSync(indexPath, row + "\n");
+  }
+  return { runMetrics: path.join(runDir, "metrics.csv"), index: indexPath };
+}
+
+function csvCell(v) {
+  const s = v == null ? "" : String(v);
+  return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+}
+
+function round(n, dp) {
+  const f = 10 ** dp;
+  return Math.round((Number(n) || 0) * f) / f;
+}
+
+function safeReaddir(dir) {
+  try {
+    return fs.readdirSync(dir);
+  } catch {
+    return [];
+  }
+}
+
+// ============================================================================
+// Multi-model matrix runner
+// ============================================================================
+
+/** Turn a reasoning level into gateway modelKwargs. Empty for off/none/unset. */
+function reasoningKwargs(level) {
+  if (!level) return {};
+  const l = String(level).toLowerCase();
+  if (l === "off" || l === "none" || l === "disabled") return {};
+  return { reasoning_effort: l };
+}
+
+/**
+ * Run one bench cell: allocate an isolated workspace, seed it, run the agent
+ * autonomously to completion, and record metrics. Never throws — a crashed run
+ * is captured as a failed cell so one bad run can't abort the whole matrix.
+ *
+ * @returns {object} the metrics row (plus `error` when the run threw).
+ */
+export async function runCell({
+  model,
+  reasoning,
+  apiKey,
+  baseUrl,
+  task,
+  taskPath,
+  taskFile,
+  root,
+  steps = 40,
+  wall = 0,
+  timeout = 60,
+  maxOutputChars = 16000,
+  onStart = () => {},
+  onDone = () => {},
+} = {}) {
+  const alloc = allocateRun({ model, reasoning, root });
+  onStart({ ...alloc, model, reasoning });
+
+  const seeded = seedWorkspace(alloc.workspace, { taskPath, taskFile, taskText: task });
+  const taskText = seeded.taskText || task || "";
+  fs.writeFileSync(path.join(alloc.dir, "task.md"), taskText);
+
+  const gw = new GatewayModel({
+    model,
+    baseUrl,
+    apiKey,
+    modelKwargs: reasoningKwargs(reasoning),
+  });
+  const env = new LocalEnvironment({ cwd: alloc.workspace, timeout, maxOutputChars });
+  const agent = new Agent(gw, env, {
+    mode: "autonomous",
+    stepLimit: steps,
+    wallTimeLimitSeconds: wall,
+    outputPath: path.join(alloc.dir, "trajectory.json"),
+  });
+
+  let error = null;
+  try {
+    await agent.run(taskText);
+  } catch (err) {
+    error = String(err?.message || err);
+    if (!agent.exitStatus) agent.exit("Error", "");
+  }
+
+  const metrics = collectMetrics({ agent, model: gw, reasoning });
+  if (error) metrics.exit_status = metrics.exit_status || "Error";
+  const paths = writeMetrics({ runDir: alloc.dir, root: alloc.root, metrics });
+
+  const result = { ...metrics, error, slug: alloc.slug, runId: alloc.runId, dir: alloc.dir, paths };
+  onDone(result);
+  return result;
+}
+
+/**
+ * Run a full matrix of models × reasoning levels × repeats against one task.
+ * Sequential (clean streamed output; avoids gateway rate-limit storms).
+ *
+ * @returns {object[]} one metrics row per cell.
+ */
+export async function runMatrix({
+  models = [],
+  reasonings = [""],
+  repeat = 1,
+  ...cellOpts
+} = {}) {
+  const rows = [];
+  for (const model of models) {
+    for (const reasoning of reasonings) {
+      for (let attempt = 1; attempt <= repeat; attempt++) {
+        const row = await runCell({ ...cellOpts, model, reasoning });
+        rows.push(row);
+      }
+    }
+  }
+  return rows;
+}
+
+// ============================================================================
+// Aggregation / leaderboard
+// ============================================================================
+
+/**
+ * Aggregate metrics rows into a per-(model,reasoning) leaderboard.
+ * `resolved` counts as a solve. Cost is summed only over runs that reported a
+ * cost; a group with no cost data is reported as null (rendered "n/a"), never $0.
+ */
+export function aggregate(rows) {
+  const groups = new Map();
+  for (const r of rows) {
+    const key = `${r.model}\u0000${r.reasoning || "none"}`;
+    if (!groups.has(key)) {
+      groups.set(key, {
+        model: r.model,
+        reasoning: r.reasoning || "none",
+        runs: 0,
+        solved: 0,
+        steps: 0,
+        totalTokens: 0,
+        costUsd: 0,
+        costRuns: 0,
+        costSources: new Set(),
+      });
+    }
+    const g = groups.get(key);
+    g.runs++;
+    g.solved += Number(r.resolved) ? 1 : 0;
+    g.steps += Number(r.steps) || 0;
+    g.totalTokens += Number(r.total_tokens) || 0;
+    if (r.cost_source && r.cost_source !== "") {
+      g.costUsd += Number(r.cost_usd) || 0;
+      g.costRuns++;
+      g.costSources.add(r.cost_source);
+    }
+  }
+  return [...groups.values()].map((g) => ({
+    model: g.model,
+    reasoning: g.reasoning,
+    runs: g.runs,
+    solved: g.solved,
+    solved_rate: g.runs ? g.solved / g.runs : 0,
+    avg_steps: g.runs ? g.steps / g.runs : 0,
+    avg_tokens: g.runs ? g.totalTokens / g.runs : 0,
+    cost_usd: g.costRuns ? g.costUsd : null,
+    cost_source:
+      g.costSources.size === 0
+        ? "unknown"
+        : g.costSources.size === 1
+        ? [...g.costSources][0]
+        : "mixed",
+  }));
+}
+
+// ============================================================================
+// Packaging: tarball the bench folder and (optionally) push to S3
+// ============================================================================
+
+/**
+ * Create a gzipped tarball of the bench root. Returns the tarball path.
+ * Excludes build cruft. Uses the system `tar` (zero npm deps).
+ */
+export function packBench({ root, out } = {}) {
+  const rootDir = benchRoot(root);
+  if (!fs.existsSync(rootDir)) throw new Error(`bench root not found: ${rootDir}`);
+  const parent = path.dirname(rootDir);
+  const base = path.basename(rootDir);
+  const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const tarball = path.resolve(out || path.join(parent, `${base}-${stamp}.tgz`));
+
+  const res = spawnSync(
+    "tar",
+    [
+      "--exclude=__pycache__",
+      "--exclude=.pytest_cache",
+      "--exclude=.git",
+      "-czf",
+      tarball,
+      "-C",
+      parent,
+      base,
+    ],
+    { stdio: ["ignore", "ignore", "pipe"] }
+  );
+  if (res.status !== 0) {
+    throw new Error(`tar failed: ${res.stderr?.toString() || res.error?.message || "unknown"}`);
+  }
+  return tarball;
+}
+
+/** The default S3 bucket for bench result tarballs. */
+export const DEFAULT_BENCH_BUCKET = "astra-bench-results";
+
+/** A filesystem/S3-safe timestamp prefix like 20250902-142530. */
+export function benchTimestamp(d = new Date()) {
+  const p = (n, w = 2) => String(n).padStart(w, "0");
+  return (
+    `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-` +
+    `${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`
+  );
+}
+
+/** Default push destination: s3://astra-bench-results/<datetimestamp>/. */
+export function defaultPushUri(stamp = benchTimestamp()) {
+  return `s3://${DEFAULT_BENCH_BUCKET}/${stamp}/`;
+}
+
+/** Parse an s3://bucket/key URI into { bucket, key }. */
+function parseS3Uri(uri) {
+  const m = /^s3:\/\/([^/]+)\/?(.*)$/.exec(uri);
+  if (!m) throw new Error(`invalid s3 uri: ${uri}`);
+  return { bucket: m[1], key: m[2] };
+}
+
+/** Build a browsable https console URL for an s3://bucket/key object. */
+function s3ConsoleUrl(s3Uri) {
+  const { bucket, key } = parseS3Uri(s3Uri);
+  return `https://s3.console.aws.amazon.com/s3/object/${bucket}?prefix=${encodeURIComponent(key)}`;
+}
+
+/**
+ * Upload a file to S3 via the AWS CLI. Returns { uri, url } where `uri` is the
+ * final s3:// destination and `url` is a browsable https console link.
+ * Fails gracefully (throws a descriptive error) when the AWS CLI is missing or
+ * the copy fails; callers should keep the local tarball on failure.
+ */
+export function pushToS3(file, s3Uri) {
+  if (!/^s3:\/\//.test(s3Uri)) throw new Error(`--push destination must be an s3:// URI, got: ${s3Uri}`);
+  // If the URI ends with "/", upload into it under the file's basename.
+  const dest = s3Uri.endsWith("/") ? s3Uri + path.basename(file) : s3Uri;
+  const res = spawnSync("aws", ["s3", "cp", file, dest], {
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  if (res.error && res.error.code === "ENOENT") {
+    throw new Error("aws CLI not found on PATH — install it or push the tarball manually.");
+  }
+  if (res.status !== 0) {
+    throw new Error(`aws s3 cp failed: ${res.stderr?.toString() || "unknown"}`);
+  }
+  return { uri: dest, url: s3ConsoleUrl(dest) };
+}