@hackerrank/astra-cli 0.1.4 → 0.1.6

package/README.md

@@ -204,6 +204,17 @@ astra --resume <id>         # resume (continue an interactive chat,
                             # or inspect/continue an autonomous run)
 ```
 
+Resume an interrupted benchmark without repeating its task, model, or output
+arguments:
+
+```bash
+astra bench --resume <session-id>
+```
+
+It reopens the original candidate workspace and updates that same run's
+trajectory, metrics row, and report telemetry. Explicit flags override the
+saved setting when needed.
+
 Verified working models on the gateway include `claude-sonnet-5`, `claude-opus-5`,
 `gpt-5.6-sol`, `gpt-5.6-terra`, `gemini-3.7-flash`, `grok-4.6`, `kimi-k3`, and others.
 Some routes (e.g. `glm-5.2`) may return HTTP 402 when the underlying provider is out of
@@ -228,7 +239,7 @@ credits — that's a quota issue, not a bug.
                         whatever runs already exist on disk, then exit
 -C, --cwd <path>        Working directory for commands (default: cwd)
 -o, --output <path>     Also write trajectory JSON here (autonomous mode)
--s, --steps <n>         Step limit (default: 40)
+-s, --steps <n>         Turn limit (unset = no limit)
 -w, --wall <seconds>    Wall-clock limit (default: 0 = none)
     --timeout <seconds> Per-command timeout (default: 60)
     --max-output <n>    Max chars of command output kept (default: 16000)
@@ -237,7 +248,7 @@ credits — that's a quota issue, not a bug.
     --resume <id>       Resume a saved session
     --sessions          List saved sessions and exit
 -y, --yolo              Auto-run commands (always on in autonomous mode)
--q, --quiet             Do not stream steps (autonomous mode)
+-q, --quiet             Do not stream turns (autonomous mode)
 -h, --help              Show help
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/agent.js

@@ -38,7 +38,7 @@ export class Agent {
     this.model = model;
     this.env = env;
     this.mode = opts.mode === "interactive" ? "interactive" : "autonomous";
-    this.stepLimit = opts.stepLimit ?? 40;
+    this.stepLimit = opts.stepLimit ?? 0;
     this.wallTimeLimitSeconds = opts.wallTimeLimitSeconds ?? 0;
     this.maxConsecutiveFormatErrors = opts.maxConsecutiveFormatErrors ?? 3;
     this.outputPath = opts.outputPath ?? null;
@@ -64,6 +64,7 @@ export class Agent {
     this.sessionId = opts.sessionId ?? null;
     this.sessionCreated = opts.sessionCreated ?? new Date().toISOString();
     this.title = opts.title ?? "";
+    this.bench = opts.bench ?? null;
     this._saveSession = opts.saveSession ?? null; // (id, doc) => void
   }
 
@@ -240,6 +241,7 @@ export class Agent {
       id: this.sessionId ?? undefined,
       created: this.sessionCreated,
       title: this.title || undefined,
+      bench: this.bench ?? undefined,
       info: {
         mode: this.mode,
         task: this.task || undefined,
@@ -248,10 +250,20 @@ export class Agent {
         n_steps: this.nSteps,
         model: this.model.model,
         n_calls: this.model.nCalls,
+        n_retries: this.model.nRetries,
+        counters: {
+          commands: this.nCommands,
+          failed_commands: this.nFailedCommands,
+          format_errors: this.nFormatErrors,
+          declined: this.nDeclined,
+        },
         elapsed_seconds: Math.round((Date.now() - this.startTime) / 1000),
         tokens: {
           prompt: this.model.totalPromptTokens,
           completion: this.model.totalCompletionTokens,
+          cached: this.model.totalCachedTokens,
+          cache_write: this.model.totalCacheWriteTokens,
+          reasoning: this.model.totalReasoningTokens,
           total: this.model.totalPromptTokens + this.model.totalCompletionTokens,
           last_context: this.lastContextTokens,
         },
@@ -282,8 +294,25 @@ export class Agent {
     this.task = data?.info?.task ?? this.task;
     this.mode = data?.info?.mode === "interactive" ? "interactive" : this.mode;
     this.nSteps = data?.info?.n_steps ?? this.messages.filter((m) => m.role === "assistant").length;
+    this.nCommands = data?.info?.counters?.commands ?? 0;
+    this.nFailedCommands = data?.info?.counters?.failed_commands ?? 0;
+    this.nFormatErrors = data?.info?.counters?.format_errors ?? 0;
+    this.nDeclined = data?.info?.counters?.declined ?? 0;
     this.lastContextTokens = data?.info?.tokens?.last_context ?? 0;
+    this.model.nCalls = data?.info?.n_calls ?? this.model.nCalls;
+    this.model.nRetries = data?.info?.n_retries ?? this.model.nRetries;
+    this.model.totalPromptTokens = data?.info?.tokens?.prompt ?? this.model.totalPromptTokens;
+    this.model.totalCompletionTokens = data?.info?.tokens?.completion ?? this.model.totalCompletionTokens;
+    this.model.totalCachedTokens = data?.info?.tokens?.cached ?? this.model.totalCachedTokens;
+    this.model.totalCacheWriteTokens = data?.info?.tokens?.cache_write ?? this.model.totalCacheWriteTokens;
+    this.model.totalReasoningTokens = data?.info?.tokens?.reasoning ?? this.model.totalReasoningTokens;
+    this.model.totalCostUsd = data?.info?.cost?.usd ?? this.model.totalCostUsd;
+    this.model.reportedCostUsd = data?.info?.cost?.reported_usd ?? this.model.reportedCostUsd;
+    this.model.estimatedCostUsd = data?.info?.cost?.estimated_usd ?? this.model.estimatedCostUsd;
+    this.model.costSource = data?.info?.cost?.source ?? this.model.costSource;
+    this.startTime = Date.now() - Number(data?.info?.elapsed_seconds || 0) * 1000;
     this.exitStatus = data?.info?.exit_status || null;
+    this.bench = data?.bench ?? this.bench;
     if (data?.id) this.sessionId = data.id;
     if (data?.created) this.sessionCreated = data.created;
     if (data?.title) this.title = data.title;

package/src/bench.js

@@ -26,6 +26,7 @@ import { Agent } from "./agent.js";
 
 /** CSV columns, in order, for the metrics table. */
 export const METRIC_COLUMNS = [
+  "run_path",
   "timestamp",
   "model",
   "reasoning",
@@ -211,8 +212,9 @@ export function metricsHeader() {
  * the top-level rolled-up index at <root>/metrics.csv.
  */
 export function writeMetrics({ runDir, root, metrics }) {
+  const rowMetrics = { ...metrics, run_path: path.relative(benchRoot(root), runDir) };
   const header = metricsHeader();
-  const row = metricsRow(metrics);
+  const row = metricsRow(rowMetrics);
 
   // Per-run file.
   fs.writeFileSync(path.join(runDir, "metrics.csv"), header + "\n" + row + "\n");
@@ -222,7 +224,14 @@ export function writeMetrics({ runDir, root, metrics }) {
   if (!fs.existsSync(indexPath)) {
     fs.writeFileSync(indexPath, header + "\n" + row + "\n");
   } else {
-    fs.appendFileSync(indexPath, row + "\n");
+    const lines = fs.readFileSync(indexPath, "utf8").trimEnd().split("\n");
+    const runPathIndex = lines[0].split(",").indexOf("run_path");
+    const existing = runPathIndex < 0
+      ? -1
+      : lines.findIndex((line, index) => index > 0 && line.split(",")[runPathIndex] === rowMetrics.run_path);
+    if (existing >= 0) lines[existing] = row;
+    else lines.push(row);
+    fs.writeFileSync(indexPath, lines.join("\n") + "\n");
   }
   return { runMetrics: path.join(runDir, "metrics.csv"), index: indexPath };
 }
@@ -282,7 +291,7 @@ export async function runCell({
   taskFile,
   projectMetadata,
   root,
-  steps = 40,
+  steps = 0,
   wall = 0,
   timeout = 60,
   maxOutputChars = 16000,

package/src/cli.js

@@ -17,6 +17,7 @@
  *   astra -m <model> -f path/to/instruction.md
  *   astra -m <model> -p tasks/dummy-slugify  bench run from a task directory
  *   astra bench --project ./task         project-configured bench run
+ *   astra bench --resume <session-id>    continue the same bench attempt
  *   astra --resume <session-id>             resume a saved session
  *   astra --sessions                        list saved sessions
  *
@@ -45,7 +46,7 @@
  *       --bench-root <dir>  Root folder for bench runs (default: ./bench)
  *   -C, --cwd <path>        Working directory for commands (default: cwd)
  *   -o, --output <path>     Also write trajectory JSON here (bench mode)
- *   -s, --steps <n>         Step limit (default: 40)
+ *   -s, --steps <n>         Turn limit (unset = no limit)
  *   -w, --wall <seconds>    Wall-clock limit (default: 0 = none)
  *       --timeout <seconds> Per-command timeout (default: 60)
  *       --max-output <n>    Max chars of command output kept (default: 16000)
@@ -55,7 +56,7 @@
  *       --sessions          List saved sessions and exit
  *   -y, --yolo              Auto-run commands without confirmation
  *                           (always on in bench mode)
- *   -q, --quiet             Do not stream steps (bench mode)
+ *   -q, --quiet             Do not stream turns (bench mode)
  *   -h, --help              Show this help
  *
  * API key resolution (first hit wins):
@@ -113,7 +114,7 @@ function reasoningKwargs(level) {
 }
 
 function parseArgs(argv) {
-  const args = { steps: 40, wall: 0, timeout: 60, quiet: false, "max-output": 16000, _provided: new Set() };
+  const args = { steps: 0, wall: 0, timeout: 60, quiet: false, "max-output": 16000, _provided: new Set() };
   const alias = {
     "-m": "model", "--model": "model",
     "-t": "task", "--task": "task",
@@ -198,6 +199,19 @@ async function main() {
   // Bench mode uses the autonomous engine; agent mode uses the interactive one.
   let task = args.task;
   if (args["task-file"]) task = fs.readFileSync(args["task-file"], "utf8");
+  let resumeBench = resumeDoc?.bench ?? null;
+  if (args.bench && args.resume && !resumeBench) {
+    console.error("\x1b[31m[astra] request_error: this session is not a bench run.\x1b[0m");
+    process.exit(2);
+  }
+  if (resumeBench) {
+    if (!args._provided.has("task") && !args._provided.has("task-file")) task = resumeBench.task;
+    if (!args._provided.has("path")) args.path = resumeBench.taskPath;
+    if (!args._provided.has("bench-root")) args["bench-root"] = resumeBench.root;
+    if (!args._provided.has("steps")) args.steps = resumeBench.steps;
+    if (!args._provided.has("wall")) args.wall = resumeBench.wall;
+    if (!args._provided.has("timeout")) args.timeout = resumeBench.timeout;
+  }
   let project = null;
   if (args.project) {
     try {
@@ -261,7 +275,7 @@ async function main() {
   //   5. hard fallback: glm-5.2 with medium reasoning
   // The chosen values are cached so the next `astra` needs no flags.
   let modelId = args.model || resumeDoc?.info?.model;
-  let reasoning = args.reasoning || "";
+  let reasoning = args.reasoning || resumeBench?.reasoning || "";
 
   if (mode === "autonomous") {
     if (project && !modelId) modelId = project.bench.models.join(",");
@@ -330,7 +344,21 @@ async function main() {
   // workspace under bench/<model-name-reasoning>/run-NN/. The agent's commands
   // run inside that workspace and metrics are recorded when it finishes.
   let benchRun = null;
-  if (mode === "autonomous" && !resumeDoc) {
+  if (mode === "autonomous" && resumeBench) {
+    if (!fs.existsSync(resumeBench.dir) || !fs.existsSync(resumeBench.workspace)) {
+      console.error("\x1b[31m[astra] request_error: saved bench workspace is missing.\x1b[0m");
+      process.exit(2);
+    }
+    benchRun = {
+      dir: resumeBench.dir,
+      workspace: resumeBench.workspace,
+      root: resumeBench.root,
+      slug: path.basename(path.dirname(resumeBench.dir)),
+      runId: path.basename(resumeBench.dir),
+    };
+    resumeBench.segments = Array.isArray(resumeBench.segments) ? resumeBench.segments : [];
+    resumeBench.segments.push({ started_at: new Date().toISOString(), resumed: true });
+  } else if (mode === "autonomous" && !resumeDoc) {
     benchRun = allocateRun({ model: modelId, reasoning, root: args["bench-root"] });
     const seeded = seedWorkspace(benchRun.workspace, {
       taskPath: args.path,
@@ -341,9 +369,22 @@ async function main() {
     // Persist the task text alongside the run for reproducibility.
     fs.writeFileSync(path.join(benchRun.dir, "task.md"), task || "");
     writeProjectMetadata(benchRun.dir, project?.metadata);
+    resumeBench = {
+      dir: benchRun.dir,
+      workspace: benchRun.workspace,
+      root: benchRun.root,
+      task,
+      taskPath: args.path,
+      model: modelId,
+      reasoning,
+      steps: Number(args.steps),
+      wall: Number(args.wall),
+      timeout: Number(args.timeout),
+      segments: [{ started_at: new Date().toISOString(), resumed: false }],
+    };
   }
 
-  const cmdCwd = benchRun ? benchRun.workspace : cwd;
+  const cmdCwd = benchRun?.workspace || cwd;
   const env = new LocalEnvironment({
     cwd: cmdCwd,
     timeout: Number(args.timeout),
@@ -362,12 +403,23 @@ async function main() {
       : null,
     sessionId,
     saveSession,
+    bench: resumeBench,
     onEvent: quiet || mode === "interactive" ? () => {} : (msg) => printEvent(msg),
     onStep: quiet || mode === "interactive" ? () => {} : (s) => printStep(s),
   });
 
   if (resumeDoc) {
     agent.restore(resumeDoc);
+    if (resumeBench) {
+      agent.exitStatus = null;
+      if (!args._provided.has("steps") && Number(resumeBench.steps) > 0) {
+        agent.stepLimit = agent.nSteps + Number(resumeBench.steps);
+      }
+      if (!args._provided.has("wall") && Number(resumeBench.wall) > 0) {
+        agent.wallTimeLimitSeconds =
+          Math.ceil((Date.now() - agent.startTime) / 1000) + Number(resumeBench.wall);
+      }
+    }
     console.error(`\x1b[2m[astra] resumed ${sessionId} (${agent.messages.length} messages)\x1b[0m`);
   }
 
@@ -419,20 +471,51 @@ async function main() {
     if (benchRun) {
       console.error(
         `\x1b[2m[astra] bench · ${benchRun.slug}/${benchRun.runId} ` +
-          `· model=${modelId}${reasoning ? ` reasoning=${reasoning}` : ""} steps<=${args.steps}\x1b[0m`
+          `· model=${modelId}${reasoning ? ` reasoning=${reasoning}` : ""}` +
+          `${Number(args.steps) > 0 ? ` turns<=${args.steps}` : ""}\x1b[0m`
       );
       console.error(`\x1b[2m[astra] workspace -> ${benchRun.workspace}\x1b[0m`);
+      console.error(`\x1b[2m[astra] session   -> ${sessionId}\x1b[0m`);
     } else {
-      console.error(`\x1b[2m[astra] bench · model=${modelId} cwd=${cmdCwd} steps<=${args.steps}\x1b[0m`);
+      console.error(
+        `\x1b[2m[astra] bench · model=${modelId} cwd=${cmdCwd}` +
+          `${Number(args.steps) > 0 ? ` turns<=${args.steps}` : ""}\x1b[0m`
+      );
     }
   }
 
-  const result =
-    resumeDoc && !task ? await continueAutonomous(agent) : await agent.run(task);
+  let result;
+  try {
+    result = resumeDoc ? await continueAutonomous(agent) : await agent.run(task);
+  } catch (error) {
+    agent.exitStatus = "Error";
+    const segment = agent.bench?.segments?.at(-1);
+    if (segment) {
+      segment.finished_at = new Date().toISOString();
+      segment.exit_status = "Error";
+      segment.error = String(error?.message || error);
+    }
+    agent.save();
+    if (benchRun) {
+      writeMetrics({
+        runDir: benchRun.dir,
+        root: benchRun.root,
+        metrics: collectMetrics({ agent, model, reasoning }),
+      });
+      try { refreshReport(benchRun.root); } catch {}
+    }
+    throw error;
+  }
   agent.save();
 
   // Record benchmark metrics (per-run CSV + rolled-up index).
   if (benchRun) {
+    const segment = agent.bench?.segments?.at(-1);
+    if (segment) {
+      segment.finished_at = new Date().toISOString();
+      segment.exit_status = result.exit_status;
+    }
+    agent.save();
     const metrics = collectMetrics({ agent, model, reasoning });
     const paths = writeMetrics({ runDir: benchRun.dir, root: benchRun.root, metrics });
     if (!quiet) {
@@ -451,7 +534,7 @@ async function main() {
     const pt = model.totalPromptTokens;
     const ct = model.totalCompletionTokens;
     console.error(
-      `\n\x1b[1m[astra] exit=${result.exit_status} steps=${agent.nSteps} calls=${model.nCalls}\x1b[0m`
+      `\n\x1b[1m[astra] exit=${result.exit_status} turns=${agent.nSteps}\x1b[0m`
     );
     console.error(
       `\x1b[1m[astra] tokens: prompt=${fmt(pt)} completion=${fmt(ct)} total=${fmt(pt + ct)} ` +
@@ -576,7 +659,7 @@ async function runBenchMatrix({ args, apiKey, models, reasonings, repeat, task,
       const status = r.error ? `model_error: ${r.error.split("\n")[0]}` : r.status;
       console.error(
         `  ${mark} ${String(r.model).padEnd(20)} ${String(r.reasoning).padEnd(8)} ` +
-          `${String(status).padEnd(16)} ${String(r.steps).padStart(3)} steps · ` +
+          `${String(status).padEnd(16)} ${String(r.steps).padStart(3)} turns · ` +
           `${fmt(r.total_tokens)} tok · ${fmtUsd(r.cost_usd)}` +
           `${r.cost_source === "estimated" ? "~" : ""}`
       );
@@ -620,7 +703,7 @@ function printLeaderboard(board) {
   board.sort((a, b) => b.completion_rate - a.completion_rate || a.avg_steps - b.avg_steps);
   console.error(
     `\n\x1b[1m${"model".padEnd(20)} ${"reason".padEnd(8)} ${"complete".padEnd(8)} ` +
-      `${"steps".padStart(6)} ${"tokens".padStart(9)}  ${"cost".padStart(9)}  source\x1b[0m`
+      `${"turns".padStart(6)} ${"tokens".padStart(9)}  ${"cost".padStart(9)}  source\x1b[0m`
   );
   for (const g of board) {
     const completed = `${g.completed}/${g.runs}`;
@@ -665,23 +748,22 @@ function printSessions() {
 
 function printEvent(msg) {
   // Bench mode keeps output compact: skip the verbose system/user/assistant
-  // message bodies (per-step progress is shown by printStep). Only surface a
+  // message bodies (per-turn progress is shown by printStep). Only surface a
   // non-empty terminal exit event so the run's submission is still visible.
   if (msg.role !== "exit" || !msg.content?.trim()) return;
   console.error(`\x1b[35m--- SUBMISSION ---\x1b[0m\n${msg.content}\n`);
 }
 
-/** Compact per-step status line with exact token usage from the API. */
+/** Compact per-turn status line with exact token usage from the API. */
 function printStep(s) {
-  const limit = s.stepLimit > 0 ? `/${s.stepLimit}` : "";
   const cached = s.usage.cached_tokens ? ` (cached ${fmt(s.usage.cached_tokens)})` : "";
   const cost = s.costUsd != null
-    ? ` · ${fmtUsd(s.costUsd)}${s.costKind === "estimated" ? "~" : ""}`
+    ? ` │ ${fmtUsd(s.costUsd)}${s.costKind === "estimated" ? "~" : ""}`
     : "";
   console.error(
-    `\x1b[36m[astra] step ${s.step}${limit} · call ${s.nCalls} · ` +
-      `↑${fmt(s.usage.prompt_tokens)} ↓${fmt(s.usage.completion_tokens)} tok · ` +
-      `ctx ${fmt(s.contextTokens)}${cached}${cost} · ${s.elapsedSeconds.toFixed(1)}s\x1b[0m`
+    `\x1b[36m[astra] turn ${s.step} │ ` +
+      `↑${fmt(s.usage.prompt_tokens)} ↓${fmt(s.usage.completion_tokens)} tok │ ` +
+      `ctx ${fmt(s.contextTokens)}${cached}${cost} │ ${s.elapsedSeconds.toFixed(1)}s\x1b[0m`
   );
 }
 

package/src/project.js

@@ -14,7 +14,7 @@ const DEFAULT_BENCH = {
   models: [],
   reasoning: [""],
   repeat: 1,
-  steps: 40,
+  steps: 0,
   wall_seconds: 0,
   command_timeout_seconds: 60,
 };
@@ -169,7 +169,7 @@ export function loadProject(projectPath) {
       models: stringArray(bench.models, "bench.models", DEFAULT_BENCH.models),
       reasoning: stringArray(bench.reasoning, "bench.reasoning", DEFAULT_BENCH.reasoning),
       repeat: positiveInteger(bench.repeat, "bench.repeat", DEFAULT_BENCH.repeat),
-      steps: positiveInteger(bench.steps, "bench.steps", DEFAULT_BENCH.steps),
+      steps: positiveInteger(bench.steps, "bench.steps", DEFAULT_BENCH.steps, { allowZero: true }),
       wall: positiveInteger(bench.wall_seconds, "bench.wall_seconds", DEFAULT_BENCH.wall_seconds, { allowZero: true }),
       timeout: positiveInteger(bench.command_timeout_seconds, "bench.command_timeout_seconds", DEFAULT_BENCH.command_timeout_seconds),
     },

package/src/report.js

@@ -296,6 +296,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     tokens,
     cost_usd: costSource ? num(metricsRow?.cost_usd ?? traj?.info?.cost?.usd) : null,
     cost_source: costSource || "",
+    resume_segments: Array.isArray(traj?.bench?.segments) ? traj.bench.segments : [],
     timeline,
     paths: {
       dir: path.relative(rootDir, dir),