@hackerrank/astra-cli 0.1.5 → 0.1.6

package/README.md

@@ -204,6 +204,17 @@ astra --resume <id>         # resume (continue an interactive chat,
                             # or inspect/continue an autonomous run)
 ```
 
+Resume an interrupted benchmark without repeating its task, model, or output
+arguments:
+
+```bash
+astra bench --resume <session-id>
+```
+
+It reopens the original candidate workspace and updates that same run's
+trajectory, metrics row, and report telemetry. Explicit flags override the
+saved setting when needed.
+
 Verified working models on the gateway include `claude-sonnet-5`, `claude-opus-5`,
 `gpt-5.6-sol`, `gpt-5.6-terra`, `gemini-3.7-flash`, `grok-4.6`, `kimi-k3`, and others.
 Some routes (e.g. `glm-5.2`) may return HTTP 402 when the underlying provider is out of

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/agent.js

@@ -64,6 +64,7 @@ export class Agent {
     this.sessionId = opts.sessionId ?? null;
     this.sessionCreated = opts.sessionCreated ?? new Date().toISOString();
     this.title = opts.title ?? "";
+    this.bench = opts.bench ?? null;
     this._saveSession = opts.saveSession ?? null; // (id, doc) => void
   }
 
@@ -240,6 +241,7 @@ export class Agent {
       id: this.sessionId ?? undefined,
       created: this.sessionCreated,
       title: this.title || undefined,
+      bench: this.bench ?? undefined,
       info: {
         mode: this.mode,
         task: this.task || undefined,
@@ -248,10 +250,20 @@ export class Agent {
         n_steps: this.nSteps,
         model: this.model.model,
         n_calls: this.model.nCalls,
+        n_retries: this.model.nRetries,
+        counters: {
+          commands: this.nCommands,
+          failed_commands: this.nFailedCommands,
+          format_errors: this.nFormatErrors,
+          declined: this.nDeclined,
+        },
         elapsed_seconds: Math.round((Date.now() - this.startTime) / 1000),
         tokens: {
           prompt: this.model.totalPromptTokens,
           completion: this.model.totalCompletionTokens,
+          cached: this.model.totalCachedTokens,
+          cache_write: this.model.totalCacheWriteTokens,
+          reasoning: this.model.totalReasoningTokens,
           total: this.model.totalPromptTokens + this.model.totalCompletionTokens,
           last_context: this.lastContextTokens,
         },
@@ -282,8 +294,25 @@ export class Agent {
     this.task = data?.info?.task ?? this.task;
     this.mode = data?.info?.mode === "interactive" ? "interactive" : this.mode;
     this.nSteps = data?.info?.n_steps ?? this.messages.filter((m) => m.role === "assistant").length;
+    this.nCommands = data?.info?.counters?.commands ?? 0;
+    this.nFailedCommands = data?.info?.counters?.failed_commands ?? 0;
+    this.nFormatErrors = data?.info?.counters?.format_errors ?? 0;
+    this.nDeclined = data?.info?.counters?.declined ?? 0;
     this.lastContextTokens = data?.info?.tokens?.last_context ?? 0;
+    this.model.nCalls = data?.info?.n_calls ?? this.model.nCalls;
+    this.model.nRetries = data?.info?.n_retries ?? this.model.nRetries;
+    this.model.totalPromptTokens = data?.info?.tokens?.prompt ?? this.model.totalPromptTokens;
+    this.model.totalCompletionTokens = data?.info?.tokens?.completion ?? this.model.totalCompletionTokens;
+    this.model.totalCachedTokens = data?.info?.tokens?.cached ?? this.model.totalCachedTokens;
+    this.model.totalCacheWriteTokens = data?.info?.tokens?.cache_write ?? this.model.totalCacheWriteTokens;
+    this.model.totalReasoningTokens = data?.info?.tokens?.reasoning ?? this.model.totalReasoningTokens;
+    this.model.totalCostUsd = data?.info?.cost?.usd ?? this.model.totalCostUsd;
+    this.model.reportedCostUsd = data?.info?.cost?.reported_usd ?? this.model.reportedCostUsd;
+    this.model.estimatedCostUsd = data?.info?.cost?.estimated_usd ?? this.model.estimatedCostUsd;
+    this.model.costSource = data?.info?.cost?.source ?? this.model.costSource;
+    this.startTime = Date.now() - Number(data?.info?.elapsed_seconds || 0) * 1000;
     this.exitStatus = data?.info?.exit_status || null;
+    this.bench = data?.bench ?? this.bench;
     if (data?.id) this.sessionId = data.id;
     if (data?.created) this.sessionCreated = data.created;
     if (data?.title) this.title = data.title;

package/src/bench.js

@@ -26,6 +26,7 @@ import { Agent } from "./agent.js";
 
 /** CSV columns, in order, for the metrics table. */
 export const METRIC_COLUMNS = [
+  "run_path",
   "timestamp",
   "model",
   "reasoning",
@@ -211,8 +212,9 @@ export function metricsHeader() {
  * the top-level rolled-up index at <root>/metrics.csv.
  */
 export function writeMetrics({ runDir, root, metrics }) {
+  const rowMetrics = { ...metrics, run_path: path.relative(benchRoot(root), runDir) };
   const header = metricsHeader();
-  const row = metricsRow(metrics);
+  const row = metricsRow(rowMetrics);
 
   // Per-run file.
   fs.writeFileSync(path.join(runDir, "metrics.csv"), header + "\n" + row + "\n");
@@ -222,7 +224,14 @@ export function writeMetrics({ runDir, root, metrics }) {
   if (!fs.existsSync(indexPath)) {
     fs.writeFileSync(indexPath, header + "\n" + row + "\n");
   } else {
-    fs.appendFileSync(indexPath, row + "\n");
+    const lines = fs.readFileSync(indexPath, "utf8").trimEnd().split("\n");
+    const runPathIndex = lines[0].split(",").indexOf("run_path");
+    const existing = runPathIndex < 0
+      ? -1
+      : lines.findIndex((line, index) => index > 0 && line.split(",")[runPathIndex] === rowMetrics.run_path);
+    if (existing >= 0) lines[existing] = row;
+    else lines.push(row);
+    fs.writeFileSync(indexPath, lines.join("\n") + "\n");
   }
   return { runMetrics: path.join(runDir, "metrics.csv"), index: indexPath };
 }

package/src/cli.js

@@ -17,6 +17,7 @@
  *   astra -m <model> -f path/to/instruction.md
  *   astra -m <model> -p tasks/dummy-slugify  bench run from a task directory
  *   astra bench --project ./task         project-configured bench run
+ *   astra bench --resume <session-id>    continue the same bench attempt
  *   astra --resume <session-id>             resume a saved session
  *   astra --sessions                        list saved sessions
  *
@@ -198,6 +199,19 @@ async function main() {
   // Bench mode uses the autonomous engine; agent mode uses the interactive one.
   let task = args.task;
   if (args["task-file"]) task = fs.readFileSync(args["task-file"], "utf8");
+  let resumeBench = resumeDoc?.bench ?? null;
+  if (args.bench && args.resume && !resumeBench) {
+    console.error("\x1b[31m[astra] request_error: this session is not a bench run.\x1b[0m");
+    process.exit(2);
+  }
+  if (resumeBench) {
+    if (!args._provided.has("task") && !args._provided.has("task-file")) task = resumeBench.task;
+    if (!args._provided.has("path")) args.path = resumeBench.taskPath;
+    if (!args._provided.has("bench-root")) args["bench-root"] = resumeBench.root;
+    if (!args._provided.has("steps")) args.steps = resumeBench.steps;
+    if (!args._provided.has("wall")) args.wall = resumeBench.wall;
+    if (!args._provided.has("timeout")) args.timeout = resumeBench.timeout;
+  }
   let project = null;
   if (args.project) {
     try {
@@ -261,7 +275,7 @@ async function main() {
   //   5. hard fallback: glm-5.2 with medium reasoning
   // The chosen values are cached so the next `astra` needs no flags.
   let modelId = args.model || resumeDoc?.info?.model;
-  let reasoning = args.reasoning || "";
+  let reasoning = args.reasoning || resumeBench?.reasoning || "";
 
   if (mode === "autonomous") {
     if (project && !modelId) modelId = project.bench.models.join(",");
@@ -330,7 +344,21 @@ async function main() {
   // workspace under bench/<model-name-reasoning>/run-NN/. The agent's commands
   // run inside that workspace and metrics are recorded when it finishes.
   let benchRun = null;
-  if (mode === "autonomous" && !resumeDoc) {
+  if (mode === "autonomous" && resumeBench) {
+    if (!fs.existsSync(resumeBench.dir) || !fs.existsSync(resumeBench.workspace)) {
+      console.error("\x1b[31m[astra] request_error: saved bench workspace is missing.\x1b[0m");
+      process.exit(2);
+    }
+    benchRun = {
+      dir: resumeBench.dir,
+      workspace: resumeBench.workspace,
+      root: resumeBench.root,
+      slug: path.basename(path.dirname(resumeBench.dir)),
+      runId: path.basename(resumeBench.dir),
+    };
+    resumeBench.segments = Array.isArray(resumeBench.segments) ? resumeBench.segments : [];
+    resumeBench.segments.push({ started_at: new Date().toISOString(), resumed: true });
+  } else if (mode === "autonomous" && !resumeDoc) {
     benchRun = allocateRun({ model: modelId, reasoning, root: args["bench-root"] });
     const seeded = seedWorkspace(benchRun.workspace, {
       taskPath: args.path,
@@ -341,9 +369,22 @@ async function main() {
     // Persist the task text alongside the run for reproducibility.
     fs.writeFileSync(path.join(benchRun.dir, "task.md"), task || "");
     writeProjectMetadata(benchRun.dir, project?.metadata);
+    resumeBench = {
+      dir: benchRun.dir,
+      workspace: benchRun.workspace,
+      root: benchRun.root,
+      task,
+      taskPath: args.path,
+      model: modelId,
+      reasoning,
+      steps: Number(args.steps),
+      wall: Number(args.wall),
+      timeout: Number(args.timeout),
+      segments: [{ started_at: new Date().toISOString(), resumed: false }],
+    };
   }
 
-  const cmdCwd = benchRun ? benchRun.workspace : cwd;
+  const cmdCwd = benchRun?.workspace || cwd;
   const env = new LocalEnvironment({
     cwd: cmdCwd,
     timeout: Number(args.timeout),
@@ -362,12 +403,23 @@ async function main() {
       : null,
     sessionId,
     saveSession,
+    bench: resumeBench,
     onEvent: quiet || mode === "interactive" ? () => {} : (msg) => printEvent(msg),
     onStep: quiet || mode === "interactive" ? () => {} : (s) => printStep(s),
   });
 
   if (resumeDoc) {
     agent.restore(resumeDoc);
+    if (resumeBench) {
+      agent.exitStatus = null;
+      if (!args._provided.has("steps") && Number(resumeBench.steps) > 0) {
+        agent.stepLimit = agent.nSteps + Number(resumeBench.steps);
+      }
+      if (!args._provided.has("wall") && Number(resumeBench.wall) > 0) {
+        agent.wallTimeLimitSeconds =
+          Math.ceil((Date.now() - agent.startTime) / 1000) + Number(resumeBench.wall);
+      }
+    }
     console.error(`\x1b[2m[astra] resumed ${sessionId} (${agent.messages.length} messages)\x1b[0m`);
   }
 
@@ -423,6 +475,7 @@ async function main() {
           `${Number(args.steps) > 0 ? ` turns<=${args.steps}` : ""}\x1b[0m`
       );
       console.error(`\x1b[2m[astra] workspace -> ${benchRun.workspace}\x1b[0m`);
+      console.error(`\x1b[2m[astra] session   -> ${sessionId}\x1b[0m`);
     } else {
       console.error(
         `\x1b[2m[astra] bench · model=${modelId} cwd=${cmdCwd}` +
@@ -431,12 +484,38 @@ async function main() {
     }
   }
 
-  const result =
-    resumeDoc && !task ? await continueAutonomous(agent) : await agent.run(task);
+  let result;
+  try {
+    result = resumeDoc ? await continueAutonomous(agent) : await agent.run(task);
+  } catch (error) {
+    agent.exitStatus = "Error";
+    const segment = agent.bench?.segments?.at(-1);
+    if (segment) {
+      segment.finished_at = new Date().toISOString();
+      segment.exit_status = "Error";
+      segment.error = String(error?.message || error);
+    }
+    agent.save();
+    if (benchRun) {
+      writeMetrics({
+        runDir: benchRun.dir,
+        root: benchRun.root,
+        metrics: collectMetrics({ agent, model, reasoning }),
+      });
+      try { refreshReport(benchRun.root); } catch {}
+    }
+    throw error;
+  }
   agent.save();
 
   // Record benchmark metrics (per-run CSV + rolled-up index).
   if (benchRun) {
+    const segment = agent.bench?.segments?.at(-1);
+    if (segment) {
+      segment.finished_at = new Date().toISOString();
+      segment.exit_status = result.exit_status;
+    }
+    agent.save();
     const metrics = collectMetrics({ agent, model, reasoning });
     const paths = writeMetrics({ runDir: benchRun.dir, root: benchRun.root, metrics });
     if (!quiet) {

package/src/report.js

@@ -296,6 +296,7 @@ function readRunDir({ rootDir, slug, runId, dir }) {
     tokens,
     cost_usd: costSource ? num(metricsRow?.cost_usd ?? traj?.info?.cost?.usd) : null,
     cost_source: costSource || "",
+    resume_segments: Array.isArray(traj?.bench?.segments) ? traj.bench.segments : [],
     timeline,
     paths: {
       dir: path.relative(rootDir, dir),