@hackerrank/astra-cli 0.1.5 → 0.1.7

package/README.md

@@ -29,7 +29,7 @@ Every run is a resumable session, with exact token accounting and USD cost track
 | `src/model.js` | Gateway client (OpenAI-compatible `/chat/completions`). Handles retries, the `max_tokens` vs `max_completion_tokens` difference, and cost accounting. |
 | `src/environment.js` | Runs one command per action in a fresh subshell, with a per-command timeout that kills the whole process group and output truncation. |
 | `src/prompts.js` | System / instance / format-error / observation templates + a tiny `{{var}}` renderer, split into interactive vs autonomous rules. |
-| `src/agent.js` | The turn-based engine: query → parse one command → execute → observe → repeat. Handles submission, step/time limits, and format errors. |
+| `src/agent.js` | The turn-based engine: query → parse shell blocks → execute → observe → repeat. Autonomous runs can combine related blocks; interactive runs remain strict. |
 | `src/repl.js` | The interactive assistant loop: prompts, command approval, slash commands. |
 | `src/session.js` | Resumable sessions stored under `~/.astra/sessions/`. |
 | `src/config.js` | Dedicated `~/.astra/config.json`, credential resolution, interactive key prompt. |
@@ -141,19 +141,26 @@ it through the existing bench persona:
 astra bench --project ./task
 ```
 
-`./task/.astra/project.toml` selects the public workspace, instruction,
-models, limits, output root, and optional bench template/extensions. CLI flags
-still override bench limits and output paths. Astra copies only the configured
-workspace; container lifecycle, services, secrets, verification, and scoring
-remain harness responsibilities. Run artifacts include `project.json` with the
-project and input hashes. `completed` means the agent emitted its completion
-sentinel—not that the candidate is correct; `model_error` and `request_error`
-distinguish generation/provider failures from invalid invocation/configuration.
+`./task/.astra/project.toml` selects the task type (`brownfield` or
+`greenfield`), public workspace, instruction, models, limits, output root, and
+optional context extensions. A task-owned `verifier/run_verifier.py` is
+discovered automatically; an explicit `[verification]` entry may override its
+command and lifecycle settings. CLI flags still override bench limits and
+output paths.
+
+A project-configured run copies only the public workspace, freezes each
+candidate, invokes the task-owned verifier outside that workspace, reads its
+versioned JSON report, and writes a merged result. Astra contains no task
+business logic. If no verifier exists, the run is still recorded with
+`solved_score = "NA"`; if a verifier is configured but fails to produce a
+valid report, the result is an infrastructure failure. `model_error`,
+`candidate`, `verifier`, and `infrastructure` failures remain distinct.
 
 ```toml
 [project]
 id = "peoplecore-scim"
 version = 1
+type = "brownfield"
 instruction = "instruction.md"
 workspace = "public/codebase"
 
@@ -165,12 +172,26 @@ steps = 60
 wall_seconds = 3600
 command_timeout_seconds = 120
 
+[verification]
+command = "python3 verifier/run_verifier.py --base-url $SCIM_BASE_URL --token $SCIM_BEARER_TOKEN --report $ASTRA_VERIFIER_REPORT"
+base_url = "http://127.0.0.1:3000/api/v1/scim"
+readiness_url = "http://127.0.0.1:3000/health"
+timeout_seconds = 3600
+report = "verifier-report.json"
+token_env = "SCIM_BEARER_TOKEN"
+
 [templates]
 bench = ".astra/templates/bench.md"
 [extensions]
 directories = [".astra/extensions"]
 ```
 
+Project runs are appendable. The durable cell key is task/version/model/
+reasoning/repeat. The report display key is `model-reasoning`, so adding a
+model, reasoning effort, or a higher per-cell repeat count preserves completed
+cells and processes only missing cells. If a process stops, rerunning the same
+project resumes the cell/session or verifier stage from the ledger.
+
 ### HTML report
 
 Every matrix run (and every single `-p`/`-t`/`-f` bench run) rebuilds
@@ -204,6 +225,17 @@ astra --resume <id>         # resume (continue an interactive chat,
                             # or inspect/continue an autonomous run)
 ```
 
+Resume an interrupted benchmark without repeating its task, model, or output
+arguments:
+
+```bash
+astra bench --resume <session-id>
+```
+
+It reopens the original candidate workspace and updates that same run's
+trajectory, metrics row, and report telemetry. Explicit flags override the
+saved setting when needed.
+
 Verified working models on the gateway include `claude-sonnet-5`, `claude-opus-5`,
 `gpt-5.6-sol`, `gpt-5.6-terra`, `gemini-3.7-flash`, `grok-4.6`, `kimi-k3`, and others.
 Some routes (e.g. `glm-5.2`) may return HTTP 402 when the underlying provider is out of

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/agent.js

@@ -64,6 +64,7 @@ export class Agent {
     this.sessionId = opts.sessionId ?? null;
     this.sessionCreated = opts.sessionCreated ?? new Date().toISOString();
     this.title = opts.title ?? "";
+    this.bench = opts.bench ?? null;
     this._saveSession = opts.saveSession ?? null; // (id, doc) => void
   }
 
@@ -149,7 +150,7 @@ export class Agent {
     // --- 2. Parse a command (may be absent) ---
     let command;
     try {
-      command = parseCommand(content);
+      command = parseCommand(content, { allowMultiple: this.mode === "autonomous" });
       this.formatErrorStreak = 0;
     } catch (err) {
       // In interactive mode, "no command" is a normal chat reply that yields
@@ -240,6 +241,7 @@ export class Agent {
       id: this.sessionId ?? undefined,
       created: this.sessionCreated,
       title: this.title || undefined,
+      bench: this.bench ?? undefined,
       info: {
         mode: this.mode,
         task: this.task || undefined,
@@ -248,10 +250,20 @@ export class Agent {
         n_steps: this.nSteps,
         model: this.model.model,
         n_calls: this.model.nCalls,
+        n_retries: this.model.nRetries,
+        counters: {
+          commands: this.nCommands,
+          failed_commands: this.nFailedCommands,
+          format_errors: this.nFormatErrors,
+          declined: this.nDeclined,
+        },
         elapsed_seconds: Math.round((Date.now() - this.startTime) / 1000),
         tokens: {
           prompt: this.model.totalPromptTokens,
           completion: this.model.totalCompletionTokens,
+          cached: this.model.totalCachedTokens,
+          cache_write: this.model.totalCacheWriteTokens,
+          reasoning: this.model.totalReasoningTokens,
           total: this.model.totalPromptTokens + this.model.totalCompletionTokens,
           last_context: this.lastContextTokens,
         },
@@ -282,8 +294,25 @@ export class Agent {
     this.task = data?.info?.task ?? this.task;
     this.mode = data?.info?.mode === "interactive" ? "interactive" : this.mode;
     this.nSteps = data?.info?.n_steps ?? this.messages.filter((m) => m.role === "assistant").length;
+    this.nCommands = data?.info?.counters?.commands ?? 0;
+    this.nFailedCommands = data?.info?.counters?.failed_commands ?? 0;
+    this.nFormatErrors = data?.info?.counters?.format_errors ?? 0;
+    this.nDeclined = data?.info?.counters?.declined ?? 0;
     this.lastContextTokens = data?.info?.tokens?.last_context ?? 0;
+    this.model.nCalls = data?.info?.n_calls ?? this.model.nCalls;
+    this.model.nRetries = data?.info?.n_retries ?? this.model.nRetries;
+    this.model.totalPromptTokens = data?.info?.tokens?.prompt ?? this.model.totalPromptTokens;
+    this.model.totalCompletionTokens = data?.info?.tokens?.completion ?? this.model.totalCompletionTokens;
+    this.model.totalCachedTokens = data?.info?.tokens?.cached ?? this.model.totalCachedTokens;
+    this.model.totalCacheWriteTokens = data?.info?.tokens?.cache_write ?? this.model.totalCacheWriteTokens;
+    this.model.totalReasoningTokens = data?.info?.tokens?.reasoning ?? this.model.totalReasoningTokens;
+    this.model.totalCostUsd = data?.info?.cost?.usd ?? this.model.totalCostUsd;
+    this.model.reportedCostUsd = data?.info?.cost?.reported_usd ?? this.model.reportedCostUsd;
+    this.model.estimatedCostUsd = data?.info?.cost?.estimated_usd ?? this.model.estimatedCostUsd;
+    this.model.costSource = data?.info?.cost?.source ?? this.model.costSource;
+    this.startTime = Date.now() - Number(data?.info?.elapsed_seconds || 0) * 1000;
     this.exitStatus = data?.info?.exit_status || null;
+    this.bench = data?.bench ?? this.bench;
     if (data?.id) this.sessionId = data.id;
     if (data?.created) this.sessionCreated = data.created;
     if (data?.title) this.title = data.title;
@@ -291,9 +320,11 @@ export class Agent {
 }
 
 /** Extract exactly one command from a fenced bash block. */
-export function parseCommand(text) {
-  // Accept ```bash, ```sh, or a bare ``` block.
-  const re = /```(?:bash|sh)?\s*\n([\s\S]*?)```/g;
+export function parseCommand(text, { allowMultiple = false } = {}) {
+  // Accept common shell fence labels. Autonomous mode can combine several
+  // fenced blocks into one script; interactive mode retains the strict one
+  // command contract.
+  const re = /```(?:bash|sh|shell|zsh)?[ \t]*\r?\n([\s\S]*?)```/gi;
   const blocks = [];
   let m;
   while ((m = re.exec(text)) !== null) blocks.push(m[1].trim());
@@ -303,12 +334,12 @@ export function parseCommand(text) {
     err.code = "NO_COMMAND";
     throw err;
   }
-  if (blocks.length > 1) {
+  if (blocks.length > 1 && !allowMultiple) {
     const err = new Error(`Found ${blocks.length} code blocks; provide exactly one.`);
     err.code = "MULTI_COMMAND";
     throw err;
   }
-  const cmd = blocks[0].trim();
+  const cmd = blocks.join("\n").trim();
   if (!cmd) {
     const err = new Error("The bash code block was empty.");
     err.code = "EMPTY_COMMAND";

package/src/bench.js

@@ -23,9 +23,11 @@ import { spawnSync } from "node:child_process";
 import { GatewayModel } from "./model.js";
 import { LocalEnvironment } from "./environment.js";
 import { Agent } from "./agent.js";
+import { loadSession, newSessionId, saveSession } from "./session.js";
 
 /** CSV columns, in order, for the metrics table. */
 export const METRIC_COLUMNS = [
+  "run_path",
   "timestamp",
   "model",
   "reasoning",
@@ -48,6 +50,11 @@ export const METRIC_COLUMNS = [
   "last_context_tokens",
   "cost_usd",
   "cost_source",
+  "verification_status",
+  "verification_score",
+  "solved_score",
+  "verification_hard_pass",
+  "failure_owner",
 ];
 
 /**
@@ -211,8 +218,9 @@ export function metricsHeader() {
  * the top-level rolled-up index at <root>/metrics.csv.
  */
 export function writeMetrics({ runDir, root, metrics }) {
+  const rowMetrics = { ...metrics, run_path: path.relative(benchRoot(root), runDir) };
   const header = metricsHeader();
-  const row = metricsRow(metrics);
+  const row = metricsRow(rowMetrics);
 
   // Per-run file.
   fs.writeFileSync(path.join(runDir, "metrics.csv"), header + "\n" + row + "\n");
@@ -222,7 +230,14 @@ export function writeMetrics({ runDir, root, metrics }) {
   if (!fs.existsSync(indexPath)) {
     fs.writeFileSync(indexPath, header + "\n" + row + "\n");
   } else {
-    fs.appendFileSync(indexPath, row + "\n");
+    const lines = fs.readFileSync(indexPath, "utf8").trimEnd().split("\n");
+    const runPathIndex = lines[0].split(",").indexOf("run_path");
+    const existing = runPathIndex < 0
+      ? -1
+      : lines.findIndex((line, index) => index > 0 && line.split(",")[runPathIndex] === rowMetrics.run_path);
+    if (existing >= 0) lines[existing] = row;
+    else lines.push(row);
+    fs.writeFileSync(indexPath, lines.join("\n") + "\n");
   }
   return { runMetrics: path.join(runDir, "metrics.csv"), index: indexPath };
 }
@@ -286,15 +301,33 @@ export async function runCell({
   wall = 0,
   timeout = 60,
   maxOutputChars = 16000,
+  runDir: existingRunDir = null,
+  sessionId: existingSessionId = null,
+  resume = false,
+  cell = null,
   onStart = () => {},
   onDone = () => {},
 } = {}) {
-  const alloc = allocateRun({ model, reasoning, root });
+  const alloc = existingRunDir
+    ? {
+        dir: path.resolve(existingRunDir),
+        workspace: path.join(path.resolve(existingRunDir), "workspace"),
+        root: benchRoot(root),
+        slug: benchSlug(model, reasoning),
+        runId: path.basename(path.resolve(existingRunDir)),
+      }
+    : allocateRun({ model, reasoning, root });
+  fs.mkdirSync(alloc.workspace, { recursive: true });
   onStart({ ...alloc, model, reasoning });
 
-  const seeded = seedWorkspace(alloc.workspace, { taskPath, taskFile, taskText: task });
-  const taskText = seeded.taskText || task || "";
-  fs.writeFileSync(path.join(alloc.dir, "task.md"), taskText);
+  let taskText = task || "";
+  if (!resume) {
+    const seeded = seedWorkspace(alloc.workspace, { taskPath, taskFile, taskText: task });
+    taskText = seeded.taskText || task || "";
+    fs.writeFileSync(path.join(alloc.dir, "task.md"), taskText);
+  } else if (!taskText) {
+    taskText = fs.existsSync(path.join(alloc.dir, "task.md")) ? fs.readFileSync(path.join(alloc.dir, "task.md"), "utf8") : "";
+  }
   writeProjectMetadata(alloc.dir, projectMetadata);
 
   const gw = new GatewayModel({
@@ -304,16 +337,40 @@ export async function runCell({
     modelKwargs: reasoningKwargs(reasoning),
   });
   const env = new LocalEnvironment({ cwd: alloc.workspace, timeout, maxOutputChars });
+  const sessionId = existingSessionId || newSessionId();
   const agent = new Agent(gw, env, {
     mode: "autonomous",
     stepLimit: steps,
     wallTimeLimitSeconds: wall,
     outputPath: path.join(alloc.dir, "trajectory.json"),
+    sessionId,
+    saveSession,
+    task: taskText,
+    bench: { dir: alloc.dir, workspace: alloc.workspace, root: alloc.root, task: taskText, taskPath, model, reasoning, steps, wall, timeout, segments: [] },
   });
 
+  if (resume && existingSessionId) {
+    const saved = loadSession(existingSessionId);
+    agent.restore(saved);
+    agent.exitStatus = null;
+    if (Number(steps) > 0) agent.stepLimit = agent.nSteps + Number(steps);
+    if (Number(wall) > 0) agent.wallTimeLimitSeconds = Math.ceil((Date.now() - agent.startTime) / 1000) + Number(wall);
+  }
+
   let error = null;
   try {
-    await agent.run(taskText);
+    if (resume && existingSessionId) {
+      while (true) {
+        if (agent.stepLimit > 0 && agent.nSteps >= agent.stepLimit) {
+          agent.exit("LimitsExceeded", "");
+          break;
+        }
+        const turn = await agent.runTurn();
+        if (turn.kind === "exit") break;
+      }
+    } else {
+      await agent.run(taskText);
+    }
   } catch (err) {
     error = String(err?.message || err);
     if (!agent.exitStatus) agent.exit("Error", "");
@@ -323,7 +380,7 @@ export async function runCell({
   if (error) metrics.exit_status = metrics.exit_status || "Error";
   const paths = writeMetrics({ runDir: alloc.dir, root: alloc.root, metrics });
 
-  const result = { ...metrics, error, slug: alloc.slug, runId: alloc.runId, dir: alloc.dir, paths };
+  const result = { ...metrics, error, slug: alloc.slug, runId: alloc.runId, dir: alloc.dir, sessionId, cell, paths };
   onDone(result);
   return result;
 }

package/src/cli.js

@@ -17,6 +17,7 @@
  *   astra -m <model> -f path/to/instruction.md
  *   astra -m <model> -p tasks/dummy-slugify  bench run from a task directory
  *   astra bench --project ./task         project-configured bench run
+ *   astra bench --resume <session-id>    continue the same bench attempt
  *   astra --resume <session-id>             resume a saved session
  *   astra --sessions                        list saved sessions
  *
@@ -103,6 +104,8 @@ import {
   benchRoot,
 } from "./bench.js";
 import { refreshReport } from "./report.js";
+import { runProjectBench } from "./project-bench.js";
+import { loadLedger } from "./ledger.js";
 
 /** Map a reasoning level to gateway modelKwargs. Empty for off/none/unset. */
 function reasoningKwargs(level) {
@@ -186,7 +189,7 @@ async function main() {
 
   // Load a session to resume (if any) to infer defaults.
   let resumeDoc = null;
-  if (args.resume) {
+  if (args.resume && !args.project) {
     if (!sessionExists(args.resume)) {
       console.error(`\x1b[31m[astra] no such session: ${args.resume}\x1b[0m`);
       process.exit(2);
@@ -198,6 +201,19 @@ async function main() {
   // Bench mode uses the autonomous engine; agent mode uses the interactive one.
   let task = args.task;
   if (args["task-file"]) task = fs.readFileSync(args["task-file"], "utf8");
+  let resumeBench = resumeDoc?.bench ?? null;
+  if (args.bench && args.resume && !resumeBench) {
+    console.error("\x1b[31m[astra] request_error: this session is not a bench run.\x1b[0m");
+    process.exit(2);
+  }
+  if (resumeBench) {
+    if (!args._provided.has("task") && !args._provided.has("task-file")) task = resumeBench.task;
+    if (!args._provided.has("path")) args.path = resumeBench.taskPath;
+    if (!args._provided.has("bench-root")) args["bench-root"] = resumeBench.root;
+    if (!args._provided.has("steps")) args.steps = resumeBench.steps;
+    if (!args._provided.has("wall")) args.wall = resumeBench.wall;
+    if (!args._provided.has("timeout")) args.timeout = resumeBench.timeout;
+  }
   let project = null;
   if (args.project) {
     try {
@@ -249,6 +265,51 @@ async function main() {
     }
   }
 
+  if (project) {
+    try {
+      let resumeCell = null;
+      if (args.resume) {
+        const ledgerPath = path.join(project.outputRoot, "benchmark.json");
+        if (fs.existsSync(ledgerPath)) {
+          const ledger = loadLedger(ledgerPath);
+          resumeCell = Object.values(ledger.cells).find((cell) => cell.sessionId === args.resume) || null;
+        }
+        if (!resumeCell) {
+          throw new Error(`no project benchmark cell found for resume session ${args.resume}`);
+        }
+      }
+      const effectiveProject = {
+        ...project,
+        outputRoot: args._provided.has("bench-root") ? path.resolve(args["bench-root"]) : project.outputRoot,
+        bench: {
+          ...project.bench,
+          models: args._provided.has("model") ? String(args.model).split(",").map((value) => value.trim()).filter(Boolean) : resumeCell ? [resumeCell.model] : project.bench.models,
+          reasoning: args._provided.has("reasoning") ? String(args.reasoning).split(",").map((value) => value.trim()).filter(Boolean) : resumeCell ? [resumeCell.reasoning] : project.bench.reasoning,
+          repeat: args._provided.has("repeat") ? Math.max(1, Number(args.repeat) || 1) : resumeCell ? resumeCell.repeatIndex : project.bench.repeat,
+          steps: args._provided.has("steps") ? Number(args.steps) : project.bench.steps,
+          wall: args._provided.has("wall") ? Number(args.wall) : project.bench.wall,
+          timeout: args._provided.has("timeout") ? Number(args.timeout) : project.bench.timeout,
+        },
+      };
+      const results = await runProjectBench({
+        project: effectiveProject,
+        apiKey,
+        baseUrl: args["base-url"],
+        resume: Boolean(args.resume),
+        resumeSessionId: args.resume || null,
+        quiet: !!args.quiet,
+        onCell: ({ phase, cell }) => {
+          if (!args.quiet) console.error(`\x1b[2m[astra] ${phase} · ${cell.displayKey} repeat=${cell.repeatIndex}\x1b[0m`);
+        },
+      });
+      console.error(`\x1b[1m[astra] project bench complete · ${results.length} cell(s) processed\x1b[0m`);
+    } catch (error) {
+      console.error(`\x1b[31m[astra] project bench failed: ${error.message}\x1b[0m`);
+      process.exit(1);
+    }
+    process.exit(0);
+  }
+
   // -------------------- MODEL + REASONING RESOLUTION --------------------
   // Bench (autonomous) mode: an explicit --model is required so runs are
   // reproducible and never silently pick a cached/default model.
@@ -261,7 +322,7 @@ async function main() {
   //   5. hard fallback: glm-5.2 with medium reasoning
   // The chosen values are cached so the next `astra` needs no flags.
   let modelId = args.model || resumeDoc?.info?.model;
-  let reasoning = args.reasoning || "";
+  let reasoning = args.reasoning || resumeBench?.reasoning || "";
 
   if (mode === "autonomous") {
     if (project && !modelId) modelId = project.bench.models.join(",");
@@ -330,7 +391,21 @@ async function main() {
   // workspace under bench/<model-name-reasoning>/run-NN/. The agent's commands
   // run inside that workspace and metrics are recorded when it finishes.
   let benchRun = null;
-  if (mode === "autonomous" && !resumeDoc) {
+  if (mode === "autonomous" && resumeBench) {
+    if (!fs.existsSync(resumeBench.dir) || !fs.existsSync(resumeBench.workspace)) {
+      console.error("\x1b[31m[astra] request_error: saved bench workspace is missing.\x1b[0m");
+      process.exit(2);
+    }
+    benchRun = {
+      dir: resumeBench.dir,
+      workspace: resumeBench.workspace,
+      root: resumeBench.root,
+      slug: path.basename(path.dirname(resumeBench.dir)),
+      runId: path.basename(resumeBench.dir),
+    };
+    resumeBench.segments = Array.isArray(resumeBench.segments) ? resumeBench.segments : [];
+    resumeBench.segments.push({ started_at: new Date().toISOString(), resumed: true });
+  } else if (mode === "autonomous" && !resumeDoc) {
     benchRun = allocateRun({ model: modelId, reasoning, root: args["bench-root"] });
     const seeded = seedWorkspace(benchRun.workspace, {
       taskPath: args.path,
@@ -341,9 +416,22 @@ async function main() {
     // Persist the task text alongside the run for reproducibility.
     fs.writeFileSync(path.join(benchRun.dir, "task.md"), task || "");
     writeProjectMetadata(benchRun.dir, project?.metadata);
+    resumeBench = {
+      dir: benchRun.dir,
+      workspace: benchRun.workspace,
+      root: benchRun.root,
+      task,
+      taskPath: args.path,
+      model: modelId,
+      reasoning,
+      steps: Number(args.steps),
+      wall: Number(args.wall),
+      timeout: Number(args.timeout),
+      segments: [{ started_at: new Date().toISOString(), resumed: false }],
+    };
   }
 
-  const cmdCwd = benchRun ? benchRun.workspace : cwd;
+  const cmdCwd = benchRun?.workspace || cwd;
   const env = new LocalEnvironment({
     cwd: cmdCwd,
     timeout: Number(args.timeout),
@@ -362,12 +450,23 @@ async function main() {
       : null,
     sessionId,
     saveSession,
+    bench: resumeBench,
     onEvent: quiet || mode === "interactive" ? () => {} : (msg) => printEvent(msg),
     onStep: quiet || mode === "interactive" ? () => {} : (s) => printStep(s),
   });
 
   if (resumeDoc) {
     agent.restore(resumeDoc);
+    if (resumeBench) {
+      agent.exitStatus = null;
+      if (!args._provided.has("steps") && Number(resumeBench.steps) > 0) {
+        agent.stepLimit = agent.nSteps + Number(resumeBench.steps);
+      }
+      if (!args._provided.has("wall") && Number(resumeBench.wall) > 0) {
+        agent.wallTimeLimitSeconds =
+          Math.ceil((Date.now() - agent.startTime) / 1000) + Number(resumeBench.wall);
+      }
+    }
     console.error(`\x1b[2m[astra] resumed ${sessionId} (${agent.messages.length} messages)\x1b[0m`);
   }
 
@@ -423,6 +522,7 @@ async function main() {
           `${Number(args.steps) > 0 ? ` turns<=${args.steps}` : ""}\x1b[0m`
       );
       console.error(`\x1b[2m[astra] workspace -> ${benchRun.workspace}\x1b[0m`);
+      console.error(`\x1b[2m[astra] session   -> ${sessionId}\x1b[0m`);
     } else {
       console.error(
         `\x1b[2m[astra] bench · model=${modelId} cwd=${cmdCwd}` +
@@ -431,12 +531,38 @@ async function main() {
     }
   }
 
-  const result =
-    resumeDoc && !task ? await continueAutonomous(agent) : await agent.run(task);
+  let result;
+  try {
+    result = resumeDoc ? await continueAutonomous(agent) : await agent.run(task);
+  } catch (error) {
+    agent.exitStatus = "Error";
+    const segment = agent.bench?.segments?.at(-1);
+    if (segment) {
+      segment.finished_at = new Date().toISOString();
+      segment.exit_status = "Error";
+      segment.error = String(error?.message || error);
+    }
+    agent.save();
+    if (benchRun) {
+      writeMetrics({
+        runDir: benchRun.dir,
+        root: benchRun.root,
+        metrics: collectMetrics({ agent, model, reasoning }),
+      });
+      try { refreshReport(benchRun.root); } catch {}
+    }
+    throw error;
+  }
   agent.save();
 
   // Record benchmark metrics (per-run CSV + rolled-up index).
   if (benchRun) {
+    const segment = agent.bench?.segments?.at(-1);
+    if (segment) {
+      segment.finished_at = new Date().toISOString();
+      segment.exit_status = result.exit_status;
+    }
+    agent.save();
     const metrics = collectMetrics({ agent, model, reasoning });
     const paths = writeMetrics({ runDir: benchRun.dir, root: benchRun.root, metrics });
     if (!quiet) {

package/src/ledger.js

@@ -0,0 +1,68 @@
+import fs from "node:fs";
+import path from "node:path";
+
+const LEDGER_VERSION = 1;
+const STALE_AFTER_MS = 10 * 60 * 1000;
+
+function slug(value) {
+  return String(value ?? "none").trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "none";
+}
+
+function atomicWrite(file, value) {
+  const temp = `${file}.tmp-${process.pid}`;
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(temp, JSON.stringify(value, null, 2) + "\n");
+  fs.renameSync(temp, file);
+}
+
+export function loadLedger(file) {
+  if (!fs.existsSync(file)) return { version: LEDGER_VERSION, cells: {}, path: file };
+  const value = JSON.parse(fs.readFileSync(file, "utf8"));
+  if (value.version !== LEDGER_VERSION || !value.cells || typeof value.cells !== "object") throw new Error("invalid benchmark ledger");
+  return { ...value, path: file };
+}
+
+function nextRunPath(ledger, cell) {
+  const prefix = `${slug(cell.displayKey)}/run-`;
+  const used = Object.values(ledger.cells).filter((item) => item.runPath?.startsWith(prefix));
+  const next = used.reduce((max, item) => Math.max(max, Number(item.runPath.slice(prefix.length)) || 0), 0) + 1;
+  return `${prefix}${String(next).padStart(2, "0")}`;
+}
+
+export function claimCell(ledger, cell) {
+  const existing = ledger.cells[cell.cellKey];
+  if (existing) return existing;
+  const record = {
+    ...cell,
+    status: "pending",
+    runPath: nextRunPath(ledger, cell),
+    createdAt: new Date().toISOString(),
+    leaseAt: null,
+  };
+  ledger.cells[cell.cellKey] = record;
+  atomicWrite(ledger.path, { version: LEDGER_VERSION, cells: ledger.cells });
+  return record;
+}
+
+export function transitionCell(ledger, cellKey, status, patch = {}) {
+  const cell = ledger.cells[cellKey];
+  if (!cell) throw new Error(`unknown benchmark cell: ${cellKey}`);
+  const next = { ...cell, ...patch, status, updatedAt: new Date().toISOString() };
+  if (["generating", "verifying"].includes(status) && patch.leaseAt === undefined) next.leaseAt = next.updatedAt;
+  ledger.cells[cellKey] = next;
+  atomicWrite(ledger.path, { version: LEDGER_VERSION, cells: ledger.cells });
+  return next;
+}
+
+export function selectWork(ledger, desiredCells, now = Date.now(), { force = false, sessionId = null } = {}) {
+  return desiredCells.filter((cell) => {
+    const existing = ledger.cells[cell.cellKey];
+    if (!existing) return true;
+    if (["completed", "failed"].includes(existing.status)) return false;
+    if (sessionId && existing.sessionId !== sessionId) return false;
+    if (["generating", "verifying"].includes(existing.status)) {
+      return force || !existing.leaseAt || now - Date.parse(existing.leaseAt) > STALE_AFTER_MS;
+    }
+    return !sessionId || existing.sessionId === sessionId;
+  });
+}

package/src/model.js

@@ -89,9 +89,9 @@ export class GatewayModel {
     const body = {
       model: this.model,
       messages: messages.map((m) => ({ role: m.role, content: m.content })),
-      [this._tokenParam]: this.maxTokens,
       ...this.modelKwargs,
     };
+    if (this._tokenParam) body[this._tokenParam] = this.maxTokens;
 
     let lastErr;
     for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
@@ -226,6 +226,13 @@ export class GatewayModel {
       this._tokenParam = "max_tokens";
       return true;
     }
+    // Some gateway routes reject either token-cap parameter. Retry once
+    // without one rather than making the model unavailable for benchmarking.
+    if (/max_tokens/.test(errText) && /not supported|unsupported/.test(errText) && "max_tokens" in body) {
+      delete body.max_tokens;
+      this._tokenParam = null;
+      return true;
+    }
     return false;
   }
 }

package/src/models.js

@@ -17,6 +17,7 @@ export const AVAILABLE_MODELS = [
   "deepseek-v4-pro",
   "glm-5.2",
   "kimi-k3",
+  "minimax-m3",
   "qwen-3.8",
 ];