@hackerrank/astra-cli 0.1.6 → 0.1.7

package/README.md

@@ -29,7 +29,7 @@ Every run is a resumable session, with exact token accounting and USD cost track
 | `src/model.js` | Gateway client (OpenAI-compatible `/chat/completions`). Handles retries, the `max_tokens` vs `max_completion_tokens` difference, and cost accounting. |
 | `src/environment.js` | Runs one command per action in a fresh subshell, with a per-command timeout that kills the whole process group and output truncation. |
 | `src/prompts.js` | System / instance / format-error / observation templates + a tiny `{{var}}` renderer, split into interactive vs autonomous rules. |
-| `src/agent.js` | The turn-based engine: query → parse one command → execute → observe → repeat. Handles submission, step/time limits, and format errors. |
+| `src/agent.js` | The turn-based engine: query → parse shell blocks → execute → observe → repeat. Autonomous runs can combine related blocks; interactive runs remain strict. |
 | `src/repl.js` | The interactive assistant loop: prompts, command approval, slash commands. |
 | `src/session.js` | Resumable sessions stored under `~/.astra/sessions/`. |
 | `src/config.js` | Dedicated `~/.astra/config.json`, credential resolution, interactive key prompt. |
@@ -141,19 +141,26 @@ it through the existing bench persona:
 astra bench --project ./task
 ```
 
-`./task/.astra/project.toml` selects the public workspace, instruction,
-models, limits, output root, and optional bench template/extensions. CLI flags
-still override bench limits and output paths. Astra copies only the configured
-workspace; container lifecycle, services, secrets, verification, and scoring
-remain harness responsibilities. Run artifacts include `project.json` with the
-project and input hashes. `completed` means the agent emitted its completion
-sentinel—not that the candidate is correct; `model_error` and `request_error`
-distinguish generation/provider failures from invalid invocation/configuration.
+`./task/.astra/project.toml` selects the task type (`brownfield` or
+`greenfield`), public workspace, instruction, models, limits, output root, and
+optional context extensions. A task-owned `verifier/run_verifier.py` is
+discovered automatically; an explicit `[verification]` entry may override its
+command and lifecycle settings. CLI flags still override bench limits and
+output paths.
+
+A project-configured run copies only the public workspace, freezes each
+candidate, invokes the task-owned verifier outside that workspace, reads its
+versioned JSON report, and writes a merged result. Astra contains no task
+business logic. If no verifier exists, the run is still recorded with
+`solved_score = "NA"`; if a verifier is configured but fails to produce a
+valid report, the result is an infrastructure failure. `model_error`,
+`candidate`, `verifier`, and `infrastructure` failures remain distinct.
 
 ```toml
 [project]
 id = "peoplecore-scim"
 version = 1
+type = "brownfield"
 instruction = "instruction.md"
 workspace = "public/codebase"
 
@@ -165,12 +172,26 @@ steps = 60
 wall_seconds = 3600
 command_timeout_seconds = 120
 
+[verification]
+command = "python3 verifier/run_verifier.py --base-url $SCIM_BASE_URL --token $SCIM_BEARER_TOKEN --report $ASTRA_VERIFIER_REPORT"
+base_url = "http://127.0.0.1:3000/api/v1/scim"
+readiness_url = "http://127.0.0.1:3000/health"
+timeout_seconds = 3600
+report = "verifier-report.json"
+token_env = "SCIM_BEARER_TOKEN"
+
 [templates]
 bench = ".astra/templates/bench.md"
 [extensions]
 directories = [".astra/extensions"]
 ```
 
+Project runs are appendable. The durable cell key is task/version/model/
+reasoning/repeat. The report display key is `model-reasoning`, so adding a
+model, reasoning effort, or a higher per-cell repeat count preserves completed
+cells and processes only missing cells. If a process stops, rerunning the same
+project resumes the cell/session or verifier stage from the ledger.
+
 ### HTML report
 
 Every matrix run (and every single `-p`/`-t`/`-f` bench run) rebuilds

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/agent.js

@@ -150,7 +150,7 @@ export class Agent {
     // --- 2. Parse a command (may be absent) ---
     let command;
     try {
-      command = parseCommand(content);
+      command = parseCommand(content, { allowMultiple: this.mode === "autonomous" });
       this.formatErrorStreak = 0;
     } catch (err) {
       // In interactive mode, "no command" is a normal chat reply that yields
@@ -320,9 +320,11 @@ export class Agent {
 }
 
 /** Extract exactly one command from a fenced bash block. */
-export function parseCommand(text) {
-  // Accept ```bash, ```sh, or a bare ``` block.
-  const re = /```(?:bash|sh)?\s*\n([\s\S]*?)```/g;
+export function parseCommand(text, { allowMultiple = false } = {}) {
+  // Accept common shell fence labels. Autonomous mode can combine several
+  // fenced blocks into one script; interactive mode retains the strict one
+  // command contract.
+  const re = /```(?:bash|sh|shell|zsh)?[ \t]*\r?\n([\s\S]*?)```/gi;
   const blocks = [];
   let m;
   while ((m = re.exec(text)) !== null) blocks.push(m[1].trim());
@@ -332,12 +334,12 @@ export function parseCommand(text) {
     err.code = "NO_COMMAND";
     throw err;
   }
-  if (blocks.length > 1) {
+  if (blocks.length > 1 && !allowMultiple) {
     const err = new Error(`Found ${blocks.length} code blocks; provide exactly one.`);
     err.code = "MULTI_COMMAND";
     throw err;
   }
-  const cmd = blocks[0].trim();
+  const cmd = blocks.join("\n").trim();
   if (!cmd) {
     const err = new Error("The bash code block was empty.");
     err.code = "EMPTY_COMMAND";

package/src/bench.js

@@ -23,6 +23,7 @@ import { spawnSync } from "node:child_process";
 import { GatewayModel } from "./model.js";
 import { LocalEnvironment } from "./environment.js";
 import { Agent } from "./agent.js";
+import { loadSession, newSessionId, saveSession } from "./session.js";
 
 /** CSV columns, in order, for the metrics table. */
 export const METRIC_COLUMNS = [
@@ -49,6 +50,11 @@ export const METRIC_COLUMNS = [
   "last_context_tokens",
   "cost_usd",
   "cost_source",
+  "verification_status",
+  "verification_score",
+  "solved_score",
+  "verification_hard_pass",
+  "failure_owner",
 ];
 
 /**
@@ -295,15 +301,33 @@ export async function runCell({
   wall = 0,
   timeout = 60,
   maxOutputChars = 16000,
+  runDir: existingRunDir = null,
+  sessionId: existingSessionId = null,
+  resume = false,
+  cell = null,
   onStart = () => {},
   onDone = () => {},
 } = {}) {
-  const alloc = allocateRun({ model, reasoning, root });
+  const alloc = existingRunDir
+    ? {
+        dir: path.resolve(existingRunDir),
+        workspace: path.join(path.resolve(existingRunDir), "workspace"),
+        root: benchRoot(root),
+        slug: benchSlug(model, reasoning),
+        runId: path.basename(path.resolve(existingRunDir)),
+      }
+    : allocateRun({ model, reasoning, root });
+  fs.mkdirSync(alloc.workspace, { recursive: true });
   onStart({ ...alloc, model, reasoning });
 
-  const seeded = seedWorkspace(alloc.workspace, { taskPath, taskFile, taskText: task });
-  const taskText = seeded.taskText || task || "";
-  fs.writeFileSync(path.join(alloc.dir, "task.md"), taskText);
+  let taskText = task || "";
+  if (!resume) {
+    const seeded = seedWorkspace(alloc.workspace, { taskPath, taskFile, taskText: task });
+    taskText = seeded.taskText || task || "";
+    fs.writeFileSync(path.join(alloc.dir, "task.md"), taskText);
+  } else if (!taskText) {
+    taskText = fs.existsSync(path.join(alloc.dir, "task.md")) ? fs.readFileSync(path.join(alloc.dir, "task.md"), "utf8") : "";
+  }
   writeProjectMetadata(alloc.dir, projectMetadata);
 
   const gw = new GatewayModel({
@@ -313,16 +337,40 @@ export async function runCell({
     modelKwargs: reasoningKwargs(reasoning),
   });
   const env = new LocalEnvironment({ cwd: alloc.workspace, timeout, maxOutputChars });
+  const sessionId = existingSessionId || newSessionId();
   const agent = new Agent(gw, env, {
     mode: "autonomous",
     stepLimit: steps,
     wallTimeLimitSeconds: wall,
     outputPath: path.join(alloc.dir, "trajectory.json"),
+    sessionId,
+    saveSession,
+    task: taskText,
+    bench: { dir: alloc.dir, workspace: alloc.workspace, root: alloc.root, task: taskText, taskPath, model, reasoning, steps, wall, timeout, segments: [] },
   });
 
+  if (resume && existingSessionId) {
+    const saved = loadSession(existingSessionId);
+    agent.restore(saved);
+    agent.exitStatus = null;
+    if (Number(steps) > 0) agent.stepLimit = agent.nSteps + Number(steps);
+    if (Number(wall) > 0) agent.wallTimeLimitSeconds = Math.ceil((Date.now() - agent.startTime) / 1000) + Number(wall);
+  }
+
   let error = null;
   try {
-    await agent.run(taskText);
+    if (resume && existingSessionId) {
+      while (true) {
+        if (agent.stepLimit > 0 && agent.nSteps >= agent.stepLimit) {
+          agent.exit("LimitsExceeded", "");
+          break;
+        }
+        const turn = await agent.runTurn();
+        if (turn.kind === "exit") break;
+      }
+    } else {
+      await agent.run(taskText);
+    }
   } catch (err) {
     error = String(err?.message || err);
     if (!agent.exitStatus) agent.exit("Error", "");
@@ -332,7 +380,7 @@ export async function runCell({
   if (error) metrics.exit_status = metrics.exit_status || "Error";
   const paths = writeMetrics({ runDir: alloc.dir, root: alloc.root, metrics });
 
-  const result = { ...metrics, error, slug: alloc.slug, runId: alloc.runId, dir: alloc.dir, paths };
+  const result = { ...metrics, error, slug: alloc.slug, runId: alloc.runId, dir: alloc.dir, sessionId, cell, paths };
   onDone(result);
   return result;
 }

package/src/cli.js

@@ -104,6 +104,8 @@ import {
   benchRoot,
 } from "./bench.js";
 import { refreshReport } from "./report.js";
+import { runProjectBench } from "./project-bench.js";
+import { loadLedger } from "./ledger.js";
 
 /** Map a reasoning level to gateway modelKwargs. Empty for off/none/unset. */
 function reasoningKwargs(level) {
@@ -187,7 +189,7 @@ async function main() {
 
   // Load a session to resume (if any) to infer defaults.
   let resumeDoc = null;
-  if (args.resume) {
+  if (args.resume && !args.project) {
     if (!sessionExists(args.resume)) {
       console.error(`\x1b[31m[astra] no such session: ${args.resume}\x1b[0m`);
       process.exit(2);
@@ -263,6 +265,51 @@ async function main() {
     }
   }
 
+  if (project) {
+    try {
+      let resumeCell = null;
+      if (args.resume) {
+        const ledgerPath = path.join(project.outputRoot, "benchmark.json");
+        if (fs.existsSync(ledgerPath)) {
+          const ledger = loadLedger(ledgerPath);
+          resumeCell = Object.values(ledger.cells).find((cell) => cell.sessionId === args.resume) || null;
+        }
+        if (!resumeCell) {
+          throw new Error(`no project benchmark cell found for resume session ${args.resume}`);
+        }
+      }
+      const effectiveProject = {
+        ...project,
+        outputRoot: args._provided.has("bench-root") ? path.resolve(args["bench-root"]) : project.outputRoot,
+        bench: {
+          ...project.bench,
+          models: args._provided.has("model") ? String(args.model).split(",").map((value) => value.trim()).filter(Boolean) : resumeCell ? [resumeCell.model] : project.bench.models,
+          reasoning: args._provided.has("reasoning") ? String(args.reasoning).split(",").map((value) => value.trim()).filter(Boolean) : resumeCell ? [resumeCell.reasoning] : project.bench.reasoning,
+          repeat: args._provided.has("repeat") ? Math.max(1, Number(args.repeat) || 1) : resumeCell ? resumeCell.repeatIndex : project.bench.repeat,
+          steps: args._provided.has("steps") ? Number(args.steps) : project.bench.steps,
+          wall: args._provided.has("wall") ? Number(args.wall) : project.bench.wall,
+          timeout: args._provided.has("timeout") ? Number(args.timeout) : project.bench.timeout,
+        },
+      };
+      const results = await runProjectBench({
+        project: effectiveProject,
+        apiKey,
+        baseUrl: args["base-url"],
+        resume: Boolean(args.resume),
+        resumeSessionId: args.resume || null,
+        quiet: !!args.quiet,
+        onCell: ({ phase, cell }) => {
+          if (!args.quiet) console.error(`\x1b[2m[astra] ${phase} · ${cell.displayKey} repeat=${cell.repeatIndex}\x1b[0m`);
+        },
+      });
+      console.error(`\x1b[1m[astra] project bench complete · ${results.length} cell(s) processed\x1b[0m`);
+    } catch (error) {
+      console.error(`\x1b[31m[astra] project bench failed: ${error.message}\x1b[0m`);
+      process.exit(1);
+    }
+    process.exit(0);
+  }
+
   // -------------------- MODEL + REASONING RESOLUTION --------------------
   // Bench (autonomous) mode: an explicit --model is required so runs are
   // reproducible and never silently pick a cached/default model.

package/src/ledger.js

@@ -0,0 +1,68 @@
+import fs from "node:fs";
+import path from "node:path";
+
+const LEDGER_VERSION = 1;
+const STALE_AFTER_MS = 10 * 60 * 1000;
+
+function slug(value) {
+  return String(value ?? "none").trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "none";
+}
+
+function atomicWrite(file, value) {
+  const temp = `${file}.tmp-${process.pid}`;
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(temp, JSON.stringify(value, null, 2) + "\n");
+  fs.renameSync(temp, file);
+}
+
+export function loadLedger(file) {
+  if (!fs.existsSync(file)) return { version: LEDGER_VERSION, cells: {}, path: file };
+  const value = JSON.parse(fs.readFileSync(file, "utf8"));
+  if (value.version !== LEDGER_VERSION || !value.cells || typeof value.cells !== "object") throw new Error("invalid benchmark ledger");
+  return { ...value, path: file };
+}
+
+function nextRunPath(ledger, cell) {
+  const prefix = `${slug(cell.displayKey)}/run-`;
+  const used = Object.values(ledger.cells).filter((item) => item.runPath?.startsWith(prefix));
+  const next = used.reduce((max, item) => Math.max(max, Number(item.runPath.slice(prefix.length)) || 0), 0) + 1;
+  return `${prefix}${String(next).padStart(2, "0")}`;
+}
+
+export function claimCell(ledger, cell) {
+  const existing = ledger.cells[cell.cellKey];
+  if (existing) return existing;
+  const record = {
+    ...cell,
+    status: "pending",
+    runPath: nextRunPath(ledger, cell),
+    createdAt: new Date().toISOString(),
+    leaseAt: null,
+  };
+  ledger.cells[cell.cellKey] = record;
+  atomicWrite(ledger.path, { version: LEDGER_VERSION, cells: ledger.cells });
+  return record;
+}
+
+export function transitionCell(ledger, cellKey, status, patch = {}) {
+  const cell = ledger.cells[cellKey];
+  if (!cell) throw new Error(`unknown benchmark cell: ${cellKey}`);
+  const next = { ...cell, ...patch, status, updatedAt: new Date().toISOString() };
+  if (["generating", "verifying"].includes(status) && patch.leaseAt === undefined) next.leaseAt = next.updatedAt;
+  ledger.cells[cellKey] = next;
+  atomicWrite(ledger.path, { version: LEDGER_VERSION, cells: ledger.cells });
+  return next;
+}
+
+export function selectWork(ledger, desiredCells, now = Date.now(), { force = false, sessionId = null } = {}) {
+  return desiredCells.filter((cell) => {
+    const existing = ledger.cells[cell.cellKey];
+    if (!existing) return true;
+    if (["completed", "failed"].includes(existing.status)) return false;
+    if (sessionId && existing.sessionId !== sessionId) return false;
+    if (["generating", "verifying"].includes(existing.status)) {
+      return force || !existing.leaseAt || now - Date.parse(existing.leaseAt) > STALE_AFTER_MS;
+    }
+    return !sessionId || existing.sessionId === sessionId;
+  });
+}

package/src/model.js

@@ -89,9 +89,9 @@ export class GatewayModel {
     const body = {
       model: this.model,
       messages: messages.map((m) => ({ role: m.role, content: m.content })),
-      [this._tokenParam]: this.maxTokens,
       ...this.modelKwargs,
     };
+    if (this._tokenParam) body[this._tokenParam] = this.maxTokens;
 
     let lastErr;
     for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
@@ -226,6 +226,13 @@ export class GatewayModel {
       this._tokenParam = "max_tokens";
       return true;
     }
+    // Some gateway routes reject either token-cap parameter. Retry once
+    // without one rather than making the model unavailable for benchmarking.
+    if (/max_tokens/.test(errText) && /not supported|unsupported/.test(errText) && "max_tokens" in body) {
+      delete body.max_tokens;
+      this._tokenParam = null;
+      return true;
+    }
     return false;
   }
 }

package/src/models.js

@@ -17,6 +17,7 @@ export const AVAILABLE_MODELS = [
   "deepseek-v4-pro",
   "glm-5.2",
   "kimi-k3",
+  "minimax-m3",
   "qwen-3.8",
 ];
 

package/src/project-bench.js

@@ -0,0 +1,154 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { matrixCells } from "./project.js";
+import { runCell as defaultRunCell, writeMetrics } from "./bench.js";
+import { candidateTreeHash, mergeRunResult } from "./result-contract.js";
+import { loadLedger, claimCell, selectWork, transitionCell } from "./ledger.js";
+import { runVerifier as defaultRunVerifier } from "./verifier-runner.js";
+import { newSessionId } from "./session.js";
+import { refreshReport } from "./report.js";
+
+function writeJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(value, null, 2) + "\n");
+}
+
+export async function runProjectBench({ project, apiKey, baseUrl, quiet = false, resume = false, resumeSessionId = null, runCellFn = defaultRunCell, verifierFn = defaultRunVerifier, onCell = () => {} } = {}) {
+  const root = project.outputRoot;
+  const ledger = loadLedger(path.join(root, "benchmark.json"));
+  const desired = matrixCells({ project });
+  const work = selectWork(ledger, desired, Date.now(), { force: resume, sessionId: resumeSessionId });
+  const results = [];
+
+  for (const cell of work) {
+    const record = claimCell(ledger, cell);
+    const runDir = path.join(root, record.runPath);
+    const candidateDir = path.join(runDir, "workspace");
+    const reuseFrozen = ["candidate-frozen", "verifying"].includes(record.status);
+    const isResume = Boolean(record.sessionId && ["generating", "interrupted"].includes(record.status));
+    const sessionId = record.sessionId || newSessionId();
+
+    let generation;
+    if (reuseFrozen) {
+      const generationPath = path.join(runDir, "generation.json");
+      try {
+        generation = JSON.parse(fs.readFileSync(generationPath, "utf8"));
+        if (!fs.existsSync(candidateDir)) throw new Error("frozen candidate workspace is missing");
+        const actualHash = candidateTreeHash(candidateDir);
+        if (record.candidateSha256 && actualHash !== record.candidateSha256) {
+          throw new Error("frozen candidate workspace hash does not match the ledger");
+        }
+      } catch (error) {
+        const failure = { status: "error", failureOwner: "infrastructure", error: String(error?.message || error) };
+        transitionCell(ledger, cell.cellKey, "failed", { failureOwner: "infrastructure", error: failure.error });
+        results.push({ cell, generation: { status: "error" }, verification: failure });
+        continue;
+      }
+      onCell({ phase: "candidate-frozen", cell, record });
+    } else {
+      transitionCell(ledger, cell.cellKey, "generating", { sessionId, leaseAt: new Date().toISOString() });
+      onCell({ phase: "generating", cell, record });
+      try {
+        generation = await runCellFn({
+          model: cell.model,
+          reasoning: cell.reasoning,
+          apiKey,
+          baseUrl,
+          task: project.taskText,
+          taskPath: project.workspace,
+          projectMetadata: project.metadata,
+          root,
+          runDir,
+          sessionId,
+          resume: isResume,
+          cell,
+          steps: project.bench.steps,
+          wall: project.bench.wall,
+          timeout: project.bench.timeout,
+        });
+      } catch (error) {
+        const failure = { status: "error", failureOwner: "infrastructure", error: String(error?.message || error) };
+        transitionCell(ledger, cell.cellKey, "failed", { failureOwner: "infrastructure", error: failure.error });
+        results.push({ cell, generation: { status: "error" }, verification: failure });
+        continue;
+      }
+
+      const candidateSha256 = candidateTreeHash(candidateDir);
+      writeJson(path.join(runDir, "generation.json"), generation);
+      transitionCell(ledger, cell.cellKey, "candidate-frozen", { candidateSha256, sessionId });
+    }
+
+    const candidateSha256 = record.candidateSha256 || candidateTreeHash(candidateDir);
+
+    let verification = {
+      status: "not_configured",
+      failureOwner: null,
+      score: null,
+      solvedScore: "NA",
+      reason: "task verifier is not present or not configured",
+    };
+    if (project.verification && generation.status === "completed") {
+      transitionCell(ledger, cell.cellKey, "verifying", { candidateSha256 });
+      try {
+        verification = await verifierFn({
+          project,
+          cell,
+          runDir,
+          candidateDir,
+          config: project.verification,
+          expected: { taskId: project.id, taskVersion: project.version, candidateSha256 },
+        });
+      } catch (error) {
+        verification = {
+          status: "error",
+          failureOwner: "infrastructure",
+          score: null,
+          solvedScore: "NA",
+          error: String(error?.message || error),
+        };
+      }
+      if (verification.score && typeof verification.score.percentage === "number") {
+        verification.solvedScore = verification.score.percentage;
+      }
+      writeJson(path.join(runDir, "verifier.json"), verification);
+    } else if (generation.status !== "completed") {
+      verification = {
+        status: "not_run",
+        failureOwner: "candidate",
+        score: null,
+        solvedScore: "NA",
+        reason: "generation did not submit a candidate; verification was skipped",
+      };
+    }
+
+    const merged = mergeRunResult({
+      task: { id: project.id, version: project.version, type: project.type || "brownfield" },
+      cell,
+      generation: { ...generation, candidateSha256 },
+      verification,
+      artifacts: { runPath: record.runPath, candidateSha256 },
+    });
+    writeMetrics({
+      runDir,
+      root,
+      metrics: {
+        ...generation,
+        model: cell.model,
+        reasoning: cell.reasoning,
+        verification_status: verification.status,
+        verification_score: verification.score?.percentage ?? "",
+        solved_score: verification.solvedScore ?? "NA",
+        verification_hard_pass: verification.score?.hardFailPassed ?? "",
+        failure_owner: verification.failureOwner ?? "",
+      },
+    });
+    writeJson(path.join(runDir, "result.json"), merged);
+    transitionCell(ledger, cell.cellKey, "completed", { candidateSha256, resultPath: path.join(record.runPath, "result.json"), failureOwner: verification.failureOwner });
+    results.push(merged);
+    try { refreshReport(root); } catch { /* report regeneration is best-effort per cell */ }
+    onCell({ phase: "completed", cell, record, result: merged });
+  }
+  try { refreshReport(root); } catch { /* report regeneration is best-effort */ }
+  return results;
+}