@gethmy/harness 1.0.0

package/src/command-metric.ts

@@ -0,0 +1,594 @@
+/**
+ * Command evidence collector (card #690, from the #616 battle report P3.7).
+ *
+ * The gap this closes: the gate runner had four collectors for seven kinds, so a
+ * `custom` gate reported `blocked` forever (`gate-collectors.ts`) and conditions
+ * could only address what those four produced. A goal phrased as a NUMBER —
+ * "homepage Lighthouse score ≥ 90, stop after 5 tries" — had no evidence source
+ * and was simply not expressible.
+ *
+ * This module supplies the missing source: run an operator-allowlisted command in
+ * the worktree, parse one value out of its stdout, and put it in the evidence doc
+ * as `structured.value`. The existing closed-enum predicate then does the gating
+ * (`{ path: "value", op: "gte", value: 90 }`), and the existing converge-loop
+ * machinery (`max_iterations` + `resolveLoopExitGate` + `decideLoopContinuation`)
+ * turns that into the /goal loop: re-run this stage until the number clears the
+ * bar or the try budget is spent. No new gate kind, no new operator, no new loop —
+ * one new *measurement*, and the parts already in the tree compose into the goal.
+ *
+ * ## Trust boundary (the reason this is an allowlist and not a command field)
+ *
+ * A playbook is workspace data: any member can edit it, and it is fetched by a
+ * daemon that executes on someone's machine. A gate carrying its own command
+ * string would therefore be remote code execution on that machine, dressed as a
+ * board edit. So the two halves are split:
+ *
+ *   - the playbook gate names a metric      → `{ kind: "custom", metric: "lh_perf" }`
+ *   - the operator's own config declares it → `agent.playbooks.metrics.lh_perf`
+ *
+ * A name that is not declared runs nothing and HOLDS with a legible reason. There
+ * is deliberately NO path for gate-supplied arguments to reach the process, and
+ * the command runs in argv form (never a shell), so no value is ever word-split,
+ * glob-expanded, or interpolated. This mirrors `entryActionAllowlist`: a vetted
+ * name maps to a vetted capability, and anything else is refused.
+ *
+ * ## Fail-closed contract
+ *
+ * Every path that does not end in a clean measurement yields `blocked` with a
+ * human-legible `structured.reason` — no metric name, an undeclared name, a
+ * malformed declaration, a non-zero exit, a timeout, unparsable output, or a
+ * missing JSON path. `blocked` evidence can never satisfy a gate (`gateEvaluate`
+ * short-circuits on it even when conditions would otherwise hold), so a broken
+ * measurement can never read as a passing goal. Nothing here throws: the collector
+ * is total, so a flaky tool can't crash the worker.
+ *
+ * The *config-defect* subset of those blocks (no metric name, an undeclared name,
+ * a malformed declaration) additionally carries the `GATE_CONFIG_ERROR_MARK`, so
+ * the advancement engine holds the card instead of re-running the stage against
+ * inputs that cannot change (#823 — see `gate-config-error.ts`).
+ *
+ * ## Process supervision (#823)
+ *
+ * The measurement runs as a **process-group leader** (`spawnInGroup`), never as a
+ * bare child. The headline use case forks: `lighthouse` starts Chrome, so the
+ * plain `execFileSync` this collector shipped with (#690) SIGTERM'd only the
+ * direct child on a timeout and left the browser holding its debug port and CPU —
+ * once per converge-loop iteration, with no reaper. Every exit path here now
+ * escalates over the whole group (`terminateGroup`) and sweeps the group by its
+ * recorded pgid afterwards (`reapGroup`), so a timed-out measurement leaves no
+ * descendants behind.
+ *
+ * The run is also genuinely **asynchronous**. The daemon is one Node process: the
+ * realtime watcher, the `/health` endpoint, the reconcile heartbeat, the merge
+ * monitor and every pool worker share its event loop. A synchronous
+ * `execFileSync` froze all of them for the length of the measurement — up to
+ * fifteen minutes for a Lighthouse run. `collect()` was already `async` and every
+ * caller up to the worker already awaited it, so awaiting a spawned child here
+ * keeps the daemon responsive for free.
+ */
+
+import type { ChildProcess } from "node:child_process";
+import type {
+  GateEvidence,
+  GateEvidenceCollector,
+  GateEvidenceContext,
+  GateKind,
+} from "@harmony/shared";
+import { resolvePath } from "@harmony/shared";
+import {
+  DEFAULT_METRIC_TIMEOUT_MS,
+  type PlaybookMetricDef,
+} from "./exec-types.js";
+import { GATE_CONFIG_ERROR_MARK } from "./gate-config-error.js";
+import { log } from "./log.js";
+import { reapGroup, spawnInGroup, terminateGroup } from "./process-group.js";
+
+const TAG = "command-metric";
+
+/** Cap on captured stdout kept in the evidence row — the doc is a jsonb column,
+ * and a real tool (a full Lighthouse report) prints far more than is useful as a
+ * diagnostic. The parsed `value` is the signal; `raw` is only for reading later. */
+const MAX_RAW_CHARS = 2000;
+
+/** Same cap the other verification primitives use for a child process's output. */
+const MAX_OUTPUT_BUFFER = 10 * 1024 * 1024;
+
+/** Cap on captured stderr. It only ever feeds a truncated failure reason, so
+ * buffering a runaway error stream would cost memory for no diagnostic gain. */
+const MAX_STDERR_CHARS = 64 * 1024;
+
+/** Hard ceiling on a metric's timeout, whatever the operator declared. The run no
+ * longer blocks the daemon's event loop (#823), but a stage still waits on it: an
+ * unbounded value strands the card in its column with a live session and a held
+ * worker slot, and no board-facing reason. 15 min is well past any real measurement
+ * tool and still bounded. */
+const MAX_METRIC_TIMEOUT_MS = 900_000;
+
+/** Grace windows for the escalating group termination on a timeout / overflow.
+ * A measurement tool gets a moment to shut down cleanly (Chrome flushes its
+ * profile) before the group is SIGKILLed — the same SIGINT → SIGTERM → SIGKILL
+ * ladder the Claude subprocess uses, just with shorter waits: nothing here has
+ * uncommitted work worth a long drain. */
+const METRIC_SIGINT_GRACE_MS = 2_000;
+const METRIC_SIGTERM_GRACE_MS = 3_000;
+
+/** How long to wait after the leader exits for its stdio pipes to deliver the
+ * tail of the output. Bounded on purpose: a grandchild that left the group with
+ * its own `setsid()` keeps the pipe open forever, and a measurement that already
+ * produced its exit code must not wait on it. */
+const STDIO_DRAIN_GRACE_MS = 500;
+
+/** The value kinds a metric may resolve to. Non-primitives are refused — a
+ * condition can only compare primitives, so an object here is a config error
+ * worth naming rather than a silently-failing gate. */
+export type MetricValue = string | number | boolean | null;
+
+/** A parsed, validated `parse` directive. */
+export type ParseMode =
+  | { kind: "number" }
+  | { kind: "json"; path: string }
+  | { kind: "invalid"; reason: string };
+
+/**
+ * Parse the `parse` directive of a metric declaration. Pure + total.
+ *
+ * `"number"` reads the whole trimmed stdout as a number; `"json:<dot.path>"`
+ * reads stdout as JSON and resolves the path. An unrecognized directive is
+ * `invalid` — never silently treated as one of the two, because guessing here
+ * would turn a typo into a gate that measures the wrong thing.
+ */
+export function parseParseMode(parse: unknown): ParseMode {
+  if (typeof parse !== "string" || parse.length === 0) {
+    return { kind: "invalid", reason: "`parse` must be a non-empty string" };
+  }
+  if (parse === "number") return { kind: "number" };
+  if (parse.startsWith("json:")) {
+    const path = parse.slice("json:".length).trim();
+    if (!path) {
+      return { kind: "invalid", reason: '`parse` "json:" is missing a path' };
+    }
+    return { kind: "json", path };
+  }
+  return {
+    kind: "invalid",
+    reason: `unknown \`parse\` mode "${parse}" (expected "number" or "json:<path>")`,
+  };
+}
+
+/** Outcome of reading a value out of a command's stdout. */
+export type MetricParseResult =
+  | { ok: true; value: MetricValue }
+  | { ok: false; reason: string };
+
+/**
+ * Extract the metric value from captured stdout. Pure + total — the one place
+ * output becomes a number, so the collector body stays about process control.
+ *
+ * A `number` parse rejects empty/NaN output rather than coercing it: `Number("")`
+ * is 0, and a metric that silently reads 0 would fail a `gte` gate for a reason
+ * no one could see. `json` rejects a resolved value that is `undefined` (path
+ * absent) or non-primitive (an object can't be compared by any operator).
+ */
+export function parseMetricValue(
+  mode: ParseMode,
+  stdout: string,
+): MetricParseResult {
+  if (mode.kind === "invalid") return { ok: false, reason: mode.reason };
+
+  if (mode.kind === "number") {
+    const trimmed = stdout.trim();
+    if (!trimmed) {
+      return {
+        ok: false,
+        reason: "command produced no output to read a number from",
+      };
+    }
+    const value = Number(trimmed);
+    if (!Number.isFinite(value)) {
+      return {
+        ok: false,
+        reason: `command output is not a finite number: ${JSON.stringify(truncate(trimmed, 120))}`,
+      };
+    }
+    return { ok: true, value };
+  }
+
+  let doc: unknown;
+  try {
+    doc = JSON.parse(stdout);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return { ok: false, reason: `command output is not valid JSON: ${msg}` };
+  }
+  // Reuse the evaluator's own resolver: same dot-path semantics a condition uses,
+  // and it never descends into the prototype chain.
+  const resolved = resolvePath(doc, mode.path);
+  if (resolved === undefined) {
+    return {
+      ok: false,
+      reason: `JSON path "${mode.path}" is absent in the command output`,
+    };
+  }
+  if (
+    resolved !== null &&
+    typeof resolved !== "number" &&
+    typeof resolved !== "string" &&
+    typeof resolved !== "boolean"
+  ) {
+    return {
+      ok: false,
+      reason: `JSON path "${mode.path}" resolved to a ${Array.isArray(resolved) ? "array" : typeof resolved}, which no gate operator can compare`,
+    };
+  }
+  return { ok: true, value: resolved };
+}
+
+/** The one argument shape a metric runner receives. */
+export interface MetricCommandRun {
+  command: string;
+  args: string[];
+  cwd: string;
+  timeoutMs: number;
+}
+
+/** Deps the command collector needs. Injected so the collector is unit-testable
+ * with a fake runner instead of a real subprocess. */
+export interface CommandMetricDeps {
+  /** Where the command runs — the card's isolated checkout, never the user's tree. */
+  worktreePath: string;
+  /** The operator's allowlist, keyed by metric name (`agent.playbooks.metrics`). */
+  metrics: Record<string, PlaybookMetricDef>;
+  /**
+   * Inject for tests; defaults to {@link runMetricCommand}. May be sync or async —
+   * the collector awaits it either way, so the many fakes that just return a string
+   * stay valid while the real runner is a spawned child.
+   */
+  runCommand?: (args: MetricCommandRun) => string | Promise<string>;
+}
+
+/**
+ * Run the measurement for real and resolve its stdout. Rejects on a non-zero exit,
+ * a timeout, an output-cap overflow, or a missing binary — the collector turns any
+ * of those into `blocked` via {@link describeRunFailure}.
+ *
+ * Three properties matter here (#823):
+ *
+ *   1. **Group leader.** `spawnInGroup` gives the child its own process group, so
+ *      a timeout signals the whole tree rather than just the direct child. A
+ *      `lighthouse` that forked Chrome dies with its browser.
+ *   2. **Reaped on every path.** The pgid is recorded at spawn and swept with
+ *      `reapGroup` when the promise settles — clean exit included, because a tool
+ *      that backgrounded a helper leaves it reparented but still in the group.
+ *   3. **Non-blocking.** Nothing here occupies the event loop, so the daemon's
+ *      watcher, health endpoint and other workers keep running during a long
+ *      measurement.
+ *
+ * The rejection objects deliberately mimic `execFileSync`'s error shape
+ * (`code: "ETIMEDOUT" | "ENOBUFS" | "ENOENT"`, `status`, `stderr`) so the failure
+ * description stays one pure function shared with any injected runner.
+ */
+export function runMetricCommand(args: MetricCommandRun): Promise<string> {
+  return new Promise<string>((resolve, reject) => {
+    let child: ChildProcess;
+    try {
+      child = spawnInGroup(args.command, args.args, {
+        cwd: args.cwd,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+    } catch (err) {
+      // A synchronous spawn throw (unusable cwd, bad argv) — no group exists yet.
+      reject(err);
+      return;
+    }
+
+    // Record the pgid AT SPAWN: `reapGroup` addresses the group by number, so it
+    // still sweeps stragglers once the leader handle is spent.
+    const pgid = child.pid;
+    const chunks: Buffer[] = [];
+    let stdoutBytes = 0;
+    let stderr = "";
+    let settled = false;
+    let killReason: "timeout" | "overflow" | null = null;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    let drainTimer: ReturnType<typeof setTimeout> | undefined;
+
+    /** Settle once, and never without sweeping the group. */
+    const settle = (failure: Error | null): void => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      if (drainTimer) clearTimeout(drainTimer);
+      reapGroup(pgid);
+      if (failure) reject(failure);
+      else resolve(Buffer.concat(chunks).toString("utf8"));
+    };
+
+    /**
+     * Escalate SIGINT → SIGTERM → SIGKILL across the GROUP, then settle. The
+     * settle is deferred until termination returns so the collector's caller can
+     * rely on the tree being gone once `collect()` resolves — the acceptance
+     * property of this card, and the reason the `exit` handler below yields to
+     * this path once a kill is in flight.
+     */
+    const killTree = (reason: "timeout" | "overflow"): void => {
+      if (settled || killReason) return;
+      killReason = reason;
+      terminateGroup(child, {
+        sigintTimeoutMs: METRIC_SIGINT_GRACE_MS,
+        sigtermTimeoutMs: METRIC_SIGTERM_GRACE_MS,
+      })
+        .catch(() => {
+          // terminateGroup swallows its own signal errors; guard anyway so a
+          // rejection can never strand the promise unsettled.
+        })
+        .then(() => {
+          settle(
+            reason === "timeout"
+              ? Object.assign(
+                  new Error(`command timed out after ${args.timeoutMs}ms`),
+                  { code: "ETIMEDOUT" },
+                )
+              : Object.assign(new Error("stdout maxBuffer exceeded"), {
+                  code: "ENOBUFS",
+                }),
+          );
+        });
+    };
+
+    child.stdout?.on("data", (chunk: Buffer) => {
+      stdoutBytes += chunk.length;
+      if (stdoutBytes > MAX_OUTPUT_BUFFER) {
+        // Match `execFileSync`'s maxBuffer contract: stop buffering and kill.
+        killTree("overflow");
+        return;
+      }
+      chunks.push(chunk);
+    });
+    child.stderr?.on("data", (chunk: Buffer) => {
+      if (stderr.length >= MAX_STDERR_CHARS) return;
+      stderr += chunk.toString("utf8");
+    });
+
+    // A missing binary surfaces as an async `error` event here (an async spawn
+    // never throws it), carrying the same `ENOENT` code execFileSync threw.
+    child.once("error", (err) => settle(err));
+
+    /** Settle from a recorded exit — the leader is gone, the outcome is known. */
+    const settleFromExit = (
+      code: number | null,
+      signal: NodeJS.Signals | null,
+    ): void => {
+      if (drainTimer) clearTimeout(drainTimer);
+      if (code === 0) {
+        settle(null);
+        return;
+      }
+      // A signal we did NOT send (an operator's `kill`, an OOM killer). Reported
+      // as a plain failure, never as a timeout: `signal` is left off the error so
+      // it can't trip `describeRunFailure`'s legacy SIGTERM heuristic and blame a
+      // clock that had nothing to do with it.
+      const detail = signal
+        ? `terminated by signal ${signal}`
+        : `exited ${code}`;
+      settle(
+        Object.assign(new Error(`command ${detail}`), {
+          status: typeof code === "number" ? code : null,
+          stderr,
+        }),
+      );
+    };
+
+    child.once("exit", (code, signal) => {
+      if (killReason) return; // the kill path owns this outcome
+      // The timeout governs the command's RUNTIME, which is now over. Disarm it so
+      // a slow pipe drain can't turn a finished measurement into a reported
+      // timeout; `drainTimer` below bounds what remains.
+      if (timer) clearTimeout(timer);
+      // Reap HERE, before waiting on the pipes. A grandchild the command
+      // backgrounded inherits the stdout pipe, so `close` cannot fire while it
+      // lives — waiting for it first would hang the measurement until the timeout
+      // (which is what a plain `execFileSync` did). Killing the group both stops
+      // that leak and releases the pipe.
+      reapGroup(pgid);
+      // Then give the drained pipes a moment to deliver the tail, so a fast
+      // command can't lose output between `exit` and the last `data` event. The
+      // grace is bounded because a grandchild that escaped the group with its own
+      // `setsid()` would hold the pipe open forever.
+      drainTimer = setTimeout(
+        () => settleFromExit(code, signal),
+        STDIO_DRAIN_GRACE_MS,
+      );
+      child.once("close", () => settleFromExit(code, signal));
+    });
+
+    timer = setTimeout(() => killTree("timeout"), args.timeoutMs);
+  });
+}
+
+/**
+ * `custom` — measure an operator-allowlisted command and expose its value for the
+ * gate's conditions to judge (#690).
+ *
+ * structured shape (predicate paths address into this):
+ *   passed:  { metric: string, value: number|string|boolean|null, raw: string }
+ *   blocked: { metric: string|null, reason: string }
+ *
+ * `result` is about whether a MEASUREMENT was obtained, not whether it is good
+ * enough — the conditions own the bar. So a clean run is `passed` even for a bad
+ * number, and `{ path: "value", op: "gte", value: 90 }` is what fails the gate.
+ * A `custom` gate with no conditions therefore means "this command exits 0",
+ * which is a useful smoke-test gate in its own right.
+ */
+export class CommandMetricCollector implements GateEvidenceCollector {
+  readonly kind: GateKind = "custom";
+  constructor(private readonly deps: CommandMetricDeps) {}
+
+  async collect(context: GateEvidenceContext): Promise<GateEvidence> {
+    // ---- Configuration defects (#823) --------------------------------------
+    // Everything down to the timeout clamp is decided by static config alone: the
+    // gate's own text and the operator's declaration. Re-running the stage cannot
+    // change any of it, so these blocks are marked non-retryable and the
+    // advancement engine holds the card instead of burning its attempt budget.
+    const name = context.gate.metric;
+    if (typeof name !== "string" || name.length === 0) {
+      return configError(
+        null,
+        'Gate kind "custom" needs a `metric` name naming an allowlisted command (e.g. { "kind": "custom", "metric": "lighthouse_performance" }).',
+      );
+    }
+
+    // The allowlist check. An undeclared name never runs anything — this is the
+    // fail-closed boundary, so the reason names the config key an operator edits.
+    const def = Object.hasOwn(this.deps.metrics, name)
+      ? this.deps.metrics[name]
+      : undefined;
+    if (!def) {
+      return configError(
+        name,
+        `Metric "${name}" is not declared in this daemon's allowlist — add it under \`agent.playbooks.metrics\` to permit it.`,
+      );
+    }
+    if (typeof def.command !== "string" || def.command.length === 0) {
+      return configError(name, `Metric "${name}" declares no \`command\`.`);
+    }
+    const mode = parseParseMode(def.parse);
+    if (mode.kind === "invalid") {
+      return configError(name, `Metric "${name}": ${mode.reason}.`);
+    }
+
+    // Clamped, not merely validated. The measurement no longer blocks the daemon's
+    // event loop, but the card's stage waits on it with a live session and a held
+    // worker slot, so the operator's number is honoured only up to a ceiling.
+    const requested =
+      typeof def.timeoutMs === "number" && def.timeoutMs > 0
+        ? Math.floor(def.timeoutMs)
+        : DEFAULT_METRIC_TIMEOUT_MS;
+    const timeoutMs = Math.min(requested, MAX_METRIC_TIMEOUT_MS);
+    if (timeoutMs < requested) {
+      // Say so: otherwise the blocked reason quotes a duration that appears
+      // nowhere in the operator's config, and they go looking for a typo.
+      log.warn(
+        TAG,
+        `Metric "${name}" declares timeoutMs=${requested}, clamped to the ${MAX_METRIC_TIMEOUT_MS}ms ceiling`,
+      );
+    }
+    const run = this.deps.runCommand ?? runMetricCommand;
+
+    let stdout: string;
+    try {
+      stdout = await run({
+        command: def.command,
+        args: def.args ?? [],
+        cwd: this.deps.worktreePath,
+        timeoutMs,
+      });
+    } catch (err) {
+      // Non-zero exit, timeout, or a missing binary: no measurement exists, so we
+      // cannot judge the goal. `blocked` (not `failed`) is the honest signal, and
+      // it can never satisfy the gate. NOT marked as a config error: unlike an
+      // undeclared metric, a run failure can genuinely differ on the next attempt
+      // (the tool is flaky, the branch under measurement changes), so it keeps the
+      // ordinary gate-unmet retry path.
+      const reason = describeRunFailure(err, timeoutMs);
+      log.warn(
+        TAG,
+        `Metric "${name}" did not produce a measurement: ${reason}`,
+      );
+      return blocked(name, reason);
+    }
+
+    const parsed = parseMetricValue(mode, stdout);
+    if (!parsed.ok) {
+      log.warn(TAG, `Metric "${name}" output unusable: ${parsed.reason}`);
+      return blocked(name, `Metric "${name}": ${parsed.reason}.`, stdout);
+    }
+
+    log.info(TAG, `Metric "${name}" measured: ${JSON.stringify(parsed.value)}`);
+    return {
+      result: "passed",
+      structured: {
+        metric: name,
+        value: parsed.value,
+        raw: truncate(stdout, MAX_RAW_CHARS),
+      },
+    };
+  }
+}
+
+/** Build the `blocked` evidence shape, with the reason a human will actually read. */
+function blocked(
+  metric: string | null,
+  reason: string,
+  raw?: string,
+): GateEvidence {
+  return {
+    result: "blocked",
+    structured: {
+      metric,
+      reason,
+      ...(raw === undefined ? {} : { raw: truncate(raw, MAX_RAW_CHARS) }),
+    },
+  };
+}
+
+/**
+ * A `blocked` that also says "re-running cannot help" (#823). Reserved for defects
+ * decided entirely by static configuration — an absent metric name, an undeclared
+ * one, a declaration with no `command` or an unknown `parse` mode. The advancement
+ * engine holds the card on these instead of spending a full stage re-run (and a
+ * give-up attempt) to compute the identical answer.
+ */
+function configError(metric: string | null, reason: string): GateEvidence {
+  const evidence = blocked(metric, reason);
+  return {
+    ...evidence,
+    structured: { ...evidence.structured, ...GATE_CONFIG_ERROR_MARK },
+  };
+}
+
+/**
+ * Turn an `execFileSync` throw into a legible reason. A timeout surfaces as an
+ * ETIMEDOUT/SIGTERM child rather than an exit code, so it is named explicitly —
+ * "timed out after 5m" is actionable where "command failed" is not.
+ */
+function describeRunFailure(err: unknown, timeoutMs: number): string {
+  const e = err as {
+    code?: unknown;
+    signal?: unknown;
+    status?: unknown;
+    stderr?: unknown;
+    message?: unknown;
+  };
+  // ORDER MATTERS: a `maxBuffer` overflow is also SIGTERM'd (`{ code: "ENOBUFS",
+  // signal: "SIGTERM" }`), so it must be named before the timeout branch or a tool
+  // that simply prints too much reads as a slow one. That is not hypothetical for
+  // this card: a full Lighthouse JSON report is the headline use case and the
+  // biggest output any of these commands produces, so the misreport would land
+  // exactly where it costs the most — sending an operator after a performance
+  // problem that does not exist.
+  if (e?.code === "ENOBUFS") {
+    return `command produced more than ${MAX_OUTPUT_BUFFER} bytes of output (use a flag that prints only the metric, or parse a smaller report)`;
+  }
+  if (e?.code === "ETIMEDOUT" || e?.signal === "SIGTERM") {
+    return `command timed out after ${timeoutMs}ms`;
+  }
+  if (e?.code === "ENOENT") {
+    return "command not found on PATH";
+  }
+  const stderr =
+    typeof e?.stderr === "string"
+      ? e.stderr
+      : e?.stderr instanceof Buffer
+        ? e.stderr.toString("utf8")
+        : "";
+  const detail = stderr.trim() || String(e?.message ?? err);
+  const status = typeof e?.status === "number" ? e.status : null;
+  return status === null
+    ? `command failed: ${truncate(detail, 400)}`
+    : `command exited ${status}: ${truncate(detail, 400)}`;
+}
+
+/** Trim a string to `max` chars, marking that it was cut. */
+function truncate(value: string, max: number): string {
+  return value.length <= max ? value : `${value.slice(0, max)}…[truncated]`;
+}

package/src/error-classifier.ts

@@ -0,0 +1,95 @@
+/**
+ * Classify a failed Claude CLI run into an actionable error class.
+ *
+ * The implement worker captures the CLI's stderr and folds it into the
+ * rejection message (`claude exited with code N: <stderr>`). When a run dies
+ * from an Anthropic API condition — auth, billing, rate limit, usage cap — the
+ * CLI exits non-zero with a recognisable message. Without classification the
+ * worker treats every such failure as a generic crash, counts it against the
+ * card's give-up budget, and (historically) stranded the card. Classifying
+ * lets the worker requeue WITHOUT burning an attempt and lets the pool back off
+ * so it doesn't hot-loop the same failing call across every in-flight card.
+ *
+ * `kind: null` means "not a recognised API/quota error" — treat it as a generic
+ * crash (requeue + count the attempt, same as a timeout).
+ */
+export type ApiErrorKind =
+  | "rate_limit"
+  | "out_of_credits"
+  | "usage_limit"
+  | "auth";
+
+export interface RunErrorClass {
+  kind: ApiErrorKind | null;
+  /** Parsed from a `Retry-After` hint when present (milliseconds). */
+  retryAfterMs?: number;
+}
+
+// Order matters: "usage limit" and "credit balance" both contain tokens that
+// the rate-limit patterns would otherwise swallow, so the more specific
+// billing/usage checks run before the generic rate-limit check.
+const AUTH =
+  /\b401\b|invalid x-api-key|authentication_error|unauthorized|oauth token (?:has )?expired|please run .*login|invalid bearer token/i;
+const OUT_OF_CREDITS =
+  /\b402\b|credit balance is too low|insufficient (?:funds|credit|balance)|billing|payment required|purchase more credits/i;
+const USAGE_LIMIT =
+  /usage limit|daily limit|monthly limit|quota (?:exceeded|reached)|reached your .{0,20}limit|usage_limit_reached|limit will reset/i;
+const RATE_LIMIT =
+  /\b429\b|\b529\b|rate[ _-]?limit|too many requests|overloaded_error|"type"\s*:\s*"overloaded"/i;
+
+/** Best-effort `Retry-After: <seconds>` extraction. */
+function parseRetryAfterMs(message: string): number | undefined {
+  const match = message.match(/retry[- ]?after["':\s]+(\d+)/i);
+  if (!match) return undefined;
+  const seconds = Number(match[1]);
+  if (!Number.isFinite(seconds) || seconds <= 0) return undefined;
+  return seconds * 1000;
+}
+
+export function classifyRunError(message: string): RunErrorClass {
+  if (!message) return { kind: null };
+  const retryAfterMs = parseRetryAfterMs(message);
+
+  // Auth first: a bad/expired key fails every card identically, so the pool
+  // pauses wholesale rather than churning. Check before billing because some
+  // 401 bodies also mention "billing".
+  if (AUTH.test(message)) return { kind: "auth", retryAfterMs };
+  if (OUT_OF_CREDITS.test(message))
+    return { kind: "out_of_credits", retryAfterMs };
+  if (USAGE_LIMIT.test(message)) return { kind: "usage_limit", retryAfterMs };
+  if (RATE_LIMIT.test(message)) return { kind: "rate_limit", retryAfterMs };
+
+  return { kind: null };
+}
+
+/** Human-readable, board-facing reason for an API error class. */
+export function describeApiError(kind: ApiErrorKind): string {
+  switch (kind) {
+    case "auth":
+      return "Anthropic auth error — agent paused, check API credentials";
+    case "out_of_credits":
+      return "Anthropic credit balance too low — retrying after top-up";
+    case "usage_limit":
+      return "Anthropic usage limit reached — retrying after reset";
+    case "rate_limit":
+      return "Anthropic rate limit hit — retrying shortly";
+  }
+}
+
+/**
+ * Default daemon-wide cooldown for an API error class, used when the API gave
+ * no explicit `Retry-After`. Billing/usage conditions reset slowly (or need a
+ * human), so they back off longer; transient rate limits clear fast.
+ */
+export function cooldownMsFor(kind: ApiErrorKind): number {
+  switch (kind) {
+    case "rate_limit":
+      return 60_000; // 1 min — transient, clears fast
+    case "usage_limit":
+      return 15 * 60_000; // 15 min — resets on a schedule
+    case "out_of_credits":
+      return 30 * 60_000; // 30 min — usually needs a human top-up
+    case "auth":
+      return 30 * 60_000; // fallback; pool pauses wholesale on auth anyway
+  }
+}