catherd-cli 1.4.0 → 2.0.0-beta.20261022113309

package/src/entry/scorecard-command.ts

@@ -0,0 +1,187 @@
+import { join, resolve } from "node:path";
+import { defineCommand } from "citty";
+import { CatherdError } from "../domain/errors.ts";
+import {
+  passBar,
+  SCORECARD_TASKS,
+  type ScorecardFile,
+  type ScorecardTaskId,
+  type Verdict,
+} from "../domain/scorecard.ts";
+import {
+  defaultPreparedRoot,
+  loadTask,
+  type Prepared,
+  prepareTask,
+  readScorecard,
+  resultsFile,
+  saveTaskScore,
+  scoreTask,
+  scorecardRoot,
+} from "../services/scorecard.ts";
+import { JSON_ARG, mark, printJson, refuseInRole } from "./cli-kit.ts";
+
+const BASELINE = "1.5.0";
+
+/** POSIX single-quote escaping; a value of plain path characters stays bare. */
+const quote = (v: string): string => (/^[\w@%+=:,./-]+$/.test(v) ? v : `'${v.replaceAll("'", `'\\''`)}'`);
+
+/** The prepared task as the operator needs it: where to start, what to paste, and how to record. */
+export function formatPrepared(p: Prepared, label = "<label>"): string[] {
+  const env =
+    p.limits && p.shim ? `CATHERD_SCORECARD_LIMITS=${quote(p.limits)} PATH=${quote(p.shim)}:"$PATH" ` : "";
+  return [
+    `${mark("ok")} ${p.task} prepared in ${p.dir} at ${p.startCommit.slice(0, 8)}`,
+    `  start:  cd ${quote(p.repo)} && ${env}claude`,
+    `  prompt: ${p.prompt}`,
+    `  record: catherd scorecard record ${quote(p.dir)} --label ${label}`,
+  ];
+}
+
+const yes = (b: boolean): string => (b ? "yes" : "no");
+const num = (n: number): string => String(Math.round(n * 10) / 10);
+const k = (n: number): string => `${Math.round(n / 1000)}k`;
+
+function row(id: ScorecardTaskId, current: ScorecardFile, baseline: ScorecardFile | null): string {
+  const s = current.tasks[id];
+  if (!s) return `${id}  -`;
+  const was = baseline?.tasks[id];
+  const input = s.quota.backends.reduce((n, b) => n + b.input, 0);
+  const output = s.quota.backends.reduce((n, b) => n + b.output, 0);
+  return [
+    id,
+    s.catherdVersion,
+    `finished ${yes(s.finished)}`,
+    `interventions ${s.interventions.total}`,
+    `rounds ${s.reviewRounds}`,
+    `wall ${num(s.wallMinutes)} min (${was ? num(was.wallMinutes) : "?"} min)`,
+    `first try ${yes(s.firstTry)}`,
+    `quota ${k(input)}/${k(output)}`,
+  ].join("  ");
+}
+
+export function formatComparison(
+  current: ScorecardFile,
+  baseline: ScorecardFile | null,
+  v: Verdict,
+): string[] {
+  return [
+    `scorecard ${current.label} against ${baseline ? baseline.label : "no baseline"}`,
+    ...SCORECARD_TASKS.map((id) => row(id, current, baseline)),
+    ...(v.pass ? ["PASS"] : v.failures.map((f) => `FAIL: ${f}`)),
+  ];
+}
+
+/** The contributor commands read fixtures from the checkout; an installed package ships none. */
+function checkoutRoot(): string {
+  const root = scorecardRoot();
+  if (!root)
+    throw new CatherdError("E_INPUT_INVALID", "this catherd is not a source checkout", {
+      fix: "run it from a catherd checkout: bun src/cli.ts scorecard …",
+    });
+  return root;
+}
+
+const stamp = (d: Date): string => d.toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
+
+const RESULTS_ARG = {
+  results: { type: "string", description: "results folder (default: scorecard/results)" },
+} as const;
+const resultsDir = (root: string, given: string | undefined): string =>
+  given ? resolve(given) : join(root, "results");
+
+const prepare = defineCommand({
+  meta: { name: "prepare", description: `Prepare a task's starting point: ${SCORECARD_TASKS.join(", ")}` },
+  args: {
+    task: { type: "positional", required: true, description: SCORECARD_TASKS.join("|") },
+    source: {
+      type: "string",
+      description: "the repository to clone (a path or URL); the Sanitell task needs one",
+    },
+    dir: { type: "string", description: "where prepared tasks go (default: catherd's data dir)" },
+    setup: {
+      type: "boolean",
+      default: true,
+      description: "run the task's setup commands (--no-setup skips them)",
+    },
+    ...JSON_ARG,
+  },
+  async run({ args }) {
+    refuseInRole(process.env, "catherd scorecard prepare");
+    const root = checkoutRoot();
+    const task = loadTask(args.task, root);
+    const now = new Date();
+    const into = join(args.dir ? resolve(args.dir) : defaultPreparedRoot(), `${task.id}-${stamp(now)}`);
+    const p = await prepareTask({ task, root, into, source: args.source, now, setup: args.setup });
+    if (args.json) return printJson(p);
+    for (const line of formatPrepared(p)) console.log(line);
+  },
+});
+
+const record = defineCommand({
+  meta: { name: "record", description: "Score a prepared task's run and store its row under a label" },
+  args: {
+    dir: { type: "positional", required: true, description: "the folder `prepare` printed" },
+    label: {
+      type: "string",
+      required: true,
+      description: "a version or plan name: 1.5.0, plan-29, 2.0.0-beta.1",
+    },
+    run: { type: "string", description: "the run to score, when the folder holds more than one" },
+    chat: { type: "string", description: "messages you typed into the session after the start prompt" },
+    ...RESULTS_ARG,
+    ...JSON_ARG,
+  },
+  async run({ args }) {
+    refuseInRole(process.env, "catherd scorecard record");
+    const chat = args.chat === undefined ? 0 : Number(args.chat);
+    if (!Number.isInteger(chat) || chat < 0)
+      throw new CatherdError("E_INPUT_INVALID", `--chat ${args.chat} is not a count`, {
+        fix: "--chat <the number of messages you typed into the session>",
+      });
+    const root = checkoutRoot();
+    const out = resultsFile(resultsDir(root, args.results), args.label);
+    const s = await scoreTask({ dir: resolve(args.dir), root, run: args.run, chat, now: Date.now() });
+    const saved = saveTaskScore(out, args.label, s);
+    if (args.json) return printJson({ file: out, score: s });
+    const baseline = readScorecard(resultsFile(resultsDir(root, args.results), BASELINE));
+    console.log(
+      formatComparison(saved, baseline, passBar(saved, baseline))[1 + SCORECARD_TASKS.indexOf(s.task)],
+    );
+    console.log(out);
+  },
+});
+
+const compare = defineCommand({
+  meta: {
+    name: "compare",
+    description: `Check a label against the baseline (${BASELINE}) by the spec's pass bar`,
+  },
+  args: {
+    label: { type: "positional", required: true, description: "the label to check" },
+    baseline: { type: "string", description: `the label to compare with (default: ${BASELINE})` },
+    ...RESULTS_ARG,
+    ...JSON_ARG,
+  },
+  run({ args }) {
+    const dir = resultsDir(checkoutRoot(), args.results);
+    const current = readScorecard(resultsFile(dir, args.label));
+    if (!current)
+      throw new CatherdError("E_INPUT_INVALID", `no scorecard ${args.label} yet`, {
+        fix: `catherd scorecard record <dir> --label ${args.label}`,
+      });
+    const baseline = readScorecard(resultsFile(dir, args.baseline ?? BASELINE));
+    const verdict = passBar(current, baseline);
+    process.exitCode = verdict.pass ? 0 : 1;
+    if (args.json) return printJson({ current, baseline, verdict });
+    for (const line of formatComparison(current, baseline, verdict)) console.log(line);
+  },
+});
+
+export const scorecardCommand = defineCommand({
+  meta: {
+    name: "scorecard",
+    description: "Contributors: measure a catherd version on four fixed tasks against the 1.5.0 baseline",
+  },
+  subCommands: { prepare, record, compare },
+});

package/src/entry/supervise-bin.ts

@@ -1,5 +1,5 @@
 #!/usr/bin/env bun
-// The detached supervisor's entry: it imports only what supervising needs, never src/cli.ts and its TUI.
+// The detached supervisor's entry: it imports only what supervising needs, never src/cli.ts.
 import { errorMessage } from "../domain/errors.ts";
 import { runSupervise } from "./supervise-command.ts";
 

package/src/entry/supervise-command.ts

@@ -1,8 +1,11 @@
 import { defineCommand } from "citty";
 import { adapterFor } from "../adapters/registry.ts";
 import { workerEnv } from "../infra/env.ts";
+import { log } from "../infra/log.ts";
+import { loginEnv } from "../infra/login-env.ts";
 import { readVersioned } from "../infra/store.ts";
 import { SuperviseSpecSchema, supervise } from "../infra/supervisor.ts";
+import { resolveRefs } from "../services/gate-env.ts";
 import "../adapters/all.ts";
 
 /**
@@ -10,22 +13,46 @@ import "../adapters/all.ts";
  * spec.json holds only the adapter's env overrides; the worker's env is built here, from this process's
  * own inherited env (already without catherd's secrets), so no credential is ever on disk (spec §10.4).
  */
+/**
+ * The worker's env: this process's env, the gate env's references (plan 23: read here from this env, else the
+ * login env), then the spec's own env last, so no reference ever takes the role's identity, scratch or home.
+ */
+export function superviseEnv(
+  read: { env: Record<string, string>; envFrom?: Record<string, string>; cwd: string; dispatchDir?: string },
+  base: Record<string, string | undefined>,
+  login: () => Record<string, string | undefined>,
+): Record<string, string> {
+  const refs = resolveRefs({}, read.envFrom ?? {}, base, login);
+  if (refs.missing.length) log("warn", "gate-env", { dispatch: read.dispatchDir, unset: refs.missing });
+  return workerEnv(base, { ...refs.env, ...read.env }, read.cwd);
+}
+
 export async function runSupervise(specPath: string): Promise<void> {
   const read = readVersioned(specPath, SuperviseSpecSchema, 1);
-  const spec = { ...read, env: workerEnv(process.env, read.env, read.cwd) };
+  const spec = { ...read, env: superviseEnv(read, process.env, loginEnv) };
   const a = adapterFor(spec.backend);
   const busy = a?.isBusy?.bind(a);
   const stop = a?.interrupt?.bind(a);
-  await supervise(spec, {
+  const retrying = a?.providerRetry?.bind(a);
+  const ended = await supervise(spec, {
     onLine: (line) => {
       const d = a?.parse(line) ?? {};
-      return { final: d.final, thread: d.thread, item: d.item };
+      return { final: d.final, thread: d.thread, item: d.item, step: d.step };
     },
     isBusy: busy
       ? (thread, sinceMs) => (thread ? busy(thread, spec.cwd, sinceMs) : Promise.resolve(false))
       : undefined,
     interrupt: stop ? (thread) => (thread ? stop(thread, spec.cwd) : Promise.resolve()) : undefined,
+    providerRetry: retrying
+      ? (thread, sinceMs) => (thread ? retrying(thread, spec.cwd, sinceMs) : Promise.resolve(null))
+      : undefined,
   });
+  // plan 22: a Codex owner hears of the role from here too, when its thread's MCP server is gone. Loaded only
+  // once the worker ended (a supervisor that lost the lock to another pushes nothing); never throws.
+  if (ended && process.env.CATHERD_NO_END_PUSH !== "1") {
+    const { pushFromEnd } = await import("../services/end-push.ts");
+    await pushFromEnd(spec.dispatchDir);
+  }
 }
 
 export const superviseCommand = defineCommand({

package/src/entry/workspace-command.ts

@@ -0,0 +1,44 @@
+import { resolve } from "node:path";
+import { defineCommand } from "citty";
+import { inspectWorkspace, workspaceStatus } from "../services/workspace-service.ts";
+import { JSON_ARG, printJson } from "./cli-kit.ts";
+import { defaultDeps } from "./deps.ts";
+
+const inspect = defineCommand({
+  meta: { name: "inspect", description: "Read the explicit repositories in a workspace manifest" },
+  args: {
+    root: { type: "positional", description: "workspace root (default: current directory)" },
+    ...JSON_ARG,
+  },
+  async run({ args }) {
+    const view = await inspectWorkspace(resolve(args.root ?? process.cwd()));
+    if (args.json) return printJson(view);
+    console.log(view.root);
+    for (const [name, path] of Object.entries(view.repos)) console.log(`${name}: ${path}`);
+  },
+});
+
+const status = defineCommand({
+  meta: { name: "status", description: "Read a workspace run's children, dependencies and budget" },
+  args: {
+    workspace: { type: "positional", required: true, description: "workspace run id" },
+    ...JSON_ARG,
+  },
+  async run({ args }) {
+    const view = await workspaceStatus(defaultDeps(), args.workspace);
+    if (args.json) return printJson(view);
+    console.log(`${view.workspace.id}: ${view.workspace.title}`);
+    for (const step of view.steps) {
+      const blocked = step.blockedBy.length ? ` (waiting for ${step.blockedBy.join(", ")})` : "";
+      console.log(`${step.id}: ${step.repo} ${step.state}${step.run ? ` ${step.run}` : ""}${blocked}`);
+    }
+    console.log(
+      `spend: ${view.spend.tokens} tokens, ${view.spend.minutes.toFixed(2)} min, $${view.spend.usd.toFixed(2)}`,
+    );
+  },
+});
+
+export const workspaceCommand = defineCommand({
+  meta: { name: "workspace", description: "Inspect workspace members and coordinated runs" },
+  subCommands: { inspect, status },
+});

package/src/infra/boot-lock.ts

@@ -0,0 +1,25 @@
+import { join } from "node:path";
+import { tryLock } from "./filelock.ts";
+import { log } from "./log.ts";
+import { locksDir } from "./paths.ts";
+import { ensurePrivateDir } from "./store.ts";
+
+/** The machine-wide lock the MCP server that runs the boot sync and reconcile holds for its life. */
+export const bootLockTarget = (): string => join(locksDir(), "mcp-boot");
+
+/**
+ * Plan 22 (the three MCP servers of one Codex session): boot sync and reconcile are single-flight across
+ * processes. The first server takes this lock (its pid and start time in it; a dead holder's is reclaimed) and
+ * keeps it until it closes; a second server finds it held and skips both. Returns the release, or null while
+ * another live server holds it. A lock that cannot be taken for another reason never costs a reconcile: the
+ * server runs the boot work, with nothing to release.
+ */
+export function takeBootLock(): (() => void) | null {
+  try {
+    ensurePrivateDir(locksDir());
+    return tryLock(bootLockTarget());
+  } catch (e) {
+    log("warn", "boot", { lock: bootLockTarget(), error: e instanceof Error ? e.message : String(e) });
+    return () => {};
+  }
+}

package/src/infra/delivery.ts

@@ -24,6 +24,9 @@ const AttemptSchema = z
   .refine((a) => a.status !== "accepted" || Boolean(a.msgId?.trim()), "accepted attempt needs a receipt");
 const DeliverySchema = z.looseObject({ schema: z.literal(1), attempts: z.array(AttemptSchema) });
 export type DeliveryAttempt = z.infer<typeof AttemptSchema>;
+/** How a delivery to a role's thread is recorded (spec 1.5 plan 21); `status` warns about it. */
+export const ROLE_THREAD_REFUSAL = "refused: the target is a role's thread";
+
 export type DeliveryState = "pending" | "enqueue-accepted" | "ambiguous" | "collected";
 
 export function readDelivery(dir: string): DeliveryAttempt[] {

package/src/infra/dispatch-dir.ts

@@ -45,6 +45,8 @@ export function dispatchPaths(dir: string) {
     stall: join(dir, "stall.json"),
     /** the message that announced the stall, once sent */
     stallNotified: join(dir, "stall-notified.json"),
+    /** the role's own thread, written by the supervisor once the CLI names it (spec 1.5 plan 21) */
+    thread: join(dir, "thread.json"),
   };
 }
 

package/src/infra/env.ts

@@ -9,14 +9,15 @@ const SECRET_ENV = new Set([
   "CODEX_SESSION_ID",
   "CATHERD_ORCHESTRATION_HOST",
   "TYPESAFE_API_KEY",
-  // spec 1.2 §9: the user's Artificial Analysis key reads scores for catherd alone
-  "ARTIFICIAL_ANALYSIS_API_KEY",
   "CLAUDE_CODE_MESSAGING_SOCKET",
   "CLAUDE_CODE_MESSAGING_TOKEN",
   "CLAUDE_CODE_SESSION_ID",
   "CLAUDE_CODE_HOST_SESSION_ID",
 ]);
 
+/** Whether `name` is one of catherd's own secrets, which no process catherd starts ever gets. */
+export const isCatherdSecret = (name: string): boolean => SECRET_ENV.has(name);
+
 /** `base` without catherd's own secrets and without unset keys; for every process catherd starts. */
 export function scrubSecrets(base: Record<string, string | undefined>): Record<string, string> {
   const env: Record<string, string> = {};
@@ -62,11 +63,18 @@ const CHECK_ENV = new Set([
   "CATHERD_LOCK_SLOTS",
 ]);
 const CHECK_ENV_PATTERN = /^(LC_[A-Z_]+|XDG_[A-Z_]+_(HOME|DIR|DIRS))$/;
+/**
+ * Plan 23: where the toolchains a check runs find their daemon, proxy and caches (DOCKER_HOST for OrbStack or
+ * rootless Docker, a proxy, Go's and pnpm's caches); without them a check that would run fails as if broken.
+ */
+const TOOL_ENV_PATTERN =
+  /^(DOCKER_(HOST|CONTEXT|CONFIG|CERT_PATH|TLS_VERIFY)|TESTCONTAINERS_[A-Z_]+|(HTTPS?|NO|ALL)_PROXY|(https?|no|all)_proxy|GO(PATH|CACHE|MODCACHE|FLAGS|PROXY|PRIVATE|NOSUMDB|TOOLCHAIN)|PNPM_HOME|npm_config_store_dir|BUN_INSTALL(_CACHE_DIR)?|NVM_DIR|JAVA_HOME|CARGO_HOME|RUSTUP_HOME)$/;
 
 /** A strict allowlist of `base` plus PWD, for lane-authored shell text such as preflight checks. */
 export function checkEnv(base: Record<string, string | undefined>, cwd: string): Record<string, string> {
   const env: Record<string, string> = {};
   for (const [k, v] of Object.entries(base))
-    if (v !== undefined && (CHECK_ENV.has(k) || CHECK_ENV_PATTERN.test(k))) env[k] = v;
+    if (v !== undefined && (CHECK_ENV.has(k) || CHECK_ENV_PATTERN.test(k) || TOOL_ENV_PATTERN.test(k)))
+      env[k] = v;
   return { ...env, PWD: cwd };
 }

package/src/infra/git.ts

@@ -54,6 +54,12 @@ export async function gitToplevel(dir: string): Promise<string | null> {
   return r.kind === "ok" ? r.out.trim() || null : null;
 }
 
+/** The repo's `origin` remote URL, or null when it has none (or git cannot say). */
+export async function gitOrigin(repo: string): Promise<string | null> {
+  const r = await git(repo, ["config", "--get", "remote.origin.url"]);
+  return (r.kind === "ok" && r.out.trim()) || null;
+}
+
 /** The short HEAD, or null when git cannot say (no commit, not a repo, failure, timeout). */
 export async function gitHead(repo: string, timeoutMs?: number): Promise<string | null> {
   const r = await git(repo, ["rev-parse", "--short", "HEAD"], timeoutMs);
@@ -89,3 +95,67 @@ export async function statusSnapshot(repo: string, timeoutMs?: number): Promise<
   for (const { xy, path } of parsePorcelainZ(r.out)) snap[path] = fingerprint(repo, xy, path);
   return snap;
 }
+
+/** What a git command printed and how it ended, for the callers that show a failure (a commit hook's output). */
+export interface GitOutput {
+  code: number | null;
+  out: string;
+  err: string;
+  timedOut: boolean;
+}
+
+/**
+ * `git -C repo <args>` with both streams, for a command that may run hooks (a commit): a long timeout, and no
+ * optional lock avoidance, since this one writes the index.
+ */
+export async function gitRun(repo: string, args: string[], timeoutMs = 300_000): Promise<GitOutput> {
+  let p: Bun.Subprocess<"ignore", "pipe", "pipe">;
+  try {
+    p = Bun.spawn(["git", "-C", repo, ...args], {
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+      env: { ...scrubSecrets(process.env), LC_ALL: "C" },
+    });
+  } catch (e) {
+    return { code: null, out: "", err: (e as Error).message, timedOut: false };
+  }
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    p.kill("SIGKILL");
+  }, timeoutMs);
+  const out = new Response(p.stdout).text().catch(() => "");
+  const err = new Response(p.stderr).text().catch(() => "");
+  const code = await p.exited;
+  clearTimeout(timer);
+  return {
+    code: timedOut ? null : code,
+    out: timedOut ? "" : await out,
+    err: timedOut ? "" : await err,
+    timedOut,
+  };
+}
+
+/**
+ * `git fetch -q origin <base>` with a cap: catherd's one network git call (spec 2.0 decision 21). True when it
+ * exits 0; a failure never throws, the caller works from the last fetch and says so.
+ */
+export async function fetchBase(repo: string, base: string, timeoutMs = 60_000): Promise<boolean> {
+  const r = await gitRun(repo, ["fetch", "-q", "origin", base], timeoutMs);
+  return r.code === 0;
+}
+
+/** Whether `commit` is an ancestor of `ref`: true on exit 0, false on 1, null when git cannot tell. */
+export async function isAncestor(repo: string, commit: string, ref: string): Promise<boolean | null> {
+  const r = await git(repo, ["merge-base", "--is-ancestor", commit, ref]);
+  if (r.kind === "ok") return true;
+  return r.kind === "failed" && r.exit === 1 ? false : null;
+}
+
+/** The checked-out branch, or null when HEAD is detached (or git cannot say). */
+export async function currentBranch(repo: string): Promise<string | null> {
+  const r = await git(repo, ["rev-parse", "--abbrev-ref", "HEAD"]);
+  const name = r.kind === "ok" ? r.out.trim() : "";
+  return name && name !== "HEAD" ? name : null;
+}

package/src/infra/heavy-lock.ts

@@ -1,8 +1,39 @@
-import { availableParallelism } from "node:os";
+import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync, rmSync } from "node:fs";
+import { availableParallelism, loadavg } from "node:os";
 import { join } from "node:path";
-import { tryLock } from "./filelock.ts";
+import { roleScopeOfScratch } from "../domain/role-scope.ts";
+import { tryLock, withFileLock } from "./filelock.ts";
 import { locksDir } from "./paths.ts";
-import { ensurePrivateDir } from "./store.ts";
+import { isAlive, selfIdentity } from "./proc.ts";
+import { ensurePrivateDir, writeJsonAtomic } from "./store.ts";
+
+/** Set in a locked command's env: a `catherd lock` inside it runs at once, holding nothing more. */
+export const LOCK_HELD_ENV = "CATHERD_LOCK_HELD";
+
+const envLoadWaitMs = Number(process.env.CATHERD_HEAVY_LOAD_WAIT_MS ?? 900_000);
+
+/**
+ * Spec 2.0 decision 18: heavy slots wait while the load is above twice the cores, up to 15 minutes.
+ * CATHERD_HEAVY_LOAD_WAIT_MS overrides the wait (0 = none); the test preload sets 0, since a busy CI runner
+ * would otherwise stall every lock in the suite, spawned `catherd lock` processes included.
+ */
+export const heavyLimits = {
+  loadWaitMs: envLoadWaitMs >= 0 ? envLoadWaitMs : 900_000,
+  loadPollMs: 15_000,
+  load: (): number => loadavg()[0] ?? 0,
+  cores: (): number => availableParallelism(),
+};
+
+/**
+ * The emptied TMPDIR and GOTMPDIR of one slot: under the role's scratch when the caller's TMPDIR is one (a role
+ * is told by its TMPDIR, so the slot dir must stay under it), else under catherd's locks dir.
+ */
+export function slotTmpDir(slot: number, inherited?: string): string {
+  return roleScopeOfScratch(inherited)
+    ? join(inherited as string, `heavy-${slot}`)
+    : join(locksDir(), "tmp", `slot-${slot}`);
+}
 
 /** A profile's `lock.heavy` as a slot count: a number, or half the cores. */
 export function heavySlots(setting: number | "cpus/2" | undefined): number {
@@ -16,21 +47,117 @@ export function heavySlots(setting: number | "cpus/2" | undefined): number {
  */
 export async function withHeavySlot<T>(
   slots: number,
-  fn: (slot: number) => T | Promise<T>,
-  o: { pollMs?: number } = {},
+  fn: (slot: number, tmp: string) => T | Promise<T>,
+  o: { pollMs?: number; onWait?: () => void; tmpBase?: string } = {},
 ): Promise<T> {
+  const r = await withHeavySlotWithin(slots, Number.POSITIVE_INFINITY, fn, o);
+  if (r.busy) throw new Error("unreachable: an unbounded wait never gives up");
+  return r.value;
+}
+
+/**
+ * withHeavySlot with a wait budget (plan 23): `{ busy: true }` when no slot came free within `waitMs`, so a
+ * caller with a deadline of its own (preflight) reports the lock as busy instead of timing out behind it.
+ */
+export async function withHeavySlotWithin<T>(
+  slots: number,
+  waitMs: number,
+  fn: (slot: number, tmp: string) => T | Promise<T>,
+  o: { pollMs?: number; onWait?: () => void; tmpBase?: string } = {},
+): Promise<{ busy: false; value: T } | { busy: true }> {
   const dir = locksDir();
   ensurePrivateDir(dir);
+  const deadline = Date.now() + waitMs;
+  // an overloaded host runs the command late, not never: the wait is bounded, and the slot loop follows it
+  const loadDeadline = Date.now() + Math.min(heavyLimits.loadWaitMs, waitMs);
+  while (heavyLimits.load() > 2 * heavyLimits.cores() && Date.now() < loadDeadline) {
+    o.onWait?.();
+    await Bun.sleep(Math.min(heavyLimits.loadPollMs, Math.max(1, loadDeadline - Date.now())));
+  }
   for (;;) {
     for (let i = 0; i < slots; i++) {
       const release = tryLock(join(dir, `slot-${i}`));
       if (!release) continue;
+      const tmp = slotTmpDir(i, o.tmpBase);
       try {
-        return await fn(i);
+        rmSync(tmp, { recursive: true, force: true });
+        ensurePrivateDir(tmp);
+        return { busy: false, value: await fn(i, tmp) };
       } finally {
+        rmSync(tmp, { recursive: true, force: true });
         release();
       }
     }
+    const left = deadline - Date.now();
+    if (left <= 0) return { busy: true };
+    o.onWait?.();
+    await Bun.sleep(Math.min(o.pollMs ?? 500, left));
+  }
+}
+
+/** A process inside a role lock: who, so a dead one is dropped, and a token, so one process can hold twice. */
+interface RoleHolder {
+  pid: number;
+  startTime: string | null;
+  token: string;
+}
+
+interface RoleLockState {
+  /** whose the lock is: a run id (holders of one run share it), or a lone process's own key */
+  owner: string;
+  holders: RoleHolder[];
+}
+
+function readRoleState(file: string): RoleLockState | null {
+  if (!existsSync(file)) return null;
+  try {
+    const v = JSON.parse(readFileSync(file, "utf8")) as RoleLockState;
+    return typeof v?.owner === "string" && Array.isArray(v.holders) ? v : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Spec 1.5 "Cross-run verifier contention": one owner at a time machine-wide holds the lock of `role`.
+ * Commands of the same owner (one run's verifier, side by side within its slots) share it; another owner's
+ * wait until the last of them ends, so two runs' verifiers never overlap. A dead holder is dropped.
+ */
+export async function withRoleLock<T>(
+  role: string,
+  owner: string,
+  fn: () => T | Promise<T>,
+  o: { pollMs?: number; onWait?: () => void } = {},
+): Promise<T> {
+  const dir = locksDir();
+  ensurePrivateDir(dir);
+  const file = join(dir, `role-${role}.json`);
+  const me: RoleHolder = { ...selfIdentity(), token: randomUUID() };
+  const enter = () =>
+    withFileLock(file, () => {
+      const state = readRoleState(file);
+      const live = (state?.holders ?? []).filter((h) => isAlive(h.pid, h.startTime));
+      if (live.length && state?.owner !== owner) return false;
+      writeJsonAtomic(file, { owner, holders: [...live, me] } satisfies RoleLockState);
+      return true;
+    });
+  while (!(await enter())) {
+    o.onWait?.();
     await Bun.sleep(o.pollMs ?? 500);
   }
+  try {
+    return await fn();
+  } finally {
+    await withFileLock(file, () => {
+      const state = readRoleState(file);
+      if (state)
+        writeJsonAtomic(file, { ...state, holders: state.holders.filter((h) => h.token !== me.token) });
+    });
+  }
+}
+
+/** Who holds the role lock now, or null. */
+export function roleLockOwner(role: string): string | null {
+  const state = readRoleState(join(locksDir(), `role-${role}.json`));
+  return state && state.holders.some((h) => isAlive(h.pid, h.startTime)) ? state.owner : null;
 }

package/src/infra/host-probe.ts

@@ -0,0 +1,16 @@
+// Plan 23: a host probe that fails once may have caught the network mid-change (a VPN just dropped, a route
+// still settling). catherd's own host probes run twice, this far apart, before they call the host blocked.
+
+/** The pause before the second try; tests shorten it. */
+export const hostProbe = { retryMs: 5_000 };
+
+/** `probe`, and once more after hostProbe.retryMs when `ok` says the first answer failed. */
+export async function probeTwice<T>(
+  probe: () => Promise<T>,
+  ok: (r: T) => boolean,
+): Promise<{ result: T; attempts: 1 | 2 }> {
+  const first = await probe();
+  if (ok(first)) return { result: first, attempts: 1 };
+  await Bun.sleep(hostProbe.retryMs);
+  return { result: await probe(), attempts: 2 };
+}