catherd-cli 1.4.0 → 2.0.0-beta.20261022113309

package/src/infra/supervisor.ts

@@ -1,11 +1,13 @@
 import { scrubSecrets } from "./env.ts";
 import type { Subprocess } from "bun";
 import { appendFileSync, closeSync, existsSync, openSync, readSync, statSync } from "node:fs";
+import { basename } from "node:path";
 import { z } from "zod";
 import { errorMessage } from "../domain/errors.ts";
 import type { ExitInfo, ExitReason } from "../domain/record.ts";
 import { dispatchPaths, supervisorLockTarget } from "./dispatch-dir.ts";
 import { tryLock } from "./filelock.ts";
+import { lastLockOutput } from "./lock-activity.ts";
 import { log } from "./log.ts";
 import { killGroup, processStartTime } from "./proc.ts";
 import { PRIVATE_FILE, writeJsonAtomic } from "./store.ts";
@@ -17,6 +19,8 @@ export const SuperviseSpecSchema = z.looseObject({
   cmd: z.string(),
   args: z.array(z.string()),
   env: z.record(z.string(), z.string()),
+  /** plan 23: env vars set from another one at spawn (a gate env secret by reference): name → source name */
+  envFrom: z.record(z.string(), z.string()).optional(),
   cwd: z.string(),
   stdinPath: z.string().nullable(),
   idleMs: z.number().positive(),
@@ -24,15 +28,28 @@ export const SuperviseSpecSchema = z.looseObject({
   killGraceMs: z.number().nonnegative(),
   graceAfterFinalMs: z.number().nonnegative().nullable(),
   pollMs: z.number().positive(),
+  /** plan 23: when provider retries end an attempt; PROVIDER_RETRY when absent (tests shorten it) */
+  providerRetry: z
+    .object({ attempts: z.number().positive(), ms: z.number().positive(), pollMs: z.number().positive() })
+    .partial()
+    .optional(),
 });
 export type SuperviseSpec = z.infer<typeof SuperviseSpecSchema>;
 
+/**
+ * Plan 23: an attempt that has only retried its provider (3 retries, or 3 min of nothing else) ends as
+ * provider-unavailable; a quiet session's own retry field is read every pollMs.
+ */
+export const PROVIDER_RETRY = { attempts: 3, ms: 180_000, pollMs: 30_000 };
+
 /** What one stream line tells the supervisor: the terminal event, the thread, a tool call opening or closing. */
 interface LineInfo {
   final?: boolean;
   thread?: string;
   /** a tool call the CLI started (`open`) or finished: while one is open the run is busy, however quiet */
   item?: { id: string; open: boolean };
+  /** plan 23: a model step started; one right after another, with nothing between, is a provider retry */
+  step?: boolean;
 }
 
 export interface SuperviseHooks {
@@ -40,8 +57,23 @@ export interface SuperviseHooks {
   /** `sinceMs`: when this run started, so a backend can ignore what an earlier run on the thread left */
   isBusy?(thread: string | null, sinceMs: number): Promise<boolean>;
   interrupt?(thread: string | null): Promise<void>;
+  /** plan 23: the provider retry attempt the session waits out (its own retry field), or null */
+  providerRetry?(thread: string | null, sinceMs: number): Promise<number | null>;
+  /**
+   * The poll loop's time: what it reads as now and how it waits a poll. The real clock when absent; a test
+   * passes its own so the wall, idle and retry limits are decided by the time it drives, never by how fast a
+   * loaded runner happens to schedule timers.
+   */
+  clock?: SuperviseClock;
+}
+
+export interface SuperviseClock {
+  now(): number;
+  sleep(ms: number): Promise<void>;
 }
 
+const REAL_CLOCK: SuperviseClock = { now: () => Date.now(), sleep: (ms) => Bun.sleep(ms) };
+
 /** Reads whatever the child appended to `file` since `offset`, as complete lines. */
 function readNew(file: string, state: { offset: number; rest: string; decoder: TextDecoder }): string[] {
   if (!existsSync(file)) return [];
@@ -79,7 +111,7 @@ async function bounded<T>(call: () => Promise<T> | undefined, ms: number, fallba
   }
 }
 
-function groupAlive(pgid: number): boolean {
+export function groupAlive(pgid: number): boolean {
   try {
     process.kill(-pgid, 0);
     return true;
@@ -89,7 +121,7 @@ function groupAlive(pgid: number): boolean {
 }
 
 /** SIGTERM the group, give it `graceMs` to leave, then SIGKILL whatever is left of it. */
-async function stopGroup(pgid: number, graceMs: number, pollMs: number): Promise<void> {
+export async function stopGroup(pgid: number, graceMs: number, pollMs: number): Promise<void> {
   killGroup(pgid, "SIGTERM");
   const end = Date.now() + graceMs;
   while (Date.now() < end && groupAlive(pgid))
@@ -168,6 +200,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
   }
 
   const hookMs = Math.min(10_000, spec.idleMs);
+  const clock = hooks.clock ?? REAL_CLOCK;
   let reason: ExitReason | null = null;
   let thread: string | null = null;
   let failure: { error: unknown } | null = null;
@@ -187,8 +220,12 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
       startedAt: new Date().toISOString(),
     });
 
-    const started = Date.now();
+    const started = clock.now();
+    const dispatchId = basename(spec.dispatchDir);
     let lastActivity = started;
+    // plan 23: the last output seen from a `catherd lock` of the role, kept after the lock exits (its file goes),
+    // so the wall counts from that output, never from the role's start again
+    let lockAt = started;
     let finalAt: number | null = null;
     // spec §3.6: a quiet stretch of half the idle timeout, not busy, is a stall, reported once per dispatch
     let stalled = false;
@@ -196,34 +233,88 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
     const open = new Set<string>();
     const stream = { offset: 0, rest: "", decoder: new TextDecoder("utf-8") };
 
+    // plan 23: a step that starts right after another with nothing between is a provider retry; retries are
+    // not progress, and enough of them (or a long enough stretch) end the attempt as provider-unavailable
+    let lastWasStep = false;
+    let retries = 0;
+    let retrySince: number | null = null;
+    let lastRetryPoll = started;
+    const retry = { ...PROVIDER_RETRY, ...spec.providerRetry };
+
     while (!done && reason === null) {
-      await Bun.sleep(spec.pollMs);
+      await clock.sleep(spec.pollMs);
       const lines = readNew(p.events, stream);
-      if (lines.length) {
-        lastActivity = Date.now();
-        stallChecked = false;
-      }
+      let progress = false;
       for (const line of lines) {
         let d: LineInfo | undefined;
         try {
           d = hooks.onLine?.(line);
         } catch {
-          continue; // one line the hook cannot read must not end supervision
+          // one line the hook cannot read must not end supervision; it is output all the same
+          progress = true;
+          lastWasStep = false;
+          continue;
         }
-        if (d?.thread) thread = d.thread;
-        if (d?.final) finalAt ??= Date.now();
+        if (d?.step && lastWasStep) {
+          retries++;
+          retrySince ??= clock.now();
+        } else {
+          progress = true;
+          if (!d?.step) {
+            retries = 0;
+            retrySince = null;
+          }
+        }
+        lastWasStep = d?.step === true;
+        if (d?.thread && d.thread !== thread) {
+          thread = d.thread;
+          // spec 1.5 plan 21: a live role's thread is on disk, so no delivery ever targets it
+          try {
+            writeJsonAtomic(p.thread, { schema: 1, thread, at: new Date().toISOString() });
+          } catch (e) {
+            log("warn", "supervise", { dispatch: spec.dispatchDir, error: errorMessage(e) });
+          }
+        }
+        if (d?.final) finalAt ??= clock.now();
         if (d?.item) {
           if (d.item.open) open.add(d.item.id);
           else open.delete(d.item.id);
         }
       }
+      if (progress) {
+        lastActivity = clock.now();
+        stallChecked = false;
+      }
+      // plan 23: a session quiet for a while may be waiting out a provider retry the stream does not show
+      if (
+        hooks.providerRetry &&
+        clock.now() - lastActivity >= retry.pollMs &&
+        clock.now() - lastRetryPoll >= retry.pollMs
+      ) {
+        lastRetryPoll = clock.now();
+        const attempt = await bounded(() => hooks.providerRetry?.(thread, started), hookMs, null);
+        if (attempt !== null) {
+          retrySince ??= clock.now();
+          retries = Math.max(retries, attempt);
+        } else {
+          // the session says it is not retrying: the steps seen were a retry that worked, and the quiet since
+          // is work the stream does not show yet (a long tool call), never a provider outage
+          retries = 0;
+          retrySince = null;
+        }
+      }
       // a worker that ended on its own is recorded as it ended, even if a cancel or a limit arrived meanwhile
       if (done) break;
-      const now = Date.now();
+      const now = clock.now();
+      lockAt = Math.max(lockAt, lastLockOutput(dispatchId) ?? lockAt);
       if (existsSync(p.cancel)) reason = "cancelled";
-      else if (now - started >= spec.wallMs) reason = "wall-timeout";
+      else if (now - lockAt >= spec.wallMs)
+        // plan 23: once a `catherd lock` of the role has written, the wall counts from its last output
+        reason = "wall-timeout";
       else if (finalAt !== null && spec.graceAfterFinalMs !== null && now - finalAt >= spec.graceAfterFinalMs)
         reason = "after-final";
+      else if (retries >= retry.attempts || (retrySince !== null && now - retrySince >= retry.ms))
+        reason = "provider-unavailable";
       else if (now - lastActivity >= spec.idleMs) {
         const busy = open.size > 0 || (await bounded(() => hooks.isBusy?.(thread, started), hookMs, false));
         // the busy check can take up to hookMs: a worker that ended meanwhile is recorded as it ended, read
@@ -231,7 +322,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
         if (done || child.exitCode !== null || child.signalCode !== null) break;
         if (busy) {
           // a new quiet stretch starts: it gets its own stall check
-          lastActivity = Date.now();
+          lastActivity = clock.now();
           stallChecked = false;
         } else reason = "idle-timeout";
       } else if (!stalled && !stallChecked && now - lastActivity >= spec.idleMs / 2) {
@@ -240,7 +331,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
         stallChecked = true;
         const busy = open.size > 0 || (await bounded(() => hooks.isBusy?.(thread, started), hookMs, false));
         if (busy) {
-          lastActivity = Date.now();
+          lastActivity = clock.now();
           stallChecked = false;
         } else if (!done) {
           stalled = true;
@@ -249,7 +340,7 @@ async function superviseHeld(spec: SuperviseSpec, hooks: SuperviseHooks): Promis
             writeJsonAtomic(p.stall, {
               schema: 1,
               at: new Date().toISOString(),
-              quietMs: Date.now() - lastActivity,
+              quietMs: clock.now() - lastActivity,
             });
           } catch (e) {
             log("warn", "stall", { dispatch: spec.dispatchDir, error: errorMessage(e) });

package/src/services/admission.ts

@@ -1,22 +1,29 @@
 import type { KnownHost } from "../domain/host.ts";
-import { appendFileSync, existsSync, readFileSync } from "node:fs";
+import { appendFileSync, existsSync, readFileSync, realpathSync } from "node:fs";
 import { join } from "node:path";
 import type { BackendAdapter } from "../adapters/backend.ts";
 import { budgetStatus, formatBudget } from "../domain/budget.ts";
 import { CatherdError, errorMessage } from "../domain/errors.ts";
-import { assertId, formatRung, newDispatchId, parseRung } from "../domain/ids.ts";
+import { assertId, formatRung, headlessRung, newDispatchId, parseRung } from "../domain/ids.ts";
 import { assertLaneHeader, overlaps } from "../domain/lane.ts";
+import { roleTimeouts } from "../domain/profile.ts";
 import type { RunRecord } from "../domain/record.ts";
-import { withReplyContract } from "../domain/role-prompts.ts";
+import { composeBrief } from "../domain/brief.ts";
+import { rulesBlock } from "../domain/rules.ts";
+import { briefOwns, DOCS_OWNS } from "../domain/changes.ts";
+import { grantsScratch, ROLE_SERVER_BACKENDS } from "../domain/role-tools.ts";
 import type { Role } from "../domain/roles.ts";
+import { formatRoleScope, ROLE_ENV } from "../domain/role-scope.ts";
 import { dispatchPaths, markForCollect } from "../infra/dispatch-dir.ts";
 import { withFileLock } from "../infra/filelock.ts";
 import { statusSnapshot } from "../infra/git.ts";
 import { launchSupervisor } from "../infra/launch.ts";
+import { DISPATCH_ID_ENV } from "../infra/lock-activity.ts";
 import { log } from "../infra/log.ts";
 import { processStartTime } from "../infra/proc.ts";
 import { ensurePrivateDir, PRIVATE_FILE, writeJsonAtomic, writeTextAtomic } from "../infra/store.ts";
 import { readyAdapter, standInFor } from "./backends.ts";
+import { failoverBackends } from "./failover-rungs.ts";
 import { spendOf } from "./budget.ts";
 import {
   type Admit,
@@ -27,12 +34,22 @@ import {
   listDispatches,
   pendingDispatches,
   roleDir,
+  scratchDir,
   setLatest,
 } from "./dispatches.ts";
 import { finalizeDispatch } from "./finalize.ts";
+import { gateEnvParts, readGateEnv } from "./gate-env.ts";
+import { assertNotPaused } from "./pause.ts";
+import { currentOwns } from "./lane-edit.ts";
+import { planContext } from "./plan-input.ts";
+import { standingRules } from "./run-service.ts";
+import { unfinishedAfter } from "./protocol.ts";
+import { runProfile } from "./run-pin.ts";
 import type { Deps } from "./ports.ts";
-import { readRecords, recordsOnThread, type Run, runPaths } from "./run-store.ts";
+import { readRecords, recordsOnThread, type Run, runPaths, supersededBy } from "./run-store.ts";
 import { currentSession } from "./sessions.ts";
+import { verifierBrief } from "./verifier-brief.ts";
+import { withRunAdmission } from "./workspace-admission.ts";
 
 export interface AdmitInput {
   role: Role;
@@ -47,6 +64,8 @@ export interface AdmitInput {
   /** the dispatching session (spec §3.3); absent means the session this process serves */
   sessionId?: string | null;
   host?: KnownHost;
+  /** plan 35: where the role works (its milestone's worktree in a base run); absent means the run's repo */
+  cwd?: string;
 }
 
 /** Spec §3.3: SIGTERM, then SIGKILL this long after. */
@@ -54,7 +73,15 @@ export const KILL_GRACE_MS = 10_000;
 
 export const laneFile = (run: Run, lane: string): string => join(runPaths(run.dir).lanes, `${lane}.md`);
 
-function laneOwns(run: Run, lane: string): string[] {
+/** The role's scratch folder, created (0700) and by its real path, on a backend that grants it; else null. */
+function roleScratch(run: Run, name: string, backend: string, isolated: boolean): string | null {
+  if (!grantsScratch(backend, isolated)) return null;
+  const dir = scratchDir(run, name);
+  ensurePrivateDir(dir);
+  return realpathSync(dir);
+}
+
+export function laneOwns(run: Run, lane: string): string[] {
   const file = laneFile(run, lane);
   if (!existsSync(file))
     throw new CatherdError("E_LANE_INVALID", `no lane file lanes/${lane}.md`, {
@@ -124,6 +151,15 @@ async function prepared(adapter: BackendAdapter, req: Parameters<NonNullable<Bac
   }
 }
 
+/** Refuses a dispatch into a run `runs supersede` closed, naming the run that took over. */
+function assertNotSuperseded(run: Run): void {
+  const closed = supersededBy(run);
+  if (closed)
+    throw new CatherdError("E_RUN_NOT_LIVE", `run ${run.id} is superseded by ${closed.by}`, {
+      fix: `dispatch in run ${closed.by}`,
+    });
+}
+
 /**
  * Spec §4.4 step 1. The checks that read shared state and the write that makes the dispatch live
  * happen under the run's admission lock, so parallel dispatches always see each other (audit C1).
@@ -136,8 +172,12 @@ export async function admit(
 ): Promise<{ d: Dispatch; specPath: string }> {
   assertId("role name", i.name);
   if (i.lane !== null) assertId("lane", i.lane);
+  // a machine or workspace pause refuses every dispatch it covers, with its reason (spec 1.5 "Group pause")
+  assertNotPaused(run);
+  assertNotSuperseded(run);
   const rung = parseRung(i.rung);
-  const profile = deps.profiles.forRepo(run.meta.repo);
+  // spec 1.5: the run's pinned profile, access and isolation, whatever the repo runs on now
+  const profile = runProfile(deps, run);
   const rc = profile.roles[i.role];
   if (!rc?.enabled)
     throw new CatherdError("E_ADMIT_RUNG", `the ${i.role} role is off in profile ${profile.name}`, {
@@ -147,11 +187,15 @@ export async function admit(
     throw new CatherdError("E_ADMIT_RUNG", `${i.rung} is a native Claude rung`, {
       fix: `run it as Agent(subagent_type: "${deps.profiles.agentFor(run.meta.repo, i.role, i.rung)}"), then record_agent_run`,
     });
-  const allowed = new Set([
+  const ladder = [
     ...rc.rungs,
     ...rc.rungs.flatMap((r) => standInFor(profile.failover, r, run.meta.repo) ?? []),
-  ]);
-  if (!allowed.has(formatRung(rung)))
+  ];
+  // the daemon runs every role headless (spec 2.0 decision 12): a native rung's headless twin is on its ladder too
+  const allowed = new Set([...ladder, ...ladder.map(headlessRung)]);
+  // a rung the daemon failed a limit over to may be on any configured backend (spec 2.0 decision 6)
+  const failedOver = i.failoverFrom !== null && failoverBackends(profile).has(rung.backend);
+  if (!allowed.has(formatRung(rung)) && !failedOver)
     throw new CatherdError(
       "E_ADMIT_RUNG",
       `${i.rung} is not on the ${i.role} ladder of profile ${profile.name}`,
@@ -174,7 +218,7 @@ export async function admit(
       `${adapter.id} keeps the access a thread started with: ${i.thread} ran ${started.access}, and the ${i.role} role runs ${rc.access}`,
       { fix: "dispatch a fresh thread (omit `thread`)" },
     );
-  // and the network grant: grok keeps a session's sandbox profile, agy's isolated homes differ by it.
+  // and the network grant: a CLI that keeps a session's sandbox profile keeps it.
   // A record from before 1.3 carries no grant to compare.
   if (started?.network !== undefined && adapter.resume.sameAccessOnly && started.network !== network)
     throw new CatherdError(
@@ -183,31 +227,52 @@ export async function admit(
       { fix: "dispatch a fresh thread (omit `thread`)" },
     );
   const owns = i.lane === null ? [] : laneOwns(run, i.lane);
+  // spec 1.5 plan 21: the writer's implicit docs lane, for attribution only
+  const ownsImplicit = i.role === "writer" && i.lane === null ? (briefOwns(i.brief) ?? DOCS_OWNS) : null;
+  // spec 1.5 "Lane editing": an After: line orders lanes that compile against each other
+  const before = i.lane === null ? [] : unfinishedAfter(run, i.lane);
+  if (before.length)
+    throw new CatherdError(
+      "E_ADMIT_ORDER",
+      `${i.lane} runs after ${before.join(", ")} (its After: line), which ${before.length > 1 ? "have" : "has"} not finished`,
+      { fix: `dispatch ${i.lane} once ${before.join(", ")} ${before.length > 1 ? "end" : "ends"} ok` },
+    );
   const id = newDispatchId();
   const dir = join(roleDir(run, i.name), id);
   const p = dispatchPaths(dir);
   const isolated = started?.isolated ?? profile.isolated[rung.backend] ?? false;
+  // spec 1.5 plan 21 (#42 findings 1, 2): the role server goes in whether the harness is isolated or not, so no
+  // role is refused for isolation, and a thread an isolated run started resumes as it started
+  const roleServer = ROLE_SERVER_BACKENDS.includes(rung.backend);
+  const cwd = i.cwd ?? run.meta.repo;
   await prepared(adapter, {
     rung,
     access: rc.access,
     isolated,
-    repo: run.meta.repo,
+    repo: cwd,
     network,
   });
-  const plan = adapter.plan({
-    rung,
-    access: rc.access,
-    network,
-    thread: i.thread,
-    isolated,
-    repo: run.meta.repo,
-    briefPath: p.brief,
-    replyPath: p.reply,
-    dispatchDir: dir,
-  });
 
   await finalizeFinished(run, deps.now(), onRecorded);
-  return withFileLock(runPaths(run.dir).admission, async () => {
+  return withRunAdmission(run, deps.now, async () => {
+    // again under the lock `runs supersede` writes under: a supersede that landed since the check above wins
+    assertNotSuperseded(run);
+    // the scratch is created under the admission lock `catherd runs clean` takes too, so cleanup never removes
+    // the scratch of a dispatch it does not see live yet (PR #46)
+    const scratch = roleScratch(run, i.name, rung.backend, isolated);
+    const plan = adapter.plan({
+      rung,
+      access: rc.access,
+      network,
+      thread: i.thread,
+      isolated,
+      repo: cwd,
+      briefPath: p.brief,
+      replyPath: p.reply,
+      dispatchDir: dir,
+      ...(roleServer ? { roleMcp: { run: run.id, role: i.role } } : {}),
+      ...(scratch ? { scratch } : {}),
+    });
     // A dispatch blocks until its record is written, not only while it runs: its finalizer diffs the
     // tree after the exit, so a later dispatch's writes must not land in between. A finished one here
     // could not be recorded just now. The records and the pending dispatches are one snapshot, taken
@@ -230,7 +295,9 @@ export async function admit(
         },
       );
     for (const d of pending) {
-      const shared = overlaps(owns, d.admit.owns);
+      // a running lane holds what owns_add granted it since its admission, too
+      const held = d.admit.lane ? [...d.admit.owns, ...currentOwns(run, d.admit.lane)] : d.admit.owns;
+      const shared = overlaps(owns, held);
       if (shared.length)
         throw new CatherdError(
           "E_ADMIT_OVERLAP",
@@ -246,7 +313,7 @@ export async function admit(
       throw new CatherdError("E_RUN_BUDGET", `the run budget is spent: ${formatBudget(budget)}`, {
         fix: "ask the user to raise profile.budget, or finish the run with what landed",
       });
-    const before = await statusSnapshot(run.meta.repo);
+    const before = await statusSnapshot(cwd);
     const session = currentSession(deps);
     const sessionId = i.sessionId !== undefined ? i.sessionId : (session?.sessionId ?? null);
     const admitted: Admit = {
@@ -257,6 +324,7 @@ export async function admit(
       role: i.role,
       lane: i.lane,
       owns,
+      ...(ownsImplicit ? { ownsImplicit } : {}),
       rung: formatRung(rung),
       backend: rung.backend,
       thread: i.thread,
@@ -268,13 +336,32 @@ export async function admit(
       network,
       cliVersion: probe.version,
       admittedAt: new Date(deps.now()).toISOString(),
-      repo: run.meta.repo,
+      repo: cwd,
       before,
       ...(sessionId ? { sessionId, host: i.host ?? session?.host ?? "claude-code" } : {}),
     };
     ensurePrivateDir(dir);
-    // spec 1.1 §6: every brief ends with its role's reply contract, failover stand-ins' included
-    writeTextAtomic(p.brief, withReplyContract(i.role, i.brief));
+    // plan 34: the repo's standing rules for this role; a knowledge file that cannot be read adds nothing
+    const rules = await standingRules(run.meta.repo, i.role).catch(() => "");
+    // plan 23: the verifier runs with the repo's gate environment (DOCKER_HOST, a proxy, …)
+    const gate =
+      i.role === "verifier" ? gateEnvParts(await readGateEnv(run.meta.repo)) : { values: {}, refs: {} };
+    // spec 1.1 §6: every brief ends with its role's reply contract, failover stand-ins' included; spec 1.5 plan 21:
+    // before it, the lane file as it stands now, and who the role is, its scratch and its catherd tools; plan 23:
+    // a verifier's brief also carries the gate's rules and the run's recorded items
+    writeTextAtomic(
+      p.brief,
+      composeBrief(await verifierBrief(run, i.role, i.name, i.brief), {
+        run: run.id,
+        name: i.name,
+        role: i.role,
+        lane: i.lane === null ? null : { id: i.lane, text: readFileSync(laneFile(run, i.lane), "utf8") },
+        scratch,
+        roleServer,
+        version: deps.version,
+        context: [planContext(run, i.role), rulesBlock(rules, i.role)].filter(Boolean).join("\n\n") || null,
+      }),
+    );
     // Spec §10.4: the adapter's overrides only; the supervisor adds its own inherited env at spawn
     // time (src/entry/supervise-command.ts), so no credential is ever written to disk. 0600 all the same.
     writeJsonAtomic(
@@ -285,11 +372,26 @@ export async function admit(
         dispatchDir: dir,
         cmd: plan.cmd,
         args: plan.args,
-        env: { ...plan.env, PWD: plan.cwd },
+        // spec 1.5 plan 21: the supervisor gives the role its identity and, where granted, its scratch TMPDIR;
+        // plan 23: a `catherd lock` in the role reports to this dispatch, so a long gate keeps its wall alive
+        // plan 23: each dispatch its own testcontainers session, so parallel lanes never share a reaper;
+        // the gate env never overrides the role's identity, scratch, dispatch id, PWD or isolated home
+        env: {
+          ...gate.values,
+          ...plan.env,
+          [ROLE_ENV]: formatRoleScope({ run: run.id, name: i.name }),
+          ...(scratch ? { TMPDIR: scratch } : {}),
+          [DISPATCH_ID_ENV]: id,
+          TESTCONTAINERS_SESSION_ID: id,
+          PWD: plan.cwd,
+        },
+        // a secret of the gate env by reference only: the supervisor reads it from its own env at spawn
+        ...(Object.keys(gate.refs).length ? { envFrom: gate.refs } : {}),
         cwd: plan.cwd,
         stdinPath: plan.stdinPath,
-        idleMs: profile.timeouts.idleMin * 60_000,
-        wallMs: profile.timeouts.wallMin * 60_000,
+        // plan 23: a role's own timeouts win over the profile's (a verifier's long gate)
+        idleMs: roleTimeouts(profile, rc).idleMin * 60_000,
+        wallMs: roleTimeouts(profile, rc).wallMin * 60_000,
         killGraceMs: KILL_GRACE_MS,
         graceAfterFinalMs: adapter.graceAfterFinalMs,
         pollMs: deps.pollMs,

package/src/services/backends.ts

@@ -5,7 +5,6 @@ import { adapterFor } from "../adapters/registry.ts";
 import "../adapters/all.ts";
 import { CatherdError } from "../domain/errors.ts";
 import { formatRung, tryParseRung } from "../domain/ids.ts";
-import { PAIRED_FAILOVER } from "../domain/profile.ts";
 
 const READY_TTL_MS = 10 * 60_000;
 const ready = new Map<string, { at: number; probe: Probe }>();
@@ -17,7 +16,7 @@ export const resetReadiness = (): void => ready.clear();
 function execFormatError(e: unknown): string | null {
   if (!(e instanceof Error)) return null;
   if ((e as { code?: unknown }).code !== "ENOEXEC" && !/exec format error/i.test(e.message)) return null;
-  // macOS names the path (posix_spawn '/…/grok'), Linux only the command (uv_spawn 'grok')
+  // macOS names the path (posix_spawn '/…/bin'), Linux only the command (uv_spawn 'bin')
   const bin = /spawn '([^']+)'/.exec(e.message)?.[1];
   return bin ? (Bun.which(bin, { PATH: process.env.PATH ?? "" }) ?? bin) : "exec format error";
 }
@@ -64,18 +63,17 @@ export async function readyAdapter(backend: string): Promise<{ adapter: BackendA
   const probe = await probeBackend(adapter);
   const problem = probe.problems[0];
   // spec 1.3 §3.3: a probe that finds the CLI logged out lists E_BACKEND_NOT_LOGGED_IN, so a dispatch never
-  // reaches it (agy opens a browser and waits); logged out alone is no problem where a CLI runs without a login
+  // reaches it (it may open a browser and wait); logged out alone is no problem where a CLI runs without a login
   if (problem) throw new CatherdError(problem.code, problem.message, { fix: problem.fix });
   ready.set(backend, { at: now, probe });
   return { adapter, probe };
 }
 
 /**
- * Spec §4.5: a rung's stand-in on a usage limit: the profile's, else the same model on its paired backend
- * (spec 1.3 §7.3), else its backend's default (for `repo`, when given), else none.
+ * Spec §4.5: a rung's stand-in on a usage limit: the profile's, else its backend's default (for `repo`, when given), else none.
  */
 export function standInFor(failover: Record<string, string>, rung: string, repo?: string): string | null {
-  const own = failover[rung] ?? PAIRED_FAILOVER[rung];
+  const own = failover[rung];
   if (own) return own;
   const r = tryParseRung(rung);
   if (!r) return null;

package/src/services/budget.ts

@@ -2,13 +2,14 @@ import { existsSync, readFileSync } from "node:fs";
 import { adapterFor } from "../adapters/registry.ts";
 import "../adapters/all.ts";
 import { type Budget, type BudgetStatus, budgetStatus, type Spend } from "../domain/budget.ts";
+import { CatherdError } from "../domain/errors.ts";
 import type { RunRecord } from "../domain/record.ts";
 import { dispatchPaths } from "../infra/dispatch-dir.ts";
 import { type Dispatch, liveDispatches } from "./dispatches.ts";
 import { readAgentRuns, readRecords, type Run } from "./run-store.ts";
 
 /** Tokens a running dispatch has used so far, summed from the events its CLI has written. */
-export function liveTokens(d: Dispatch): number {
+export function liveTokens(d: Dispatch, strict = false): number {
   const a = adapterFor(d.admit.backend);
   const file = dispatchPaths(d.dir).events;
   if (!a || !existsSync(file)) return 0;
@@ -17,8 +18,13 @@ export function liveTokens(d: Dispatch): number {
     if (!line.trim()) continue;
     try {
       const t = a.parse(line).tokens;
-      if (t) n += t.input + t.output;
-    } catch {
+      if (t) {
+        if (strict && Object.values(t).some((value) => !Number.isFinite(value) || value < 0))
+          throw new CatherdError("E_RUN_CORRUPT", `dispatch ${d.admit.dispatchId} has invalid token usage`);
+        n += t.input + t.output;
+      }
+    } catch (error) {
+      if (strict && error instanceof CatherdError) throw error;
       // a line the adapter cannot read adds nothing
     }
   }
@@ -29,15 +35,29 @@ export function liveTokens(d: Dispatch): number {
  * Spec §4.6: minutes are wall clock since run_start; tokens and dollars sum every record (every
  * backend), every native subagent run the orchestrator reported, and what live roles used so far.
  */
-export function spendOf(run: Run, records: RunRecord[], live: Dispatch[], now: number): Spend {
-  const agents = readAgentRuns(run);
+export function spendOf(
+  run: Run,
+  records: RunRecord[],
+  live: Dispatch[],
+  now: number,
+  strict = false,
+): Spend {
+  const agents = readAgentRuns(run, strict);
+  if (strict) {
+    const invalid = (value: number | null) => value !== null && (!Number.isFinite(value) || value < 0);
+    if (
+      records.some((record) => Object.values(record.tokens).some(invalid) || invalid(record.costUsd)) ||
+      agents.some((agent) => invalid(agent.totalTokens) || invalid(agent.costUsd))
+    )
+      throw new CatherdError("E_RUN_CORRUPT", `run ${run.id} has invalid cost or token usage`);
+  }
   const sum = <T>(xs: T[], f: (x: T) => number) => xs.reduce((n, x) => n + f(x), 0);
   return {
     minutes: Math.max(0, (now - Date.parse(run.meta.createdAt)) / 60_000),
     tokens:
       sum(records, (r) => r.tokens.input + r.tokens.output) +
       sum(agents, (a) => a.totalTokens) +
-      sum(live, liveTokens),
+      sum(live, (dispatch) => liveTokens(dispatch, strict)),
     usd: sum(records, (r) => r.costUsd ?? 0) + sum(agents, (a) => a.costUsd ?? 0),
   };
 }