catherd-cli 1.3.0 → 1.5.0

package/src/services/admission.ts

@@ -1,17 +1,23 @@
-import { appendFileSync, existsSync, readFileSync } from "node:fs";
+import type { KnownHost } from "../domain/host.ts";
+import { appendFileSync, existsSync, readFileSync, realpathSync } from "node:fs";
 import { join } from "node:path";
 import type { BackendAdapter } from "../adapters/backend.ts";
 import { budgetStatus, formatBudget } from "../domain/budget.ts";
 import { CatherdError, errorMessage } from "../domain/errors.ts";
 import { assertId, formatRung, newDispatchId, parseRung } from "../domain/ids.ts";
 import { assertLaneHeader, overlaps } from "../domain/lane.ts";
+import { roleTimeouts } from "../domain/profile.ts";
 import type { RunRecord } from "../domain/record.ts";
-import { withReplyContract } from "../domain/role-prompts.ts";
+import { composeBrief } from "../domain/brief.ts";
+import { briefOwns, DOCS_OWNS } from "../domain/changes.ts";
+import { grantsScratch, ROLE_SERVER_BACKENDS } from "../domain/role-tools.ts";
 import type { Role } from "../domain/roles.ts";
+import { formatRoleScope, ROLE_ENV } from "../domain/role-scope.ts";
 import { dispatchPaths, markForCollect } from "../infra/dispatch-dir.ts";
 import { withFileLock } from "../infra/filelock.ts";
 import { statusSnapshot } from "../infra/git.ts";
 import { launchSupervisor } from "../infra/launch.ts";
+import { DISPATCH_ID_ENV } from "../infra/lock-activity.ts";
 import { log } from "../infra/log.ts";
 import { processStartTime } from "../infra/proc.ts";
 import { ensurePrivateDir, PRIVATE_FILE, writeJsonAtomic, writeTextAtomic } from "../infra/store.ts";
@@ -26,12 +32,20 @@ import {
   listDispatches,
   pendingDispatches,
   roleDir,
+  scratchDir,
   setLatest,
 } from "./dispatches.ts";
 import { finalizeDispatch } from "./finalize.ts";
+import { gateEnvParts, readGateEnv } from "./gate-env.ts";
+import { assertNotPaused } from "./pause.ts";
+import { currentOwns } from "./lane-edit.ts";
+import { unfinishedAfter } from "./protocol.ts";
+import { runProfile } from "./run-pin.ts";
 import type { Deps } from "./ports.ts";
-import { readRecords, recordsOnThread, type Run, runPaths } from "./run-store.ts";
+import { readRecords, recordsOnThread, type Run, runPaths, supersededBy } from "./run-store.ts";
 import { currentSession } from "./sessions.ts";
+import { verifierBrief } from "./verifier-brief.ts";
+import { withRunAdmission } from "./workspace-admission.ts";
 
 export interface AdmitInput {
   role: Role;
@@ -45,6 +59,7 @@ export interface AdmitInput {
   failoverOf?: string;
   /** the dispatching session (spec §3.3); absent means the session this process serves */
   sessionId?: string | null;
+  host?: KnownHost;
 }
 
 /** Spec §3.3: SIGTERM, then SIGKILL this long after. */
@@ -52,7 +67,15 @@ export const KILL_GRACE_MS = 10_000;
 
 export const laneFile = (run: Run, lane: string): string => join(runPaths(run.dir).lanes, `${lane}.md`);
 
-function laneOwns(run: Run, lane: string): string[] {
+/** The role's scratch folder, created (0700) and by its real path, on a backend that grants it; else null. */
+function roleScratch(run: Run, name: string, backend: string, isolated: boolean): string | null {
+  if (!grantsScratch(backend, isolated)) return null;
+  const dir = scratchDir(run, name);
+  ensurePrivateDir(dir);
+  return realpathSync(dir);
+}
+
+export function laneOwns(run: Run, lane: string): string[] {
   const file = laneFile(run, lane);
   if (!existsSync(file))
     throw new CatherdError("E_LANE_INVALID", `no lane file lanes/${lane}.md`, {
@@ -122,6 +145,15 @@ async function prepared(adapter: BackendAdapter, req: Parameters<NonNullable<Bac
   }
 }
 
+/** Refuses a dispatch into a run `runs supersede` closed, naming the run that took over. */
+function assertNotSuperseded(run: Run): void {
+  const closed = supersededBy(run);
+  if (closed)
+    throw new CatherdError("E_RUN_NOT_LIVE", `run ${run.id} is superseded by ${closed.by}`, {
+      fix: `dispatch in run ${closed.by}`,
+    });
+}
+
 /**
  * Spec §4.4 step 1. The checks that read shared state and the write that makes the dispatch live
  * happen under the run's admission lock, so parallel dispatches always see each other (audit C1).
@@ -134,8 +166,12 @@ export async function admit(
 ): Promise<{ d: Dispatch; specPath: string }> {
   assertId("role name", i.name);
   if (i.lane !== null) assertId("lane", i.lane);
+  // a machine or workspace pause refuses every dispatch it covers, with its reason (spec 1.5 "Group pause")
+  assertNotPaused(run);
+  assertNotSuperseded(run);
   const rung = parseRung(i.rung);
-  const profile = deps.profiles.forRepo(run.meta.repo);
+  // spec 1.5: the run's pinned profile, access and isolation, whatever the repo runs on now
+  const profile = runProfile(deps, run);
   const rc = profile.roles[i.role];
   if (!rc?.enabled)
     throw new CatherdError("E_ADMIT_RUNG", `the ${i.role} role is off in profile ${profile.name}`, {
@@ -181,10 +217,23 @@ export async function admit(
       { fix: "dispatch a fresh thread (omit `thread`)" },
     );
   const owns = i.lane === null ? [] : laneOwns(run, i.lane);
+  // spec 1.5 plan 21: the writer's implicit docs lane, for attribution only
+  const ownsImplicit = i.role === "writer" && i.lane === null ? (briefOwns(i.brief) ?? DOCS_OWNS) : null;
+  // spec 1.5 "Lane editing": an After: line orders lanes that compile against each other
+  const before = i.lane === null ? [] : unfinishedAfter(run, i.lane);
+  if (before.length)
+    throw new CatherdError(
+      "E_ADMIT_ORDER",
+      `${i.lane} runs after ${before.join(", ")} (its After: line), which ${before.length > 1 ? "have" : "has"} not finished`,
+      { fix: `dispatch ${i.lane} once ${before.join(", ")} ${before.length > 1 ? "end" : "ends"} ok` },
+    );
   const id = newDispatchId();
   const dir = join(roleDir(run, i.name), id);
   const p = dispatchPaths(dir);
   const isolated = started?.isolated ?? profile.isolated[rung.backend] ?? false;
+  // spec 1.5 plan 21 (#42 findings 1, 2): the role server goes in whether the harness is isolated or not, so no
+  // role is refused for isolation, and a thread an isolated run started resumes as it started
+  const roleServer = ROLE_SERVER_BACKENDS.includes(rung.backend);
   await prepared(adapter, {
     rung,
     access: rc.access,
@@ -192,21 +241,27 @@ export async function admit(
     repo: run.meta.repo,
     network,
   });
-  const plan = adapter.plan({
-    rung,
-    access: rc.access,
-    network,
-    thread: i.thread,
-    isolated,
-    repo: run.meta.repo,
-    briefPath: p.brief,
-    replyPath: p.reply,
-    dispatchDir: dir,
-  });
 
   await finalizeFinished(run, deps.now(), onRecorded);
-  const sessionId = i.sessionId !== undefined ? i.sessionId : (currentSession(deps)?.sessionId ?? null);
-  return withFileLock(runPaths(run.dir).admission, async () => {
+  return withRunAdmission(run, deps.now, async () => {
+    // again under the lock `runs supersede` writes under: a supersede that landed since the check above wins
+    assertNotSuperseded(run);
+    // the scratch is created under the admission lock `catherd runs clean` takes too, so cleanup never removes
+    // the scratch of a dispatch it does not see live yet (PR #46)
+    const scratch = roleScratch(run, i.name, rung.backend, isolated);
+    const plan = adapter.plan({
+      rung,
+      access: rc.access,
+      network,
+      thread: i.thread,
+      isolated,
+      repo: run.meta.repo,
+      briefPath: p.brief,
+      replyPath: p.reply,
+      dispatchDir: dir,
+      ...(roleServer ? { roleMcp: { run: run.id, role: i.role } } : {}),
+      ...(scratch ? { scratch } : {}),
+    });
     // A dispatch blocks until its record is written, not only while it runs: its finalizer diffs the
     // tree after the exit, so a later dispatch's writes must not land in between. A finished one here
     // could not be recorded just now. The records and the pending dispatches are one snapshot, taken
@@ -229,7 +284,9 @@ export async function admit(
         },
       );
     for (const d of pending) {
-      const shared = overlaps(owns, d.admit.owns);
+      // a running lane holds what owns_add granted it since its admission, too
+      const held = d.admit.lane ? [...d.admit.owns, ...currentOwns(run, d.admit.lane)] : d.admit.owns;
+      const shared = overlaps(owns, held);
       if (shared.length)
         throw new CatherdError(
           "E_ADMIT_OVERLAP",
@@ -245,6 +302,9 @@ export async function admit(
       throw new CatherdError("E_RUN_BUDGET", `the run budget is spent: ${formatBudget(budget)}`, {
         fix: "ask the user to raise profile.budget, or finish the run with what landed",
       });
+    const before = await statusSnapshot(run.meta.repo);
+    const session = currentSession(deps);
+    const sessionId = i.sessionId !== undefined ? i.sessionId : (session?.sessionId ?? null);
     const admitted: Admit = {
       schema: 1,
       runId: run.id,
@@ -253,6 +313,7 @@ export async function admit(
       role: i.role,
       lane: i.lane,
       owns,
+      ...(ownsImplicit ? { ownsImplicit } : {}),
       rung: formatRung(rung),
       backend: rung.backend,
       thread: i.thread,
@@ -265,12 +326,27 @@ export async function admit(
       cliVersion: probe.version,
       admittedAt: new Date(deps.now()).toISOString(),
       repo: run.meta.repo,
-      before: await statusSnapshot(run.meta.repo),
-      ...(sessionId ? { sessionId } : {}),
+      before,
+      ...(sessionId ? { sessionId, host: i.host ?? session?.host ?? "claude-code" } : {}),
     };
     ensurePrivateDir(dir);
-    // spec 1.1 §6: every brief ends with its role's reply contract, failover stand-ins' included
-    writeTextAtomic(p.brief, withReplyContract(i.role, i.brief));
+    // plan 23: the verifier runs with the repo's gate environment (DOCKER_HOST, a proxy, …)
+    const gate =
+      i.role === "verifier" ? gateEnvParts(await readGateEnv(run.meta.repo)) : { values: {}, refs: {} };
+    // spec 1.1 §6: every brief ends with its role's reply contract, failover stand-ins' included; spec 1.5 plan 21:
+    // before it, the lane file as it stands now, and who the role is, its scratch and its catherd tools; plan 23:
+    // a verifier's brief also carries the gate's rules and the run's recorded items
+    writeTextAtomic(
+      p.brief,
+      composeBrief(await verifierBrief(run, i.role, i.name, i.brief), {
+        run: run.id,
+        name: i.name,
+        role: i.role,
+        lane: i.lane === null ? null : { id: i.lane, text: readFileSync(laneFile(run, i.lane), "utf8") },
+        scratch,
+        roleServer,
+      }),
+    );
     // Spec §10.4: the adapter's overrides only; the supervisor adds its own inherited env at spawn
     // time (src/entry/supervise-command.ts), so no credential is ever written to disk. 0600 all the same.
     writeJsonAtomic(
@@ -281,11 +357,26 @@ export async function admit(
         dispatchDir: dir,
         cmd: plan.cmd,
         args: plan.args,
-        env: { ...plan.env, PWD: plan.cwd },
+        // spec 1.5 plan 21: the supervisor gives the role its identity and, where granted, its scratch TMPDIR;
+        // plan 23: a `catherd lock` in the role reports to this dispatch, so a long gate keeps its wall alive
+        // plan 23: each dispatch its own testcontainers session, so parallel lanes never share a reaper;
+        // the gate env never overrides the role's identity, scratch, dispatch id, PWD or isolated home
+        env: {
+          ...gate.values,
+          ...plan.env,
+          [ROLE_ENV]: formatRoleScope({ run: run.id, name: i.name }),
+          ...(scratch ? { TMPDIR: scratch } : {}),
+          [DISPATCH_ID_ENV]: id,
+          TESTCONTAINERS_SESSION_ID: id,
+          PWD: plan.cwd,
+        },
+        // a secret of the gate env by reference only: the supervisor reads it from its own env at spawn
+        ...(Object.keys(gate.refs).length ? { envFrom: gate.refs } : {}),
         cwd: plan.cwd,
         stdinPath: plan.stdinPath,
-        idleMs: profile.timeouts.idleMin * 60_000,
-        wallMs: profile.timeouts.wallMin * 60_000,
+        // plan 23: a role's own timeouts win over the profile's (a verifier's long gate)
+        idleMs: roleTimeouts(profile, rc).idleMin * 60_000,
+        wallMs: roleTimeouts(profile, rc).wallMin * 60_000,
         killGraceMs: KILL_GRACE_MS,
         graceAfterFinalMs: adapter.graceAfterFinalMs,
         pollMs: deps.pollMs,

package/src/services/agent-links.ts

@@ -1,3 +1,4 @@
+import type { OrchestrationHost } from "../domain/host.ts";
 import {
   existsSync,
   lstatSync,
@@ -44,10 +45,10 @@ export interface Planned {
   links: Map<string, string>;
 }
 
-export function plan(extra: string[] = []): Planned {
+export function plan(host: OrchestrationHost, extra: string[] = [], selected?: string[]): Planned {
   const files = new Map<string, AgentFile[]>();
-  for (const name of new Set([...linkedProfiles(), ...extra.filter(profileExists)]))
-    files.set(name, agentFiles(getProfile(name), VERSION));
+  for (const name of new Set(selected ?? [...linkedProfiles(), ...extra.filter(profileExists)]))
+    files.set(name, agentFiles(getProfile(name, host), VERSION));
   const links = new Map<string, string>();
   for (const name of linkedProfiles())
     for (const f of files.get(name) ?? [])
@@ -71,6 +72,27 @@ export interface Synced {
   newSessionNeededFor: string[];
 }
 
+export function pruneProfileAgents(name: string): Synced {
+  const dir = join(agentsRoot(), name);
+  const pruned: string[] = [];
+  const target = claudeAgentsDir();
+  if (existsSync(target))
+    for (const entry of readdirSync(target)) {
+      const link = join(target, entry);
+      if (isOurLink(link) && readlinkSync(link).startsWith(`${dir}${sep}`)) {
+        rmSync(link);
+        pruned.push(entry);
+      }
+    }
+  if (existsSync(dir)) rmSync(dir, { recursive: true, force: true });
+  pruned.sort();
+  return {
+    linked: [],
+    pruned,
+    newSessionNeededFor: pruned.map((entry) => basename(entry, ".md")),
+  };
+}
+
 /**
  * Writes the planned agent files and links, prunes catherd's stale ones, and says what changed. The
  * ownership check runs before the first write, so no caller can replace a file catherd does not own.
@@ -122,8 +144,11 @@ export function apply(p: Planned, removed: string[] = []): Synced {
 }
 
 /** What `doctor` compares: each link that should exist, and whether it and its file are current. */
-export function agentLinkState(): { missing: string[]; stale: string[]; ok: string[] } {
-  const p = plan();
+export function agentLinkState(
+  host: OrchestrationHost,
+  selected?: string[],
+): { missing: string[]; stale: string[]; ok: string[] } {
+  const p = plan(host, [], selected);
   const out = { missing: [] as string[], stale: [] as string[], ok: [] as string[] };
   for (const [name, files] of p.files)
     for (const f of files) {

package/src/services/backends.ts

@@ -1,3 +1,5 @@
+import type { OrchestrationHost } from "../domain/host.ts";
+import { nativeClaudeIssue } from "../domain/profile-rules.ts";
 import type { BackendAdapter, Probe } from "../adapters/backend.ts";
 import { adapterFor } from "../adapters/registry.ts";
 import "../adapters/all.ts";
@@ -80,3 +82,8 @@ export function standInFor(failover: Record<string, string>, rung: string, repo?
   const byAdapter = adapterFor(r.backend)?.failoverFor?.(r, repo) ?? null;
   return byAdapter && formatRung(byAdapter);
 }
+
+export function assertNativeHost(rung: string, host: OrchestrationHost): void {
+  const issue = nativeClaudeIssue(rung, host, "rung");
+  if (issue) throw new CatherdError("E_CONFIG_INVALID", `${issue.message}; ${issue.fix}`, { fix: issue.fix });
+}

package/src/services/budget.ts

@@ -2,13 +2,14 @@ import { existsSync, readFileSync } from "node:fs";
 import { adapterFor } from "../adapters/registry.ts";
 import "../adapters/all.ts";
 import { type Budget, type BudgetStatus, budgetStatus, type Spend } from "../domain/budget.ts";
+import { CatherdError } from "../domain/errors.ts";
 import type { RunRecord } from "../domain/record.ts";
 import { dispatchPaths } from "../infra/dispatch-dir.ts";
 import { type Dispatch, liveDispatches } from "./dispatches.ts";
 import { readAgentRuns, readRecords, type Run } from "./run-store.ts";
 
 /** Tokens a running dispatch has used so far, summed from the events its CLI has written. */
-export function liveTokens(d: Dispatch): number {
+export function liveTokens(d: Dispatch, strict = false): number {
   const a = adapterFor(d.admit.backend);
   const file = dispatchPaths(d.dir).events;
   if (!a || !existsSync(file)) return 0;
@@ -17,8 +18,13 @@ export function liveTokens(d: Dispatch): number {
     if (!line.trim()) continue;
     try {
       const t = a.parse(line).tokens;
-      if (t) n += t.input + t.output;
-    } catch {
+      if (t) {
+        if (strict && Object.values(t).some((value) => !Number.isFinite(value) || value < 0))
+          throw new CatherdError("E_RUN_CORRUPT", `dispatch ${d.admit.dispatchId} has invalid token usage`);
+        n += t.input + t.output;
+      }
+    } catch (error) {
+      if (strict && error instanceof CatherdError) throw error;
       // a line the adapter cannot read adds nothing
     }
   }
@@ -29,15 +35,29 @@ export function liveTokens(d: Dispatch): number {
  * Spec §4.6: minutes are wall clock since run_start; tokens and dollars sum every record (every
  * backend), every native subagent run the orchestrator reported, and what live roles used so far.
  */
-export function spendOf(run: Run, records: RunRecord[], live: Dispatch[], now: number): Spend {
-  const agents = readAgentRuns(run);
+export function spendOf(
+  run: Run,
+  records: RunRecord[],
+  live: Dispatch[],
+  now: number,
+  strict = false,
+): Spend {
+  const agents = readAgentRuns(run, strict);
+  if (strict) {
+    const invalid = (value: number | null) => value !== null && (!Number.isFinite(value) || value < 0);
+    if (
+      records.some((record) => Object.values(record.tokens).some(invalid) || invalid(record.costUsd)) ||
+      agents.some((agent) => invalid(agent.totalTokens) || invalid(agent.costUsd))
+    )
+      throw new CatherdError("E_RUN_CORRUPT", `run ${run.id} has invalid cost or token usage`);
+  }
   const sum = <T>(xs: T[], f: (x: T) => number) => xs.reduce((n, x) => n + f(x), 0);
   return {
     minutes: Math.max(0, (now - Date.parse(run.meta.createdAt)) / 60_000),
     tokens:
       sum(records, (r) => r.tokens.input + r.tokens.output) +
       sum(agents, (a) => a.totalTokens) +
-      sum(live, liveTokens),
+      sum(live, (dispatch) => liveTokens(dispatch, strict)),
     usd: sum(records, (r) => r.costUsd ?? 0) + sum(agents, (a) => a.costUsd ?? 0),
   };
 }

package/src/services/catalog-service.ts

@@ -99,18 +99,23 @@ export function measuredSecs(base: Catalog): Catalog["secs"] {
     if (kind) add(`${canonical}|${kind}`, secs);
   };
   for (const run of listRuns().runs) {
-    const routes = readRoutes(run);
-    for (const r of readRecords(run).records)
-      if (r.status === "ok") count(routes, r.rung, r.secs, r.lane, r.startedAt);
-    for (const a of readAgentRuns(run)) {
-      if (a.status !== "ok" || a.secs === null) continue;
-      // a row is written when the subagent ends; it started `secs` earlier
-      const lane = typeof a.lane === "string" ? a.lane : null;
-      const end = Date.parse(a.at);
-      // a duration past the Date range (record_agent_run takes any) keeps the end, not a RangeError
-      const start = new Date(end - a.secs * 1000);
-      const startedAt = Number.isNaN(start.getTime()) ? a.at : start.toISOString();
-      count(routes, a.rung, a.secs, lane, startedAt);
+    // a run whose files cannot be read gives no timings; it never fails a route (1.2 minor)
+    try {
+      const routes = readRoutes(run);
+      for (const r of readRecords(run).records)
+        if (r.status === "ok") count(routes, r.rung, r.secs, r.lane, r.startedAt);
+      for (const a of readAgentRuns(run)) {
+        if (a.status !== "ok" || a.secs === null) continue;
+        // a row is written when the subagent ends; it started `secs` earlier
+        const lane = typeof a.lane === "string" ? a.lane : null;
+        const end = Date.parse(a.at);
+        // a duration past the Date range (record_agent_run takes any) keeps the end, not a RangeError
+        const start = new Date(end - a.secs * 1000);
+        const startedAt = Number.isNaN(start.getTime()) ? a.at : start.toISOString();
+        count(routes, a.rung, a.secs, lane, startedAt);
+      }
+    } catch (e) {
+      log("debug", "catalog", { run: run.id, timings: errorMessage(e) });
     }
   }
   const secs: Catalog["secs"] = {};
@@ -340,6 +345,8 @@ function rungRows(
         date: string;
         /** spec 1.2 §5.3: the rung a borrowed or inferred value belongs to */
         from?: string;
+        /** what lent it: a treat-like's mapping (`like X`) or an inferred stand-in's guess (1.2 minor) */
+        lent?: "treat-like" | "stand-in";
       }
     > = {};
     for (const v of valuesUsed(c, info.canonical))
@@ -351,6 +358,7 @@ function rungRows(
         source: v.source,
         date: v.date,
         ...(v.from ? { from: v.from } : {}),
+        ...(v.lent ? { lent: v.lent } : {}),
       };
     const like = c.treatLike[info.canonical] ?? null;
     return {

package/src/services/credentials.ts

@@ -1,7 +1,8 @@
-import { existsSync } from "node:fs";
+import { existsSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import { z } from "zod";
 import { CatherdError, isCatherdError } from "../domain/errors.ts";
+import { withFileLockSync } from "../infra/filelock.ts";
 import { log } from "../infra/log.ts";
 import { configDir } from "../infra/paths.ts";
 import { readVersioned, writeJsonAtomic } from "../infra/store.ts";
@@ -45,13 +46,17 @@ export function savedCredential(field: CredentialField): {
 
 /**
  * Keeps every other credential, and the file at mode 600 (spec §10.4). A missing file starts empty; an
- * unreadable or newer-schema one is refused (it throws) rather than overwritten.
+ * unreadable or newer-schema one is refused (it throws) rather than overwritten. The read and the write hold
+ * the file's lock (1.2 minor), so two concurrent `init`s never lose a key.
  */
 export function saveCredential(field: CredentialField, key: string): void {
-  const cur: z.infer<typeof CredentialsSchema> = existsSync(credentialsPath())
-    ? readCredentials()
-    : { schema: 1 };
-  writeJsonAtomic(credentialsPath(), { ...cur, schema: 1, [field]: key.trim() }, { mode: 0o600 });
+  mkdirSync(configDir(), { recursive: true });
+  withFileLockSync(credentialsPath(), () => {
+    const cur: z.infer<typeof CredentialsSchema> = existsSync(credentialsPath())
+      ? readCredentials()
+      : { schema: 1 };
+    writeJsonAtomic(credentialsPath(), { ...cur, schema: 1, [field]: key.trim() }, { mode: 0o600 });
+  });
 }
 
 /**