catherd-cli 1.4.0 → 1.5.0

package/src/services/milestones.ts

@@ -2,7 +2,7 @@ import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { CatherdError } from "../domain/errors.ts";
 import type { RunRecord } from "../domain/record.ts";
-import { git } from "../infra/git.ts";
+import { commitExists, git } from "../infra/git.ts";
 import { nonBlankLines } from "../infra/store.ts";
 import { listDispatches } from "./dispatches.ts";
 import { readAgentRuns, readRecords, readRoutes, type Run, runPaths } from "./run-store.ts";
@@ -50,27 +50,95 @@ export interface MilestoneReviewer {
   record: RunRecord | null;
 }
 
-/**
- * The milestone's latest passing reviewer since its lanes started: a dispatch named reviewer-<m>…, status
- * ok; or a record_agent_run row with role reviewer and the same name, status ok. Null when there is none.
- */
-export function milestoneReviewer(
+/** Every reviewer attempt of the milestone since its lanes started that ended ok, oldest first, partial or not. */
+function reviewerAttempts(
   run: Run,
   m: string,
-  start = milestoneStart(run, m),
-): MilestoneReviewer | null {
-  const found: MilestoneReviewer[] = [
+  start: string | null,
+): (MilestoneReviewer & { partial: boolean })[] {
+  return [
     ...readRecords(run)
       .records.filter((r) => reviewsMilestone(r.name, m) && r.status === "ok" && since(r.endedAt, start))
-      .map((r) => ({ name: r.name, at: r.endedAt, record: r })),
+      .map((r) => ({ name: r.name, at: r.endedAt, record: r, partial: r.replyStatus === "partial" })),
     ...readAgentRuns(run)
       .filter(
         (a) =>
           a.role === "reviewer" && reviewsMilestone(a.name, m) && a.status === "ok" && since(a.at, start),
       )
-      .map((a) => ({ name: a.name, at: a.at, record: null })),
-  ];
-  return found.sort((a, b) => Date.parse(a.at) - Date.parse(b.at)).at(-1) ?? null;
+      .map((a) => ({ name: a.name, at: a.at, record: null, partial: a.replyStatus === "partial" })),
+  ].sort((a, b) => Date.parse(a.at) - Date.parse(b.at));
+}
+
+/**
+ * The milestone's latest passing reviewer since its lanes started: a dispatch named reviewer-<m>…, status
+ * ok; or a record_agent_run row with role reviewer and the same name, status ok. A reply of STATUS: partial
+ * read only part of the diff and is not the milestone's review (plan 23). Null when there is none.
+ */
+export function milestoneReviewer(
+  run: Run,
+  m: string,
+  start = milestoneStart(run, m),
+): MilestoneReviewer | null {
+  const full = reviewerAttempts(run, m, start).filter((a) => !a.partial);
+  const last = full.at(-1);
+  return last ? { name: last.name, at: last.at, record: last.record } : null;
+}
+
+/** Plan 23: the milestone's latest reviewer that stopped at STATUS: partial, when no full review stands; else null. */
+export function partialReviewer(
+  run: Run,
+  m: string,
+  start = milestoneStart(run, m),
+): MilestoneReviewer | null {
+  if (milestoneReviewer(run, m, start)) return null;
+  const last = reviewerAttempts(run, m, start)
+    .filter((a) => a.partial)
+    .at(-1);
+  return last ? { name: last.name, at: last.at, record: last.record } : null;
+}
+
+export const FINDING = /^\s*(?:[-*]\s*)?(BLOCKER|BUG|NIT)\b/;
+
+/** A reviewer reply's findings by severity; null when there is no reply to read. */
+export function countFindings(run: Run, r: RunRecord): { BLOCKER: number; BUG: number; NIT: number } | null {
+  const file = join(run.dir, r.replyPath);
+  if (!r.replyPath || !existsSync(file)) return null;
+  const n = { BLOCKER: 0, BUG: 0, NIT: 0 };
+  for (const line of readFileSync(file, "utf8").split("\n")) {
+    const k = FINDING.exec(line)?.[1] as keyof typeof n | undefined;
+    if (k) n[k]++;
+  }
+  return n;
+}
+
+/** The roles a fix round runs as. */
+const FIX_ROLES = new Set(["worker", "writer", "artist"]);
+
+/**
+ * Plan 23: the milestone reviewer's BLOCKER and BUG lines while no fix round has ended since (a worker,
+ * writer or artist of the milestone, ended ok after the review); null when there are none, or for a native
+ * reviewer, whose reply catherd does not keep.
+ */
+export function openFindings(
+  run: Run,
+  m: string,
+  start = milestoneStart(run, m),
+): { name: string; blocker: number; bug: number } | null {
+  const reviewer = milestoneReviewer(run, m, start);
+  if (!reviewer?.record) return null;
+  const n = countFindings(run, reviewer.record);
+  if (!n || n.BLOCKER + n.BUG === 0) return null;
+  const after = (at: string) => Date.parse(at) > Date.parse(reviewer.at);
+  const ofM = (name: string, lane: string | null | undefined) =>
+    inMilestone(lane, m) || namesMilestone(name, m);
+  const fixed =
+    readRecords(run).records.some(
+      (r) => FIX_ROLES.has(r.role) && r.status === "ok" && ofM(r.name, r.lane) && after(r.endedAt),
+    ) ||
+    readAgentRuns(run).some(
+      (a) => FIX_ROLES.has(a.role) && a.status === "ok" && ofM(a.name, a.lane) && after(a.at),
+    );
+  return fixed ? null : { name: reviewer.name, blocker: n.BLOCKER, bug: n.BUG };
 }
 
 /** A reviewer record for the milestone since its lanes started (see milestoneReviewer). */
@@ -80,6 +148,15 @@ export const reviewerPassed = (run: Run, m: string, start = milestoneStart(run,
 /** The verifier's contract puts `VERDICT: PASS` or `VERDICT: FAIL` on the reply's first line (role-prompts.ts). */
 const VERDICT_PASS = /^VERDICT: PASS\b/;
 
+/**
+ * Plan 23: `VERDICT: BLOCKED: environment — <the probe that proves it>`, a verdict on the machine, not on the
+ * work. Its probe, or null when `line` is no such verdict ("" when it names none).
+ */
+export function blockedByEnvironment(line: string | null | undefined): string | null {
+  const m = /^VERDICT:\s*BLOCKED:\s*environment\b\s*(?:—|–|-|:)?\s*(.*)$/i.exec(line?.trim() ?? "");
+  return m ? (m[1] ?? "").trim() : null;
+}
+
 /** A headless verifier's reply: its first non-blank line, trimmed; null when there is no reply to read. */
 export function replyVerdict(run: Run, r: RunRecord): string | null {
   const file = join(run.dir, r.replyPath);
@@ -106,11 +183,17 @@ export interface MilestoneVerifier {
   passed: boolean;
   /** what the attempt said: a native row's status, or a headless reply's first non-blank line ("no reply") */
   verdict: string;
+  /** plan 23: the probe of a `VERDICT: BLOCKED: environment` attempt ("" when it named none); null otherwise */
+  blocked: string | null;
 }
 
+/** The one name a milestone's verifier is counted under (spec 1.5 "The ledger"): verifier-<m>, exactly. */
+export const verifierName = (m: string): string => `verifier-${m}`;
+
 /**
  * The milestone's latest verifier attempt since its lanes started, whatever it said: a record_agent_run row
- * with role verifier whose name names the milestone (the skill records a FAIL as failed), passing when its
+ * with role verifier named exactly verifier-<m> (the skill records a FAIL as failed; verifier-M2-pre or
+ * verifier-M2-gate1 is not M2's verifier), passing when its
  * status is ok; or a headless verifier's dispatch record of the same shape, passing when it exited ok and its
  * reply opens VERDICT: PASS (status ok means only the CLI exited). A later FAIL undoes an earlier PASS. Null
  * when there is none.
@@ -122,16 +205,21 @@ export function milestoneVerifier(
 ): MilestoneVerifier | null {
   const found: MilestoneVerifier[] = [
     ...readAgentRuns(run)
-      .filter((a) => a.role === "verifier" && namesMilestone(a.name, m) && since(a.at, start))
-      .map((a) => ({
-        name: a.name,
-        at: a.at,
-        headless: false,
-        passed: a.status === "ok",
-        verdict: a.status,
-      })),
+      .filter((a) => a.role === "verifier" && a.name === verifierName(m) && since(a.at, start))
+      .map((a) => {
+        // plan 23: a native verifier's verdict line, when record_agent_run passed it
+        const blocked = a.status === "ok" ? null : blockedByEnvironment(a.verdict);
+        return {
+          name: a.name,
+          at: a.at,
+          headless: false,
+          passed: a.status === "ok",
+          verdict: blocked === null ? a.status : (a.verdict ?? a.status),
+          blocked,
+        };
+      }),
     ...readRecords(run)
-      .records.filter((r) => r.role === "verifier" && namesMilestone(r.name, m) && since(r.endedAt, start))
+      .records.filter((r) => r.role === "verifier" && r.name === verifierName(m) && since(r.endedAt, start))
       .map((r) => {
         const first = replyVerdict(run, r);
         const passed = r.status === "ok" && first !== null && VERDICT_PASS.test(first);
@@ -142,6 +230,7 @@ export function milestoneVerifier(
           headless: true,
           passed,
           verdict: r.status === "ok" ? said : `${r.status}, ${said}`,
+          blocked: r.status === "ok" ? blockedByEnvironment(first) : null,
         };
       }),
   ];
@@ -170,12 +259,16 @@ export function landedMilestones(run: Run): string[] {
 }
 
 /**
- * The files the milestone's commit range changed: from the previous landed commit (else the commit's own
- * parent) to `commit`. Throws E_IO_UNEXPECTED when git cannot say.
+ * The files the milestone's commit range changed: from the previous landed commit, else the HEAD the run
+ * started on (meta.json `startHead`, when that commit is still there), else the commit's own parent, to
+ * `commit`. Throws E_IO_UNEXPECTED when git cannot say.
  */
 export async function milestoneFiles(run: Run, commit: string): Promise<string[]> {
-  const base = landedCommits(run).at(-1);
   const repo = run.meta.repo;
+  const start = run.meta.startHead;
+  const base =
+    landedCommits(run).at(-1) ??
+    (start && (await commitExists(repo, start).catch(() => false)) ? start : undefined);
   const r = base
     ? await git(repo, ["diff", "--name-only", base, commit])
     : await git(repo, ["show", "--name-only", "--format=", "--first-parent", commit]);
@@ -186,6 +279,24 @@ export async function milestoneFiles(run: Run, commit: string): Promise<string[]
   return r.out.split("\n").filter(Boolean);
 }
 
+/** The most commits a digest lists; the rest are counted. */
+const DIGEST_COMMITS = 20;
+
+/**
+ * The commits the milestone lands, `<short hash> <subject>`, oldest first: from the previous landed commit
+ * (else the commit alone) to `commit`. Just the hash when git cannot say: a digest never fails a landing.
+ */
+export async function milestoneCommits(run: Run, commit: string): Promise<string[]> {
+  const base = landedCommits(run).at(-1);
+  const range = base ? [`${base}..${commit}`] : ["-1", commit];
+  const r = await git(run.meta.repo, ["log", "--reverse", "--format=%h %s", ...range]);
+  if (r.kind !== "ok") return [commit.slice(0, 7)];
+  const all = r.out.split("\n").filter(Boolean);
+  return all.length > DIGEST_COMMITS
+    ? [...all.slice(-DIGEST_COMMITS), `and ${all.length - DIGEST_COMMITS} earlier`]
+    : all;
+}
+
 /** `rev`'s full commit hash in the run's repo. Throws E_IO_UNEXPECTED when git cannot say. */
 export async function fullCommit(run: Run, rev: string): Promise<string> {
   const r = await git(run.meta.repo, ["rev-parse", "--verify", "--quiet", `${rev}^{commit}`]);

package/src/services/notifier.ts

@@ -1,5 +1,7 @@
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
+import { adapterFor } from "../adapters/registry.ts";
+import "../adapters/all.ts";
 import { CatherdError, errorMessage } from "../domain/errors.ts";
 import { envelope, formatNotices, type Notice, type NoticePriority, priorityOf } from "../domain/notice.ts";
 import { sessionKey, type HostSessionRef } from "../domain/host.ts";
@@ -7,6 +9,7 @@ import { sendToCodexQueue, type QueueSendResult } from "../infra/codex-queue.ts"
 import {
   deliveryState,
   readDelivery,
+  ROLE_THREAD_REFUSAL,
   writeDeliveryAttempt,
   type DeliveryAttempt,
 } from "../infra/delivery.ts";
@@ -15,7 +18,7 @@ import { awaitsCollect, dispatchPaths } from "../infra/dispatch-dir.ts";
 import { tryLock } from "../infra/filelock.ts";
 import { log } from "../infra/log.ts";
 import { type SendResult, sendToInbox } from "../infra/peer-inbox.ts";
-import { writeJsonAtomic } from "../infra/store.ts";
+import { nonBlankLines, writeJsonAtomic } from "../infra/store.ts";
 import {
   readStallQuietMs,
   type Settled,
@@ -23,7 +26,7 @@ import {
   type Stalled,
   stallHooks,
 } from "./dispatch-service.ts";
-import { type Dispatch, listDispatches, readFailover } from "./dispatches.ts";
+import { type Dispatch, listDispatches, readFailover, roleThreadOf } from "./dispatches.ts";
 import type { Deps } from "./ports.ts";
 import { listRuns, readRecords, type Run } from "./run-store.ts";
 import { currentSession, ownsRun, readSessionRows, runOwner } from "./sessions.ts";
@@ -76,7 +79,8 @@ function legacyNotified(q: Queued, target: HostSessionRef): boolean {
   }
 }
 
-function recoverSubmission(dir: string, eventId: string): void {
+/** A "submitting" claim left by a sender that died is ambiguous: retrying it may duplicate input. Under the notify lock. */
+export function recoverSubmission(dir: string, eventId: string): void {
   for (const a of readDelivery(dir)) {
     if (a.status === "submitting" && a.eventIds.includes(eventId))
       writeDeliveryAttempt(dir, {
@@ -119,6 +123,7 @@ export function finishedNotice(run: Run, d: Dispatch, r: RunRecord): Notice {
     name: r.name,
     role: r.role,
     lane: r.lane,
+    thread: r.thread,
     rung: r.rung,
     status,
     replyStatus: r.replyStatus,
@@ -129,6 +134,21 @@ export function finishedNotice(run: Run, d: Dispatch, r: RunRecord): Notice {
   };
 }
 
+/** A live fresh role's thread, as its CLI named it early in its stream; null before it did. */
+function streamThread(d: Dispatch): string | null {
+  const a = adapterFor(d.admit.backend);
+  if (!a) return null;
+  for (const line of nonBlankLines(dispatchPaths(d.dir).events)) {
+    try {
+      const t = a.parse(line).thread;
+      if (t) return t;
+    } catch {
+      // a line the adapter cannot read names no thread
+    }
+  }
+  return null;
+}
+
 /** Spec §3.6: a live role that went quiet, once, at `next`. */
 export function stalledNotice(run: Run, d: Dispatch, quietMs: number, now: number): Notice {
   return {
@@ -140,6 +160,7 @@ export function stalledNotice(run: Run, d: Dispatch, quietMs: number, now: numbe
     name: d.admit.name,
     role: d.admit.role,
     lane: d.admit.lane,
+    thread: d.admit.thread ?? streamThread(d),
     rung: d.admit.rung,
     status: `stalled: no output for ${Math.max(1, Math.round(quietMs / 60_000))} min`,
     replyStatus: null,
@@ -242,6 +263,38 @@ function notifierFor(deps: Deps, o: NotifierOptions, retryEvent?: string): Notif
           log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
         }
       }
+      // spec 1.5 plan 21: a notice never goes to a role's own thread (a `codex exec` thread exits after its turn,
+      // so what is queued there is lost). It fails loudly, and `status` says so, instead of enqueue-accepted.
+      const roles = new Map(due.map((q) => [q, roleThreadOf(q.run, target.sessionId)]));
+      for (const [q, role] of roles) {
+        if (role === null) continue;
+        due.splice(due.indexOf(q), 1);
+        const reason = `${ROLE_THREAD_REFUSAL} (${role}, ${target.host} ${target.sessionId}), not the orchestrator's; peek(run) from the orchestrator's session takes the run back`;
+        log("error", "notify", {
+          run: q.run.id,
+          dispatch: q.notice.dispatchId,
+          target: target.sessionId,
+          reason,
+        });
+        try {
+          writeDeliveryAttempt(q.dir, {
+            attemptId: crypto.randomUUID(),
+            target: {
+              host: target.host,
+              sessionId: target.sessionId,
+              hostSessionId: target.hostSessionId,
+              name: target.name,
+            },
+            eventIds: [q.notice.eventId],
+            at: new Date(deps.now()).toISOString(),
+            status: "failed",
+            msgId: null,
+            reason,
+          });
+        } catch (e) {
+          log("warn", "notify", { error: errorMessage(e), dispatch: q.notice.dispatchId });
+        }
+      }
       // nothing is sent unless every notice's claim is on disk. A claim that cannot be written rolls the batch's
       // written claims back to failed (provably not submitted, so still deliverable), keeps that notice for a
       // later pass, and claims the rest again at once: one bad file never holds back the others

package/src/services/orchestrator-wait.ts

@@ -0,0 +1,82 @@
+import type { RunRecord } from "../domain/record.ts";
+import { liveSessionFile, readSessionFiles, type SessionFile } from "../infra/claude-session.ts";
+import { awaitsCollect } from "../infra/dispatch-dir.ts";
+import { listDispatches, liveDispatches } from "./dispatches.ts";
+import { readRecords, type Run } from "./run-store.ts";
+import { runOwner } from "./sessions.ts";
+
+export const ORCHESTRATOR_STALL_MS = 5 * 60_000;
+
+/**
+ * Plan 22 (#42 finding 3): the line shows only for a current owner (its Claude Code session live, or seen in the
+ * last day) and an unread record that ended in the last day, so an abandoned run never reads "stalled" forever.
+ */
+export const WAIT_WINDOW_MS = 24 * 3_600_000;
+
+export interface OrchestratorWait {
+  since: string;
+  seconds: number;
+  stalled: boolean;
+  unread: number;
+  /** when the line stops showing, unless the owner is seen again or another record ends */
+  until: string;
+}
+
+/**
+ * When the owner was last seen acting on the run: it took the run (`since`), or one of the run's dispatches was
+ * admitted from its session.
+ */
+function ownerSeen(run: Run, owner: { sessionId: string; since: string }): number {
+  let seen = Date.parse(owner.since) || 0;
+  for (const d of listDispatches(run))
+    if (d.admit.sessionId?.toLowerCase() === owner.sessionId.toLowerCase())
+      seen = Math.max(seen, Date.parse(d.admit.admittedAt) || 0);
+  return seen;
+}
+
+/**
+ * Pure diagnostic: queue acceptance cannot show that the orchestrator processed a completion. Null unless the
+ * run has a current owner, no live role, and an unread record that ended within WAIT_WINDOW_MS.
+ */
+export function orchestratorWait(
+  run: Run,
+  now: number,
+  records: RunRecord[] = readRecords(run).records,
+  hasLive = liveDispatches(run, now).length > 0,
+  files: () => SessionFile[] = readSessionFiles,
+): OrchestratorWait | null {
+  const owner = runOwner(run);
+  if (!owner || hasLive) return null;
+  const unread = new Set(
+    listDispatches(run)
+      .filter((d) => awaitsCollect(d.dir))
+      .map((d) => d.admit.dispatchId),
+  );
+  const recent = records.filter(
+    (r) => unread.has(r.dispatchId) && now - (Date.parse(r.endedAt) || 0) < WAIT_WINDOW_MS,
+  );
+  if (!recent.length) return null;
+  const ended = Math.max(...recent.map((r) => Date.parse(r.endedAt) || 0));
+  const live = owner.host === "claude-code" && liveSessionFile(owner.sessionId, files()) !== null;
+  const seen = live ? now : ownerSeen(run, owner);
+  if (now - seen >= WAIT_WINDOW_MS) return null;
+  const since = Math.max(ended, Date.parse(owner.since) || 0);
+  const elapsed = Math.max(0, now - since);
+  return {
+    since: new Date(since).toISOString(),
+    seconds: Math.floor(elapsed / 1000),
+    stalled: elapsed >= ORCHESTRATOR_STALL_MS,
+    unread: records.filter((r) => unread.has(r.dispatchId)).length,
+    until: new Date(Math.min(ended, seen) + WAIT_WINDOW_MS).toISOString(),
+  };
+}
+
+/**
+ * The one "waiting for orchestrator" line every view shows (#42 finding 7): seconds and `stalled` from `since` at
+ * `now`, so a memoised wait stays true between recomputes; null once it is past `until`.
+ */
+export function waitingLine(w: OrchestratorWait | null | undefined, now: number): string | null {
+  if (!w || now >= Date.parse(w.until)) return null;
+  const elapsed = Math.max(0, now - Date.parse(w.since));
+  return `${elapsed >= ORCHESTRATOR_STALL_MS ? "stalled · " : ""}waiting for orchestrator ${Math.floor(elapsed / 1000)}s`;
+}

package/src/services/pause.ts

@@ -0,0 +1,161 @@
+import { join } from "node:path";
+import { CatherdError } from "../domain/errors.ts";
+import type { Span } from "../domain/util.ts";
+import { dataDir } from "../infra/paths.ts";
+import { appendJsonl, ensureJsonlHeader, readJsonl } from "../infra/store.ts";
+/** What a pause is checked against: a run, or a workspace step about to start one. */
+type Covered = { meta: { workspace?: { id: string; step: string } } };
+import { findWorkspace, workspaceDirectory } from "./workspace-store.ts";
+
+// Spec 1.5 plan 25, "Group pause": one environment blocker (a VPN, Docker down) pauses every run on the
+// machine, or every child of a workspace, at once: admission refuses with the reason until it is resumed.
+
+/** One row per pause or resume, oldest first; `reason` null is a resume. */
+interface PauseRow {
+  at: string;
+  reason: string | null;
+}
+
+/** A pause in force: whose (the machine, or one workspace), why and since when. */
+export interface Pause {
+  scope: "machine" | "workspace";
+  /** the workspace id, for a workspace pause */
+  workspace?: string;
+  reason: string;
+  since: string;
+}
+
+/** A stretch of time something was paused; `to` null while it still is. */
+export interface PauseInterval {
+  from: string;
+  to: string | null;
+}
+
+export const machinePauseFile = (): string => join(dataDir(), "pauses.jsonl");
+export const workspacePauseFile = (id: string): string => join(workspaceDirectory(id), "pauses.jsonl");
+
+const rows = (file: string): PauseRow[] =>
+  readJsonl<PauseRow>(file).rows.filter(
+    (r) => typeof r?.at === "string" && (r.reason === null || typeof r.reason === "string"),
+  );
+
+/** The pause in force in `file`, or null: its last row, when that row pauses. */
+function current(file: string): { reason: string; since: string } | null {
+  const last = rows(file).at(-1);
+  return last && last.reason !== null ? { reason: last.reason, since: last.at } : null;
+}
+
+/** Every paused stretch `file` records, oldest first. */
+export function pauseIntervals(file: string): PauseInterval[] {
+  const out: PauseInterval[] = [];
+  for (const r of rows(file)) {
+    const open = out.at(-1);
+    if (r.reason !== null && (!open || open.to !== null)) out.push({ from: r.at, to: null });
+    else if (r.reason === null && open && open.to === null) open.to = r.at;
+  }
+  return out;
+}
+
+function append(file: string, row: PauseRow): void {
+  ensureJsonlHeader(file, "pauses");
+  appendJsonl(file, row);
+}
+
+const PAUSE_HINT = "push the reason to the owner once (PushNotification); every run it covers waits on it";
+
+/** `catherd pause --machine <reason>`: every dispatch on this machine is refused until resumed. */
+export function pauseMachine(now: number, reason: string): { pause: Pause; hints: string[] } {
+  if (!reason.trim())
+    throw new CatherdError("E_INPUT_INVALID", "a pause needs a reason", {
+      fix: 'catherd pause --machine "the VPN takes the default route"',
+    });
+  const at = new Date(now).toISOString();
+  append(machinePauseFile(), { at, reason });
+  return { pause: { scope: "machine", reason, since: at }, hints: [PAUSE_HINT] };
+}
+
+/** `catherd resume --machine`: lifts the machine pause; the previous one, or null when none was in force. */
+export function resumeMachine(now: number): { resumed: Pause | null } {
+  const was = current(machinePauseFile());
+  if (was) append(machinePauseFile(), { at: new Date(now).toISOString(), reason: null });
+  return { resumed: was ? { scope: "machine", ...was } : null };
+}
+
+/** `workspace_pause(workspace, reason)`: every child of the workspace is refused admission until resumed. */
+export function pauseWorkspace(
+  now: number,
+  i: { workspace: string; reason: string },
+): { pause: Pause; hints: string[] } {
+  const workspace = findWorkspace(i.workspace);
+  if (!i.reason.trim())
+    throw new CatherdError("E_INPUT_INVALID", "a pause needs a reason", {
+      fix: 'workspace_pause(workspace, "the VPN takes the default route")',
+    });
+  const at = new Date(now).toISOString();
+  append(workspacePauseFile(workspace.id), { at, reason: i.reason });
+  return {
+    pause: { scope: "workspace", workspace: workspace.id, reason: i.reason, since: at },
+    hints: [PAUSE_HINT],
+  };
+}
+
+/** `workspace_resume(workspace)`: lifts the workspace's pause; the previous one, or null. */
+export function resumeWorkspace(now: number, i: { workspace: string }): { resumed: Pause | null } {
+  const workspace = findWorkspace(i.workspace);
+  const file = workspacePauseFile(workspace.id);
+  const was = current(file);
+  if (was) append(file, { at: new Date(now).toISOString(), reason: null });
+  return { resumed: was ? { scope: "workspace", workspace: workspace.id, ...was } : null };
+}
+
+/** The pauses in force over `run`: the machine's, then its workspace's. */
+export function pausesFor(run: Covered | null): Pause[] {
+  const out: Pause[] = [];
+  const machine = current(machinePauseFile());
+  if (machine) out.push({ scope: "machine", ...machine });
+  const id = run?.meta.workspace?.id;
+  const ws = id ? current(workspacePauseFile(id)) : null;
+  if (id && ws) out.push({ scope: "workspace", workspace: id, ...ws });
+  return out;
+}
+
+/** Every stretch the machine, or the run's workspace, was paused, in epoch ms: `land` leaves them out. */
+export function pauseSpans(run: Covered): Span[] {
+  const id = run.meta.workspace?.id;
+  const files = [machinePauseFile(), ...(id ? [workspacePauseFile(id)] : [])];
+  return files.flatMap((f) =>
+    pauseIntervals(f).map((p) => ({ from: Date.parse(p.from), to: p.to === null ? null : Date.parse(p.to) })),
+  );
+}
+
+/** The pauses in force over any of `runs`, each once, the machine's first. */
+export function pausesOver(runs: Covered[]): Pause[] {
+  const seen = new Set<string>();
+  const out: Pause[] = [];
+  for (const p of [pausesFor(null), ...runs.map(pausesFor)].flat()) {
+    const key = `${p.scope} ${p.workspace ?? ""}`;
+    if (!seen.has(key)) {
+      seen.add(key);
+      out.push(p);
+    }
+  }
+  return out;
+}
+
+const describe = (p: Pause): string =>
+  `${p.scope === "machine" ? "the machine" : `workspace ${p.workspace}`} is paused since ${p.since}: ${p.reason}`;
+
+/** One line per pause, for status and the admission refusal. */
+export const pauseLine = describe;
+
+/** Admission refuses while a pause covers the run, naming its reason and how to lift it. */
+export function assertNotPaused(run: Covered): void {
+  const [p] = pausesFor(run);
+  if (!p) return;
+  throw new CatherdError("E_ADMIT_PAUSED", describe(p), {
+    fix:
+      p.scope === "machine"
+        ? "when the blocker is gone, the owner runs: catherd resume --machine"
+        : `when the blocker is gone, call workspace_resume(${p.workspace})`,
+  });
+}