catherd-cli 1.4.0 → 1.5.0

package/src/services/summary.ts

@@ -8,11 +8,15 @@ import type { Tokens } from "../domain/record.ts";
 import { median } from "../domain/util.ts";
 import { nonBlankLines, readJsonl } from "../infra/store.ts";
 import { spendOf } from "./budget.ts";
-import { latestVerifierStep, type VerifierStep } from "./gate-service.ts";
+import { verifierStepView, type VerifierStepView } from "./verifier-step.ts";
+import { type Pause, pausesOver } from "./pause.ts";
+import { pinChanges } from "./run-pin.ts";
 import { type OpenQuestion, openQuestions } from "./questions.ts";
 import { type DispatchState, listDispatches, liveDispatches } from "./dispatches.ts";
+import { readDelivery, ROLE_THREAD_REFUSAL } from "../infra/delivery.ts";
 import { type RunSession, sessionFacts } from "./session-view.ts";
 import type { Deps } from "./ports.ts";
+import { orchestratorWait, type OrchestratorWait } from "./orchestrator-wait.ts";
 import {
   findRun,
   listRuns,
@@ -21,11 +25,18 @@ import {
   readRoutes,
   type Run,
   runPaths,
+  supersededBy,
 } from "./run-store.ts";
 
 export interface RunSummary {
+  /** No live roles and at least one durable completion the orchestrator has not collected. */
+  waiting?: OrchestratorWait | null;
   delivery: (DeliveryInspection & { name: string; dispatchId: string })[];
   id: string;
+  /** spec 1.5 "runs supersede": the run that took this one over; status() without a run hides it */
+  supersededBy?: string | null;
+  /** spec 1.5: what changed since the run's pin; dispatch keeps the pinned values */
+  pinChanges?: string[];
   /** spec 1.1 §8: the owner questions not answered yet, listed first */
   questions: OpenQuestion[];
   title: string;
@@ -45,8 +56,8 @@ export interface RunSummary {
   harness: { backend: string; native: number; isolated: number }[];
   budget: BudgetStatus | null;
   milestones: string[];
-  /** spec 1.1 §7: the verifier's latest gate_check, so the user sees where it is */
-  verifier: VerifierStep | null;
+  /** spec 1.1 §7: the verifier's latest gate_check, so the user sees where it is; plan 22: its age, and what closed it */
+  verifier: VerifierStepView | null;
   warnings: string[];
 }
 
@@ -61,6 +72,9 @@ export function summarizeRun(deps: Deps, run: Run): RunSummary {
   const sum = (f: (t: Tokens) => number) => records.reduce((n, r) => n + f(r.tokens), 0);
   let budget: BudgetStatus | null = null;
   const warnings = corrupt ? [`runs.jsonl: skipped ${corrupt} unreadable row(s)`] : [];
+  // spec 1.5 "Pinned per run": a change since the pin, which dispatch does not follow
+  const pinned = pinChanges(deps, run);
+  for (const c of pinned) warnings.push(`pinned: ${c}; dispatch keeps the pinned value (run_pin re-pins)`);
   try {
     budget = budgetStatus(
       spendOf(run, records, live, now),
@@ -69,7 +83,18 @@ export function summarizeRun(deps: Deps, run: Run): RunSummary {
   } catch (e) {
     warnings.push(`budget: ${isCatherdError(e) ? e.message : String(e)}`);
   }
+  // spec 1.5 plan 21: a notice refused because its target was a role's thread, while that is its latest attempt
+  for (const d of listDispatches(run)) {
+    try {
+      const last = readDelivery(d.dir).at(-1);
+      if (last?.status === "failed" && last.reason?.startsWith(ROLE_THREAD_REFUSAL))
+        warnings.push(`${d.admit.name}: its notice was not sent: ${last.reason}`);
+    } catch {
+      // unreadable delivery evidence is the delivery row's to show
+    }
+  }
   return {
+    waiting: orchestratorWait(run, now, records, live.length > 0),
     delivery: listDispatches(run).flatMap((d) =>
       inspectDeliveries(run, d, recorded.has(d.admit.dispatchId)).map((inspection) => ({
         name: d.admit.name,
@@ -78,6 +103,8 @@ export function summarizeRun(deps: Deps, run: Run): RunSummary {
       })),
     ),
     id: run.id,
+    supersededBy: supersededBy(run)?.by ?? null,
+    pinChanges: pinned,
     questions: openQuestions(run),
     title: run.meta.title,
     repo: run.meta.repo,
@@ -116,40 +143,53 @@ export function summarizeRun(deps: Deps, run: Run): RunSummary {
     }),
     budget,
     milestones: nonBlankLines(runPaths(run.dir).ledger).slice(1),
-    verifier: latestVerifierStep(run),
+    verifier: verifierStepView(run, now),
     warnings,
   };
 }
 
-/** `status(run?)`: that run; without one, every run with a live role, else the newest run. */
+/**
+ * `status(run?)`: that run; without one, the live runs, else the runs waiting for their orchestrator (a current
+ * owner, a recent unread record: plan 22), else the newest run.
+ */
 export function status(
   deps: Deps,
   runId?: string,
   queue: QueueCapability | null = null,
 ): {
+  /** spec 1.5 "Group pause": the machine's pause and the listed runs' workspace pauses, shown first */
+  paused: Pause[];
   version: string;
   runs: RunSummary[];
   warnings: string[];
   host: HostContext;
   queue: QueueCapability | null;
 } {
-  if (runId)
+  if (runId) {
+    const run = findRun(runId);
     return {
+      paused: pausesOver([run]),
       host: inspectionHost(deps.host),
       queue,
       version: deps.version,
-      runs: [summarizeRun(deps, findRun(runId))],
+      runs: [summarizeRun(deps, run)],
       warnings: [],
     };
-  const { runs, corrupt } = listRuns();
+  }
+  const { runs: listed, corrupt } = listRuns();
   const warnings = corrupt.map((c) => `skipped run ${c.id}: ${c.reason}`);
+  // a superseded run is closed: only status(run) shows it
+  const runs = listed.filter((r) => !supersededBy(r));
   const all = runs.map((r) => summarizeRun(deps, r));
   const live = all.filter((s) => s.live.length > 0);
+  const waiting = all.filter((s) => s.waiting);
+  const shown = live.length ? live : waiting.length ? waiting : all.slice(0, 1);
   return {
+    paused: pausesOver(runs.filter((r) => shown.some((s) => s.id === r.id))),
     host: inspectionHost(deps.host),
     queue,
     version: deps.version,
-    runs: live.length ? live : all.slice(0, 1),
+    runs: shown,
     warnings,
   };
 }

package/src/services/treat-likes.ts

@@ -5,7 +5,7 @@ import { tryParseRung } from "../domain/ids.ts";
 import { ROLES } from "../domain/roles.ts";
 import { canonicalRung, loadCatalog, readOverride, removeTreatLikes } from "./catalog-service.ts";
 import { getProfile, listProfiles } from "./profile-store.ts";
-import { type Suggestion, suggestStandIns } from "./standins.ts";
+import { barDimsIn, type Suggestion, suggestStandIns } from "./standins.ts";
 
 /** A profile's rung that would lean on an inferred stand-in, or be left with no value at all (spec 1.2 §6.4). */
 export interface LeftOnStandIn {
@@ -15,6 +15,11 @@ export interface LeftOnStandIn {
   dims: Dim[];
   /** no value of its own and no stand-in near enough: routing skips it */
   unscored: boolean;
+  /**
+   * (1.2 minor) the bar dimensions it keeps no value on at all, though it keeps others: present only when some
+   * are; no bar that needs one is cleared
+   */
+  missing?: Dim[];
 }
 
 /**
@@ -50,8 +55,19 @@ export function leftOnStandIns(
       const canonical = rungInfo(c, rung).canonical;
       if (!gone.includes(canonical)) continue;
       const s = scoresOf(c, canonical);
-      if (!s) out.push({ profile, rung, dims: [], unscored: true });
-      else if (s.inferred.length) out.push({ profile, rung, dims: s.inferred, unscored: false });
+      if (!s) {
+        out.push({ profile, rung, dims: [], unscored: true });
+        continue;
+      }
+      const missing = barDimsIn(c).filter((d) => s.values[d] === undefined);
+      if (s.inferred.length || missing.length)
+        out.push({
+          profile,
+          rung,
+          dims: s.inferred,
+          unscored: false,
+          ...(missing.length ? { missing } : {}),
+        });
     }
   }
   return out;

package/src/services/verifier-brief.ts

@@ -0,0 +1,23 @@
+import { withGateNotes } from "../domain/gate-brief.ts";
+import { gateEnvLines, readGateEnv } from "./gate-env.ts";
+import { recordedItems } from "./gate-service.ts";
+import type { Run } from "./run-store.ts";
+
+// Plan 23: a dispatched verifier's brief ends with the gate's rules and the run's facts: the milestone's
+// recorded items (reuse their names), the failed ones (a re-check runs them first, each command capped), and
+// the repo's gate environment.
+
+/** The milestone a verifier dispatch checks: the first `M<n>` its name names as a word (verifier-M1 → M1). */
+export const milestoneOfName = (name: string): string | null =>
+  /(?:^|[^A-Za-z0-9])(M\d+)(?=$|[-._\s])/.exec(name)?.[1] ?? null;
+
+/** A verifier's brief with the gate notes, once; any other role's brief as it is. */
+export async function verifierBrief(run: Run, role: string, name: string, brief: string): Promise<string> {
+  if (role !== "verifier") return brief;
+  const milestone = milestoneOfName(name);
+  return withGateNotes(brief, {
+    milestone,
+    recorded: milestone ? await recordedItems(run, milestone) : [],
+    env: gateEnvLines(await readGateEnv(run.meta.repo)),
+  });
+}

package/src/services/verifier-step.ts

@@ -0,0 +1,48 @@
+import { liveSessionFile, readSessionFiles, type SessionFile } from "../infra/claude-session.ts";
+import { readJsonl } from "../infra/store.ts";
+import { gatesFile, latestVerifierStep, type VerifierStep } from "./gate-service.ts";
+import { readAgentRuns, readRecords, type Run } from "./run-store.ts";
+import { runOwner } from "./sessions.ts";
+
+/** What ended a verifier step (plan 22: a step nothing ends shows as live forever). */
+export type StepClosedBy = "gate_pass" | "agent-run" | "record" | "owner-gone";
+
+export interface VerifierStepView extends VerifierStep {
+  /** seconds since the step began */
+  secs: number;
+  open: boolean;
+  closedBy: StepClosedBy | null;
+}
+
+/**
+ * Plan 22: the verifier's latest step with its age, closed by the first evidence after it: a `gate_pass` of its
+ * item in this run, a native verifier's `record_agent_run`, a verifier dispatch's record, or its owner session
+ * gone (another session took the run since, or the owner's Claude Code session no longer runs). A Codex owner
+ * that still holds the run has no liveness to read: its step stays open until other evidence ends it.
+ */
+export function verifierStepView(
+  run: Run,
+  now: number,
+  files: () => SessionFile[] = readSessionFiles,
+): VerifierStepView | null {
+  const step = latestVerifierStep(run);
+  if (!step) return null;
+  const at = Date.parse(step.at) || 0;
+  const after = (t: string | undefined) => (Date.parse(t ?? "") || 0) >= at;
+  const passed = readJsonl<{ run?: unknown; item?: unknown; at?: string }>(
+    gatesFile(run.meta.repo),
+  ).rows.some((p) => p?.run === run.id && p.item === step.item && after(p.at));
+  const owner = runOwner(run);
+  const closedBy: StepClosedBy | null = passed
+    ? "gate_pass"
+    : readAgentRuns(run).some((a) => a.role === "verifier" && after(a.at))
+      ? "agent-run"
+      : readRecords(run).records.some((r) => r.role === "verifier" && after(r.endedAt))
+        ? "record"
+        : owner &&
+            ((Date.parse(owner.since) || 0) > at ||
+              (owner.host === "claude-code" && liveSessionFile(owner.sessionId, files()) === null))
+          ? "owner-gone"
+          : null;
+  return { ...step, secs: Math.max(0, Math.floor((now - at) / 1000)), open: closedBy === null, closedBy };
+}

package/src/services/workspace-admission.ts

@@ -0,0 +1,267 @@
+import { existsSync, readFileSync } from "node:fs";
+import { budgetStatus, formatBudget, type BudgetStatus, type Spend } from "../domain/budget.ts";
+import { CatherdError, errorMessage, isCatherdError } from "../domain/errors.ts";
+import type { Workspace, WorkspaceStep } from "../domain/workspace.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { git, gitToplevel } from "../infra/git.ts";
+import { writeTextAtomic } from "../infra/store.ts";
+import { spendOf } from "./budget.ts";
+import { pendingDispatches } from "./dispatches.ts";
+import { landedCommits, landedMilestones } from "./milestones.ts";
+import { readRecords, type Run, runPaths } from "./run-store.ts";
+import {
+  findWorkspace,
+  saveWorkspace,
+  WORKSPACE_LOCK_WAIT_MS,
+  workspaceChildren,
+  workspaceDirectory,
+  workspacePaths,
+} from "./workspace-store.ts";
+
+/**
+ * Count each recorded or unrecorded dispatch once; elapsed minutes belong to the parent and start at its first
+ * child (#43 finding 4), none before one. With `warnings`, a child whose cost evidence cannot be read is
+ * skipped and named there (status); without, it throws E_RUN_CORRUPT (admission against a cap, which cannot
+ * prove the cap holds without it).
+ */
+export async function workspaceSpend(
+  workspace: Workspace,
+  now: number,
+  children = workspaceChildren(workspace),
+  warnings?: string[],
+): Promise<Spend> {
+  const first = Math.min(...children.map((c) => Date.parse(c.meta.createdAt)).filter(Number.isFinite));
+  const total: Spend = {
+    minutes: Number.isFinite(first) ? Math.max(0, (now - first) / 60_000) : 0,
+    tokens: 0,
+    usd: 0,
+  };
+  for (const child of children) {
+    try {
+      const spent = await withFileLock(runPaths(child.dir).runs, () => {
+        const { records, corrupt } = readRecords(child);
+        if (corrupt) throw new CatherdError("E_RUN_CORRUPT", `run ${child.id} has unreadable cost records`);
+        return spendOf(child, records, pendingDispatches(child, now, records, true), now, true);
+      });
+      total.tokens += spent.tokens;
+      total.usd += spent.usd;
+    } catch (e) {
+      if (!warnings || !isCatherdError(e) || e.code !== "E_RUN_CORRUPT") throw e;
+      warnings.push(`spend leaves out ${child.id}: ${errorMessage(e)}`);
+    }
+  }
+  return total;
+}
+
+/** The workspace budget a child run's routing sees; a child whose evidence is unreadable counts nothing. */
+export async function workspaceBudget(run: Run, now: number): Promise<BudgetStatus | null> {
+  if (!run.meta.workspace) return null;
+  const workspace = findWorkspace(run.meta.workspace.id);
+  return budgetStatus(await workspaceSpend(workspace, now, undefined, []), workspace.budget);
+}
+
+/** Why a dependency keeps a step waiting: the dependency's step id and the reason, in words. */
+export interface Blocker {
+  step: string;
+  why: string;
+}
+
+/** The dependency's uncollected dispatches, read under its records lock: a reason, or null when none. */
+async function uncollected(child: Run, now: number): Promise<string | null> {
+  const step = child.meta.workspace?.step ?? child.id;
+  try {
+    const pending = await withFileLock(runPaths(child.dir).runs, () => {
+      const { records, corrupt } = readRecords(child);
+      if (corrupt) throw new CatherdError("E_RUN_CORRUPT", `run ${child.id} has unreadable dispatch records`);
+      return pendingDispatches(child, now, records, true);
+    });
+    return pending.length
+      ? `${step} has ${pending.length} dispatch(es) not collected (${pending.map((d) => d.admit.name).join(", ")})`
+      : null;
+  } catch (e) {
+    if (isCatherdError(e) && e.code === "E_RUN_CORRUPT") return `${step} cannot be read: ${e.message}`;
+    throw e;
+  }
+}
+
+/** A landed milestone that differs from the step's only by case: `m1` for `M1` (#43 finding 7). */
+const caseTwin = (landed: string[], milestone: string): string | undefined =>
+  landed.find((m) => m !== milestone && m.toLowerCase() === milestone.toLowerCase());
+
+/**
+ * #43 finding 8: the one dependency rule admission, child start and status share. A dependency releases its
+ * dependents once its child has landed the dependency's milestone and every dispatch it admitted is
+ * collected. Reads only the dependencies' own evidence (#43 finding 2); an unreadable one blocks, named.
+ */
+export async function dependencyBlockers(
+  workspace: Workspace,
+  step: WorkspaceStep,
+  children: Run[],
+  now: number,
+  o: { merge?: boolean } = {},
+): Promise<Blocker[]> {
+  const out: Blocker[] = [];
+  for (const id of step.dependsOn) {
+    const before = workspace.steps.find((s) => s.id === id);
+    const child = children.find((c) => c.meta.workspace?.step === id);
+    if (!before || !child) {
+      out.push({ step: id, why: `${id} has not started` });
+      continue;
+    }
+    const landed = landedMilestones(child);
+    if (!landed.includes(before.milestone)) {
+      const twin = caseTwin(landed, before.milestone);
+      out.push({
+        step: id,
+        why: `${id} has not landed ${before.milestone}${twin ? ` (it landed ${twin}: the step completes on ${before.milestone})` : ""}`,
+      });
+      continue;
+    }
+    const why = (await uncollected(child, now)) ?? (o.merge ? await unmerged(before, child) : null);
+    if (why) out.push({ step: id, why });
+  }
+  return out;
+}
+
+/** The commit the ledger holds for `milestone`, its latest landing; null when it never landed. */
+function landedCommitOf(run: Run, milestone: string): string | null {
+  const commits = landedCommits(run);
+  const milestones = landedMilestones(run);
+  const at = milestones.lastIndexOf(milestone);
+  return at < 0 ? null : (commits[at] ?? null);
+}
+
+/**
+ * A `release: "merge"` step releases its dependents once its landed commit is an ancestor of its base ref
+ * (`git merge-base --is-ancestor`), in the repository as it stands: catherd never fetches or polls.
+ */
+async function unmerged(step: WorkspaceStep, child: Run): Promise<string | null> {
+  if (step.release !== "merge" || !step.base) return null;
+  const commit = landedCommitOf(child, step.milestone);
+  if (!commit) return `${step.id} has no landed commit for ${step.milestone} in its ledger`;
+  const r = await git(child.meta.repo, ["merge-base", "--is-ancestor", commit, step.base]);
+  if (r.kind === "ok") return null;
+  if (r.kind === "failed" && r.exit === 1)
+    return `${step.id}'s ${step.milestone} (${commit.slice(0, 7)}) is not merged into ${step.base} yet: merge it, git fetch, then ask again`;
+  return `${step.id}: git cannot tell whether ${commit.slice(0, 7)} is in ${step.base} (does the ref exist in ${child.meta.repo}?)`;
+}
+
+export function withRunAdmission<T>(run: Run, now: () => number, admit: () => Promise<T>): Promise<T> {
+  return withWorkspaceAdmission(run, now, () => withFileLock(runPaths(run.dir).admission, admit));
+}
+
+/** Parent → child admission → child records; siblings share the actual admission boundary. */
+export async function withWorkspaceAdmission<T>(
+  run: Run,
+  clock: number | (() => number),
+  admit: () => Promise<T>,
+): Promise<T> {
+  const link = run.meta.workspace;
+  if (!link) return admit();
+  const workspace = findWorkspace(link.id);
+  // git outside the workspace lock (#43 finding 5): the checkout check needs no sibling to wait
+  if ((await gitToplevel(run.meta.repo)) !== run.meta.repo)
+    throw new CatherdError(
+      "E_IO_PATH",
+      `workspace repository of ${link.step} is no longer the captured git root`,
+    );
+  return withFileLock(
+    workspacePaths(workspaceDirectory(workspace.id)).admission,
+    async () => {
+      const now = typeof clock === "function" ? clock() : clock;
+      const children = workspaceChildren(workspace);
+      const step = workspace.steps.find((s) => s.id === link.step);
+      if (!step || !children.some((c) => c.id === run.id))
+        throw new CatherdError("E_INPUT_INVALID", "the run is not a member of this workspace execution", {
+          fix: "start the child with workspace_child_start",
+        });
+      // #43 finding 7: a landed step's child admits again (a post-land fix); its dependents wait for it
+      const blockers = await dependencyBlockers(workspace, step, children, now);
+      if (blockers.length)
+        throw new CatherdError(
+          "E_INPUT_INVALID",
+          `${step.id} is waiting: ${blockers.map((b) => b.why).join("; ")}`,
+          {
+            fix: "land the dependency's completion milestone and collect its finished dispatches first",
+          },
+        );
+      await assertWorkspaceBudget(workspace, now, children);
+      return admit();
+    },
+    { timeoutMs: WORKSPACE_LOCK_WAIT_MS },
+  );
+}
+
+/** Whether the budget caps anything: without a cap, no sibling's evidence is read (#43 finding 2). */
+const capped = (workspace: Workspace): boolean =>
+  Object.values(workspace.budget).some((v) => typeof v === "number");
+
+/** Refuses E_RUN_BUDGET once a capped workspace budget is spent; reads every child only when capped. */
+export async function assertWorkspaceBudget(
+  workspace: Workspace,
+  now: number,
+  children: Run[],
+): Promise<void> {
+  if (!capped(workspace)) return;
+  const budget = budgetStatus(await workspaceSpend(workspace, now, children), workspace.budget);
+  if (budget && budget.fraction >= 1)
+    throw new CatherdError("E_RUN_BUDGET", `the workspace budget is spent: ${formatBudget(budget)}`, {
+      fix: `finish with the work already admitted, or raise it: workspace_budget(${workspace.id}, …)`,
+    });
+}
+
+/**
+ * `workspace_budget`: sets the caps given (a number) or removes them (null), keeping the others; the owner's
+ * way to raise a spent budget without a new workspace (#43 finding 4). Returns the budget and its status.
+ */
+export async function setWorkspaceBudget(
+  deps: { now: () => number },
+  i: { workspace: string; minutes?: number | null; tokens?: number | null; usd?: number | null },
+): Promise<{ budget: Workspace["budget"]; spend: Spend; status: BudgetStatus | null }> {
+  const workspace = findWorkspace(i.workspace);
+  return withFileLock(
+    workspacePaths(workspaceDirectory(workspace.id)).admission,
+    async () => {
+      const budget: Workspace["budget"] = { ...findWorkspace(workspace.id).budget };
+      for (const key of ["minutes", "tokens", "usd"] as const) {
+        const v = i[key];
+        if (v === null) delete budget[key];
+        else if (v !== undefined) budget[key] = v;
+      }
+      const saved = saveWorkspace({ ...findWorkspace(workspace.id), budget });
+      const spend = await workspaceSpend(saved, deps.now(), undefined, []);
+      return { budget: saved.budget, spend, status: budgetStatus(spend, saved.budget) };
+    },
+    { timeoutMs: WORKSPACE_LOCK_WAIT_MS },
+  );
+}
+
+/** The shared contract is frozen when the first child starts, then copied into child dossiers. */
+export async function workspaceContract(i: {
+  workspace: string;
+  content?: string;
+}): Promise<{ path: string; content: string }> {
+  const workspace = findWorkspace(i.workspace);
+  const paths = workspacePaths(workspaceDirectory(workspace.id));
+  if (i.content === undefined)
+    return {
+      path: paths.contract,
+      content: existsSync(paths.contract) ? readFileSync(paths.contract, "utf8") : "",
+    };
+  return withFileLock(
+    paths.admission,
+    () => {
+      if (workspaceChildren(workspace).length)
+        throw new CatherdError(
+          "E_INPUT_INVALID",
+          "the workspace contract is frozen because a child has started",
+          {
+            fix: "start a new workspace execution to change the shared contract",
+          },
+        );
+      writeTextAtomic(paths.contract, i.content as string);
+      return { path: paths.contract, content: i.content as string };
+    },
+    { timeoutMs: WORKSPACE_LOCK_WAIT_MS },
+  );
+}