catherd-cli 1.0.0 → 1.1.1

package/src/services/dispatch-service.ts

@@ -1,20 +1,18 @@
 import { existsSync, readFileSync } from "node:fs";
-import { relative } from "node:path";
 import { CatherdError, errorMessage, isCatherdError } from "../domain/errors.ts";
 import { assertId, parseRung } from "../domain/ids.ts";
-import { dispatchHints } from "../domain/hints.ts";
 import type { RunRecord } from "../domain/record.ts";
 import type { Role } from "../domain/roles.ts";
 import {
   awaitsCollect,
   dispatchPaths,
+  endCollect,
   readExit,
-  supervisorAlive,
   requestCancel,
+  supervisorAlive,
   tryCollect,
-  endCollect,
-  putBackCollect,
 } from "../infra/dispatch-dir.ts";
+import { lockHeld, withFileLock } from "../infra/filelock.ts";
 import { log } from "../infra/log.ts";
 import { isAlive, isSurelyAlive, killGroup } from "../infra/proc.ts";
 import { writeJsonAtomic } from "../infra/store.ts";
@@ -23,14 +21,20 @@ import { standInFor } from "./backends.ts";
 import {
   type Dispatch,
   dispatchState,
+  type FailoverFile,
   launchPath,
   listDispatches,
   liveDispatches,
+  pendingDispatches,
+  readFailover,
   readProc,
+  recordHints,
 } from "./dispatches.ts";
-import { finalizeDispatch, lastEvent, waitForFinish } from "./finalize.ts";
+import { finalizeDispatch, finalizingElsewhere, waitForFinish } from "./finalize.ts";
 import type { Deps } from "./ports.ts";
-import { findRun, readRecords, type Run } from "./run-store.ts";
+import { route } from "./lane-service.ts";
+import { findRun, readRecords, readRoutes, type Run } from "./run-store.ts";
+import { claimRun, ownsRun } from "./sessions.ts";
 import { type NotesPatch, refreshState } from "./state.ts";
 
 export interface DispatchInput {
@@ -64,28 +68,59 @@ export interface DispatchStarted {
   hints: string[];
 }
 
-export interface WaitInput {
-  run: string;
-  /** the role names to wait on; default: every uncollected dispatch of the run */
-  names?: string[];
-  /** wait until every one of them has finished, not only the first */
-  all?: boolean;
+/**
+ * A finalized dispatch once catherd has acted on it (spec §3.4): its record, its hints, and for a usage limit
+ * what failover did: the stand-in it launched, or the pause it wrote.
+ */
+export interface Settled {
+  run: Run;
+  d: Dispatch;
+  record: RunRecord;
+  hints: string[];
+  started: Dispatched | null;
+  pause: string | null;
+  /** state.md's refresh failed: its hint (the notes are kept in state.json) */
+  stateHints: string[];
 }
 
-export interface WaitResult {
-  /** each dispatch that finished, with its record and hints */
-  records: DispatchResult[];
-  /** failover stand-ins launched for a usage limit, not awaited */
-  started: Dispatched[];
-  /** every uncollected role of the run this call did not return, running or finished, stand-ins included */
-  running: string[];
-  hints: string[];
+/**
+ * Called with every dispatch this process settles, once each (the notifier, services/notifier.ts, is one). A
+ * hook that throws is logged and never stops the others.
+ */
+export const settledHooks = new Set<(s: Settled) => void | Promise<void>>();
+
+/** A live dispatch whose supervisor reported a stall (spec §3.6: quiet for half its idle timeout, not busy). */
+export interface Stalled {
+  run: Run;
+  d: Dispatch;
+  /** how long it had been quiet when the supervisor noticed */
+  quietMs: number;
 }
 
-export type Progress = (message: string) => void;
+/** Called once per watcher when its dispatch's stall.json appears (the notifier is one). */
+export const stallHooks = new Set<(s: Stalled) => void>();
 
-const hintsFor = (run: Run, d: Dispatch, r: RunRecord) =>
-  dispatchHints(r, d.admit.owns, relative(run.dir, d.dir));
+/** A watcher's poll: the first time the dispatch's stall.json is there, every stall hook hears of it. */
+export function stallPoll(run: Run, d: Dispatch): () => void {
+  let seen = false;
+  return () => {
+    if (seen || !existsSync(dispatchPaths(d.dir).stall)) return;
+    seen = true;
+    let quietMs = 0;
+    try {
+      quietMs = Number(JSON.parse(readFileSync(dispatchPaths(d.dir).stall, "utf8")).quietMs) || 0;
+    } catch {
+      // being written: the stall is reported without its length
+    }
+    for (const hook of stallHooks) {
+      try {
+        hook({ run, d, quietMs });
+      } catch (e) {
+        log("warn", "stall", { run: run.id, name: d.admit.name, error: errorMessage(e) });
+      }
+    }
+  };
+}
 
 const dispatchedOf = (d: Dispatch): Dispatched => ({
   name: d.admit.name,
@@ -105,7 +140,7 @@ export const launcher = { launch };
 
 /**
  * Starts an admitted dispatch's supervisor. Admission already left its collect mark, and a launch that
- * throws keeps it: the dispatch is recorded as lost once its start grace passes, and a `wait` returns that
+ * throws keeps it: the dispatch is recorded as lost once its start grace passes, and `result` reads that
  * record, as the error says.
  */
 function start(d: Dispatch, specPath: string): void {
@@ -113,13 +148,15 @@ function start(d: Dispatch, specPath: string): void {
     launcher.launch(d, specPath);
   } catch (e) {
     throw new CatherdError("E_IO_UNEXPECTED", `could not launch ${d.admit.name}: ${errorMessage(e)}`, {
-      fix: "wait(run) returns its record, lost, once its start grace passes; then dispatch it again",
+      fix: `result(run, "${d.admit.name}") reads its record, lost, once its start grace passes (30 s); then dispatch it again`,
     });
   }
 }
 
-/** The watchers `dispatch` started, each until its dispatch is finalized. */
+/** The watchers this process started, each until its dispatch is settled. */
 const watchers = new Set<Promise<void>>();
+/** The dispatches some watcher of this process is on (this module's, or reconcile's): watched once each. */
+export const watching = new Set<string>();
 
 /** Settles once every watcher has: tests await it so none outlives its test. */
 export async function watchersSettled(): Promise<void> {
@@ -127,186 +164,265 @@ export async function watchersSettled(): Promise<void> {
 }
 
 /**
- * Finalizes a dispatch as soon as it exits and refreshes state.md, as reconcile's watcher does, so its
- * after-snapshot holds only its own writes and the live list and the spend stay true. It never collects
- * the record and never fails over: `wait` does both, once. Its errors are logged; `wait` finalizes then.
+ * Finalizes a dispatch as soon as it exits and settles it (failover, state.md, the settled hooks), so its
+ * after-snapshot holds only its own writes and the live list and the spend stay true. Its errors are logged.
  */
-function watch(deps: Deps, run: Run, d: Dispatch): void {
+export function watch(deps: Deps, run: Run, d: Dispatch): void {
+  watching.add(d.admit.dispatchId);
   const w = (async () => {
-    await waitForFinish(d, { pollMs: deps.pollMs, tickMs: Number.POSITIVE_INFINITY, now: deps.now });
-    await finalizeDispatch(run, d);
-    const { hints } = await refreshState(run);
-    if (hints[0]) log("warn", "dispatch", { run: run.id, name: d.admit.name, hint: hints[0] });
+    await waitForFinish(d, { pollMs: deps.pollMs, now: deps.now, onPoll: stallPoll(run, d) });
+    const s = await settle(deps, run, d, await finalizeDispatch(run, d));
+    if (s.stateHints[0]) log("warn", "dispatch", { run: run.id, name: d.admit.name, hint: s.stateHints[0] });
   })()
     .catch((e: unknown) =>
       log("warn", "dispatch", { run: run.id, name: d.admit.name, error: errorMessage(e) }),
     )
-    .finally(() => watchers.delete(w));
+    .finally(() => {
+      watchers.delete(w);
+      watching.delete(d.admit.dispatchId);
+    });
   watchers.add(w);
 }
 
+/**
+ * Spec §3.3: a run this session has just taken over may have roles another session's server launched. This
+ * process watches each live one it is not already watching, so it settles them and its notifier announces them
+ * to their new owner (a second settle of a dispatch is harmless: finalize and failover run once).
+ */
+export function adopt(deps: Deps, run: Run): void {
+  for (const d of liveDispatches(run, deps.now())) if (!watching.has(d.admit.dispatchId)) watch(deps, run, d);
+}
+
+/**
+ * Spec §3.3 with the plan 10 fix-round ruling: `dispatch` and `peek` on a run make this session its owner. The
+ * owner, new or not, then takes care of what is left on the run: it watches the live roles no watcher here is
+ * on (`adopt`), records and settles each role that finished with no record (the earlier owner's server died, or
+ * a watcher here failed to finalize it), settles each unread usage limit that was never failed over, and
+ * settles again each unread record no message announced (the earlier owner's server settled it but never told
+ * its session). Each settle runs the hooks, so the notifier tells the owner. Nothing left, nothing done. A
+ * settle that fails is logged and never fails the call.
+ *
+ * Codex r4 ruling: the recovery never waits on another live process. A role whose finalizer claim, or a limit
+ * whose failover lock, another live process holds is left to that process (it writes the record, fails over
+ * and runs its own hooks). With `background` (peek, spec §3.7 "never waits") the recovery runs after the call
+ * returns, its errors logged; `dispatch` awaits it, bounded by those skips. One recovery per run at a time.
+ */
+export async function claim(deps: Deps, run: Run, o: { background?: boolean } = {}): Promise<void> {
+  if (!(await claimRun(deps, run)) && !ownsRun(deps, run)) return;
+  adopt(deps, run);
+  let r = recovering.get(run.id);
+  if (!r) {
+    const p: Promise<void> = recover(deps, run)
+      .catch((e: unknown) => log("warn", "claim", { run: run.id, error: errorMessage(e) }))
+      .finally(() => {
+        recovering.delete(run.id);
+        watchers.delete(p);
+      });
+    recovering.set(run.id, p);
+    watchers.add(p);
+    r = p;
+  }
+  if (!o.background) await r;
+}
+
+/** The recovery pass `claim` has in flight, by run id. */
+const recovering = new Map<string, Promise<void>>();
+
+/** A limit's failover lock another live process holds: its settle fails it over and announces it. */
+const failingOverElsewhere = (d: Dispatch): boolean => lockHeld(dispatchPaths(d.dir).failover);
+
+async function recover(deps: Deps, run: Run): Promise<void> {
+  const warn = (d: Dispatch, e: unknown) =>
+    log("warn", "claim", { run: run.id, name: d.admit.name, error: errorMessage(e) });
+  for (const d of pendingDispatches(run, deps.now())) {
+    if (d.state !== "finished" || watching.has(d.admit.dispatchId)) continue;
+    if (finalizingElsewhere(d.dir)) continue;
+    try {
+      await settle(deps, run, d, await finalizeDispatch(run, d));
+    } catch (e) {
+      warn(d, e);
+    }
+  }
+  const limits = unsettledLimits(run);
+  for (const { d, record } of limits) {
+    // a watcher here settles it (a peek never waits on its failover lock)
+    if (watching.has(d.admit.dispatchId) || failingOverElsewhere(d)) continue;
+    try {
+      await settle(deps, run, d, record);
+    } catch (e) {
+      warn(d, e);
+    }
+  }
+  const done = new Set(limits.map((l) => l.d.admit.dispatchId));
+  for (const { d, record } of unannounced(run)) {
+    if (done.has(d.admit.dispatchId) || watching.has(d.admit.dispatchId)) continue;
+    if (record.status === "limit" && failingOverElsewhere(d)) continue;
+    try {
+      await settle(deps, run, d, record);
+    } catch (e) {
+      warn(d, e);
+    }
+  }
+}
+
+/** The recorded dispatches of a run still unread that no message announced (no notified.json). */
+function unannounced(run: Run): { d: Dispatch; record: RunRecord }[] {
+  const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
+  return listDispatches(run).flatMap((d) => {
+    const record = records.get(d.admit.dispatchId);
+    return record && awaitsCollect(d.dir) && !existsSync(dispatchPaths(d.dir).notified)
+      ? [{ d, record }]
+      : [];
+  });
+}
+
 /**
  * Spec §4.4, plan 9 ruling 1: admit and launch one role, start its watcher, then refresh state.md with
- * `next` (after the launch, so nothing delays it). Returns at once; `wait` collects the record.
+ * `next` (after the launch, so nothing delays it). Returns at once; catherd announces the record (spec §3).
  */
 export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchStarted> {
   const run = findRun(i.run);
-  const { d, specPath } = await admit(deps, run, {
-    role: i.role,
-    name: i.name,
-    brief: i.brief,
-    rung: i.rung,
-    thread: i.thread ?? null,
-    lane: i.lane ?? null,
-    failoverFrom: null,
-  });
-  start(d, specPath);
-  watch(deps, run, d);
+  await claim(deps, run);
   const hints: string[] = [];
+  let rung = i.rung;
+  // spec 1.1 §6: a lane is routed before its first dispatch; a rung off the routed ladder starts at the routed one
+  if (i.lane !== undefined && !readRoutes(run).some((r) => r.lane === i.lane)) {
+    assertId("lane", i.lane);
+    const routed = await route(deps, { run: i.run, laneFile: `lanes/${i.lane}.md`, role: i.role });
+    if (!routed.ladder.includes(i.rung)) {
+      rung = routed.rung;
+      hints.push(`${i.rung} is not on ${i.lane}'s routed ladder: dispatched at ${routed.rung}`);
+    }
+  }
+  const { d, specPath } = await admit(
+    deps,
+    run,
+    {
+      role: i.role,
+      name: i.name,
+      brief: i.brief,
+      rung,
+      thread: i.thread ?? null,
+      lane: i.lane ?? null,
+      failoverFrom: null,
+    },
+    // a finished role no watcher here covers, recorded by admit: settled as a watcher would, so its owner hears
+    async (f, record) => {
+      if (!watching.has(f.admit.dispatchId)) await settle(deps, run, f, record);
+    },
+  );
+  try {
+    start(d, specPath);
+  } finally {
+    // a launch that failed is still watched: its record (lost, after the start grace) is announced
+    watch(deps, run, d);
+  }
   await refresh(run, i.next ? { next: i.next } : {}, hints);
   return { dispatched: dispatchedOf(d), hints };
 }
 
-const NOTHING = "nothing to wait for: every dispatch of this run has been collected; dispatch a role first";
-const nothingFor = (n: string) => `${n} has nothing to collect: result(run, "${n}") reads its last record`;
-const takenElsewhere = (n: string) =>
-  `${n}: its record went to another call (cancel or wait); result(run, "${n}") reads it`;
-const ABORTED = "wait was cancelled: nothing collected";
-
-/** The run's dispatches a `wait` has yet to hand back, read from disk, so a restarted server finds them. */
-const uncollected = (run: Run): Dispatch[] => listDispatches(run).filter((d) => awaitsCollect(d.dir));
+/** How long a settle waits for another process's failover of the same dispatch (a test seam). */
+export const failoverLock = { timeoutMs: 120_000 };
 
-/** Every uncollected role of the run, running or finished: what a `wait` still has to return. */
-const pendingNames = (run: Run): string[] => [...new Set(uncollected(run).map((d) => d.admit.name))];
-
-interface Collected {
-  result: DispatchResult;
+/** A failover's outcome before it is written down. */
+interface Outcome {
+  hints: string[];
   started: Dispatched | null;
   pause: string | null;
 }
 
-/** The record of a dispatch this wait has collected, with its failover; an unexpected failover error becomes a hint. */
-async function collected(deps: Deps, run: Run, d: Dispatch, record: RunRecord): Promise<Collected> {
-  if (record.status !== "limit")
-    return { result: { record, hints: hintsFor(run, d, record) }, started: null, pause: null };
-  try {
-    return await failover(deps, run, d, record);
-  } catch (e) {
-    return {
-      result: { record, hints: [...hintsFor(run, d, record), `failover: ${errorMessage(e)}`] },
-      started: null,
-      pause: `paused: ${record.backend} usage limit; resume when the user says so`,
-    };
-  }
-}
-
 /**
- * Plan 9 ruling 1: blocks until one of the named uncollected dispatches (default: every one of the run)
- * has finished, or every one with `all`; finalizes each (claim-based, so the dispatch's watcher, a cancel,
- * reconcile or another server finalizes it once), hands each record to this wait only, launches a usage
- * limit's stand-in without awaiting it, then refreshes state.md, with the pause a limit calls for.
- * `running` names every uncollected role this call did not return. One dispatch that cannot be finalized
- * becomes a hint and is dropped; an error that still escapes puts back what this call collected, and so
- * does an abort, which returns without collecting.
+ * Spec §3.4 (plan 10): what catherd does once a dispatch is finalized, whoever finalized it (its watcher,
+ * reconcile after a restart): a usage limit fails over (once per limited dispatch, under a lock, its outcome
+ * written to failover.json so a second settle or a restarted server reuses it), state.md is refreshed with the
+ * pause a limit calls for (a refresh that fails comes back in `stateHints`), and every settled hook runs. Never
+ * throws for a hook.
+ *
+ * Codex r4 ruling: a settle that times out waiting for the failover lock (another live process holds it) is
+ * not a failed failover. It writes nothing and runs no hook: the holder's settle writes failover.json, refreshes
+ * state.md and announces the outcome. It returns the hints and pause failover.json holds by then, else none.
  */
-export async function wait(
-  deps: Deps,
-  i: WaitInput,
-  onProgress?: Progress,
-  signal?: AbortSignal,
-): Promise<WaitResult> {
-  const run = findRun(i.run);
-  const names = i.names ? [...new Set(i.names)] : null;
-  for (const n of names ?? []) assertId("role name", n);
-  const hints: string[] = [];
-  const targets = uncollected(run).filter((d) => !names || names.includes(d.admit.name));
-  for (const n of names ?? []) if (!targets.some((d) => d.admit.name === n)) hints.push(nothingFor(n));
-  if (targets.length === 0) {
-    const running = pendingNames(run);
-    return { records: [], started: [], running, hints: running.length ? hints : [...hints, NOTHING] };
+export async function settle(deps: Deps, run: Run, d: Dispatch, record: RunRecord): Promise<Settled> {
+  let o: Outcome = { hints: recordHints(run, d, record), started: null, pause: null };
+  if (record.status === "limit") {
+    let entered = false;
+    try {
+      o = await withFileLock(
+        dispatchPaths(d.dir).failover,
+        () => {
+          entered = true;
+          return failoverOnce(deps, run, d, record);
+        },
+        { timeoutMs: failoverLock.timeoutMs },
+      );
+    } catch (e) {
+      if (!entered && isCatherdError(e) && e.code === "E_IO_LOCK") {
+        log("info", "settle", { run: run.id, name: d.admit.name, failover: "left to the lock's holder" });
+        const done = readFailover(d.dir);
+        return {
+          run,
+          d,
+          record,
+          hints: done?.hints ?? [...o.hints, "failover: another catherd process is failing it over"],
+          started: null,
+          pause: done?.pause ?? null,
+          stateHints: [],
+        };
+      }
+      o = {
+        hints: [...o.hints, `failover: ${errorMessage(e)}`],
+        started: null,
+        pause: `paused: ${record.backend} usage limit; resume when the user says so`,
+      };
+    }
   }
+  const stateHints: string[] = [];
+  await refresh(run, o.pause ? { next: o.pause } : {}, stateHints);
+  const s: Settled = { run, d, record, hints: o.hints, started: o.started, pause: o.pause, stateHints };
+  for (const hook of settledHooks) {
+    try {
+      await hook(s);
+    } catch (e) {
+      log("warn", "settle", { run: run.id, name: d.admit.name, error: errorMessage(e) });
+    }
+  }
+  return s;
+}
 
-  const open = new Map(targets.map((d) => [d.admit.dispatchId, d]));
-  const taken: Dispatch[] = [];
-  const records: DispatchResult[] = [];
-  const started: Dispatched[] = [];
-  let pause: string | null = null;
-  let ticked = Date.now();
-  const putBack = () => {
-    for (const d of taken) putBackCollect(d.dir);
-  };
+/** Under the failover lock: the outcome already written, else a failover run now and written down. */
+async function failoverOnce(deps: Deps, run: Run, d: Dispatch, record: RunRecord): Promise<Outcome> {
+  const done = readFailover(d.dir);
+  if (done) {
+    const next = done.standIn
+      ? listDispatches(run).find((x) => x.admit.dispatchId === done.standIn?.dispatchId)
+      : undefined;
+    return { hints: done.hints, started: next ? dispatchedOf(next) : null, pause: done.pause };
+  }
+  let o: Outcome;
   try {
-    for (;;) {
-      if (signal?.aborted) {
-        putBack();
-        return { records: [], started: [], running: pendingNames(run), hints: [ABORTED] };
-      }
-      const recorded = new Set(readRecords(run).records.map((r) => r.dispatchId));
-      for (const d of open.values()) {
-        if (!recorded.has(d.admit.dispatchId) && dispatchState(d, deps.now()) !== "finished") continue;
-        open.delete(d.admit.dispatchId);
-        let record: RunRecord;
-        try {
-          record = await finalizeDispatch(run, d);
-        } catch (e) {
-          // dropped, not retried: one dispatch that cannot be finalized must not hold every later wait
-          if (await tryCollect(d.dir)) {
-            endCollect(d.dir);
-            hints.push(
-              `${d.admit.name}: not finalized: ${errorMessage(e)}; result(run, "${d.admit.name}") reads its record once it has one`,
-            );
-          }
-          continue;
-        }
-        if (signal?.aborted) break;
-        if (!(await tryCollect(d.dir))) {
-          hints.push(takenElsewhere(d.admit.name));
-          continue;
-        }
-        taken.push(d);
-        const c = await collected(deps, run, d, record);
-        records.push(c.result);
-        if (c.started) started.push(c.started);
-        if (c.pause) pause = c.pause;
-      }
-      if (signal?.aborted) continue;
-      if (open.size === 0 || (!i.all && records.length > 0)) break;
-      await Bun.sleep(deps.pollMs);
-      if (onProgress && Date.now() - ticked >= deps.tickMs) {
-        ticked = Date.now();
-        for (const d of open.values()) {
-          const secs = Math.max(0, Math.round((ticked - Date.parse(d.admit.admittedAt)) / 1000));
-          const ev = lastEvent(d);
-          try {
-            onProgress(`${d.admit.name} · ${d.admit.rung} · ${secs}s${ev ? ` · ${ev}` : ""}`);
-          } catch {
-            // a progress report must never stop the wait
-          }
-        }
-      }
-    }
-    await refresh(run, pause ? { next: pause } : {}, hints);
+    o = await failover(deps, run, d, record);
   } catch (e) {
-    putBack();
-    throw e;
-  }
-  // aborted during the refresh: the result would reach no one, so put back what it collected
-  if (signal?.aborted) {
-    putBack();
-    return { records: [], started: [], running: pendingNames(run), hints: [ABORTED] };
+    o = {
+      hints: [...recordHints(run, d, record), `failover: ${errorMessage(e)}`],
+      started: null,
+      pause: `paused: ${record.backend} usage limit; resume when the user says so`,
+    };
   }
-  // in the order the roles finished, however the polls happened to see them
-  records.sort((a, b) => Date.parse(a.record.endedAt) - Date.parse(b.record.endedAt));
-  // the leases end only now, with the failovers launched and state.md refreshed: a crash before this
-  // leaves each lease to a dead owner, and the next wait collects that record again
-  for (const d of taken) endCollect(d.dir);
-  return { records, started, running: pendingNames(run), hints };
+  const file: FailoverFile = {
+    schema: 1,
+    at: new Date(deps.now()).toISOString(),
+    standIn: o.started ? { dispatchId: o.started.dispatchId, rung: o.started.rung } : null,
+    hints: o.hints,
+    pause: o.pause,
+  };
+  writeJsonAtomic(dispatchPaths(d.dir).failover, file);
+  return o;
 }
 
 /**
  * Whether a dispatch has concrete launch evidence: a live supervisor holding its lock (taken first thing),
  * launch.json (written by `launch` right after the spawn), proc.json (written by the supervisor once its
- * worker runs) or exit.json. Not the collect mark or lease: admission writes the mark before any launch.
+ * worker runs) or exit.json. Not the collect mark: admission writes the mark before any launch.
  */
 function launched(d: Dispatch): boolean {
   return (
@@ -338,24 +454,24 @@ function standInBrief(run: Run, d: Dispatch): string {
 /**
  * Spec §4.5: on a limit, launch the rung's stand-in on a fresh thread, through admission again (budget
  * included), without awaiting it. A stand-in that hits a limit too pauses the run. A stand-in already
- * launched for this record (by a wait whose collection was put back) is reused, never launched twice.
+ * admitted for this record (by a settle that died before writing failover.json) is reused, never launched
+ * twice.
  */
-async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord): Promise<Collected> {
-  const hints = hintsFor(run, d, limited);
-  const result = (h: string[]): DispatchResult => ({ record: limited, hints: h });
+async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord): Promise<Outcome> {
+  const hints = recordHints(run, d, limited);
   if (d.admit.failoverFrom !== null)
     return {
-      result: result(hints),
+      hints,
       started: null,
       pause: `paused: ${limited.backend} usage limit on ${limited.rung} and on ${d.admit.failoverFrom}`,
     };
   const paused = `paused: ${limited.backend} usage limit; resume when the user says so`;
-  const failedOver = (standIn: string, next: Dispatch): Collected => ({
-    result: result([
+  const failedOver = (standIn: string, next: Dispatch): Outcome => ({
+    hints: [
       `limit: ${limited.rung} hit a usage limit; failed over to ${standIn}`,
       // the stand-in's before-snapshot already holds the limited run's writes: surface them here
       ...hints.filter((h) => /^(violation|git-unavailable):/.test(h)),
-    ]),
+    ],
     started: dispatchedOf(next),
     pause: null,
   });
@@ -365,25 +481,25 @@ async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord):
     .at(-1);
   if (already && launched(already)) return failedOver(already.admit.rung, already);
   if (already && !recordOf(run, already) && dispatchState(already, deps.now()) === "starting") {
-    // Admitted (maybe marked) by a collector that died before launching it: launch it now, never report
-    // it unlaunched. Only the collector holding the limited record launches its stand-in, and we hold that
-    // record now, so that collector is dead. If it died after spawning a supervisor that has not taken the
-    // dispatch's lock yet, the second supervisor started here and that one race for the lock: exactly one
-    // runs the worker, the other exits touching nothing (supervisor.ts), so no worker ever runs twice.
+    // Admitted by a settle that died before launching it: launch it now, never report it unlaunched. Only
+    // the holder of the failover lock launches a stand-in, and we hold it now, so that settle is dead. If
+    // it died after spawning a supervisor that has not taken the dispatch's lock yet, the second supervisor
+    // started here and that one race for the lock: exactly one runs the worker, the other exits touching
+    // nothing (supervisor.ts), so no worker ever runs twice.
     start(already, dispatchPaths(already.dir).spec);
     watch(deps, run, already);
     return failedOver(already.admit.rung, already);
   }
   // else that stand-in never ran and is over (recorded as lost, or past its start grace): admit a new one
   const standIn = standInFor(deps.profiles.forRepo(run.meta.repo).failover, limited.rung, run.meta.repo);
-  if (!standIn) return { result: result(hints), started: null, pause: paused };
+  if (!standIn) return { hints, started: null, pause: paused };
   if (parseRung(standIn).backend === "claude") {
     const agent = deps.profiles.agentFor(run.meta.repo, d.admit.role, standIn);
     return {
-      result: result([
+      hints: [
         ...hints,
         `failover: run ${d.admit.name} as Agent(subagent_type: "${agent}"), standing in for ${limited.rung}`,
-      ]),
+      ],
       started: null,
       pause: null,
     };
@@ -399,11 +515,13 @@ async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord):
       lane: d.admit.lane,
       failoverFrom: limited.rung,
       failoverOf: d.admit.dispatchId,
+      // the stand-in answers to the session that dispatched the limited role, whoever fails it over
+      sessionId: d.admit.sessionId ?? null,
     });
   } catch (e) {
     if (!isCatherdError(e)) throw e;
     return {
-      result: result([...hints, `failover: ${standIn} refused: ${e.code} ${e.message}`]),
+      hints: [...hints, `failover: ${standIn} refused: ${e.code} ${e.message}`],
       started: null,
       pause: paused,
     };
@@ -413,6 +531,19 @@ async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord):
   return failedOver(standIn, next.d);
 }
 
+/**
+ * The finished dispatches of a run that still await catherd's action after a restart: a usage limit recorded,
+ * not yet failed over (no failover.json) and not yet read. Reconcile settles them for a run this session owns;
+ * a claim of the run (`claim`) settles them for the session that takes it over.
+ */
+export const unsettledLimits = (run: Run): { d: Dispatch; record: RunRecord }[] => {
+  const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
+  return listDispatches(run).flatMap((d) => {
+    const record = records.get(d.admit.dispatchId);
+    return record?.status === "limit" && !readFailover(d.dir) && awaitsCollect(d.dir) ? [{ d, record }] : [];
+  });
+};
+
 /** How long an orphaned worker gets between SIGTERM and SIGKILL; tests shorten it. */
 export const orphanLimits = { killGraceMs: KILL_GRACE_MS };
 
@@ -455,10 +586,17 @@ async function stopOrphan(deps: Deps, d: Dispatch): Promise<void> {
 }
 
 /**
- * Spec §4.7: stop a live dispatch (interrupt, SIGTERM, SIGKILL) and record it as cancelled. It collects the
- * record it returns, so no later `wait` returns it again.
+ * Spec §4.7: stop a live dispatch (interrupt, SIGTERM, SIGKILL) and record it as cancelled. With `read` (the MCP
+ * `cancel` tool, whose caller holds the record it returns) the record is marked read, so `peek` no longer lists
+ * it. Without it (the dashboard, `catherd runs cancel`) the record stays unread and catherd announces it to the
+ * run's owner at `later` (spec §3.6).
  */
-export async function cancel(deps: Deps, runId: string, name: string): Promise<DispatchResult> {
+export async function cancel(
+  deps: Deps,
+  runId: string,
+  name: string,
+  o: { read?: boolean } = {},
+): Promise<DispatchResult> {
   const run = findRun(runId);
   assertId("role name", name);
   const live = liveDispatches(run, deps.now()).find((d) => d.admit.name === name);
@@ -468,12 +606,15 @@ export async function cancel(deps: Deps, runId: string, name: string): Promise<D
     });
   requestCancel(live.dir);
   await stopOrphan(deps, live);
-  await waitForFinish(live, { pollMs: deps.pollMs, tickMs: Number.POSITIVE_INFINITY, now: deps.now });
+  await waitForFinish(live, { pollMs: deps.pollMs, now: deps.now });
   const record = await finalizeDispatch(run, live);
-  const mine = await tryCollect(live.dir);
-  // cancel returns at once, so its lease ends at once
-  if (mine) endCollect(live.dir);
-  const also = mine ? [] : [`${name}: a wait in flight also returned this record`];
+  let also: string[] = [];
+  if (o.read) {
+    const mine = await tryCollect(live.dir);
+    // cancel returns at once, so its lease ends at once
+    if (mine) endCollect(live.dir);
+    else also = [`${name}: result had already read this record`];
+  }
   const { hints } = await refreshState(run);
-  return { record, hints: [...hintsFor(run, live, record), ...also, ...hints] };
+  return { record, hints: [...recordHints(run, live, record), ...also, ...hints] };
 }