catherd-cli 0.2.0 → 1.0.0

package/src/services/dispatch-service.ts

@@ -0,0 +1,479 @@
+import { existsSync, readFileSync } from "node:fs";
+import { relative } from "node:path";
+import { CatherdError, errorMessage, isCatherdError } from "../domain/errors.ts";
+import { assertId, parseRung } from "../domain/ids.ts";
+import { dispatchHints } from "../domain/hints.ts";
+import type { RunRecord } from "../domain/record.ts";
+import type { Role } from "../domain/roles.ts";
+import {
+  awaitsCollect,
+  dispatchPaths,
+  readExit,
+  supervisorAlive,
+  requestCancel,
+  tryCollect,
+  endCollect,
+  putBackCollect,
+} from "../infra/dispatch-dir.ts";
+import { log } from "../infra/log.ts";
+import { isAlive, isSurelyAlive, killGroup } from "../infra/proc.ts";
+import { writeJsonAtomic } from "../infra/store.ts";
+import { admit, KILL_GRACE_MS, laneFile, launch } from "./admission.ts";
+import { standInFor } from "./backends.ts";
+import {
+  type Dispatch,
+  dispatchState,
+  launchPath,
+  listDispatches,
+  liveDispatches,
+  readProc,
+} from "./dispatches.ts";
+import { finalizeDispatch, lastEvent, waitForFinish } from "./finalize.ts";
+import type { Deps } from "./ports.ts";
+import { findRun, readRecords, type Run } from "./run-store.ts";
+import { type NotesPatch, refreshState } from "./state.ts";
+
+export interface DispatchInput {
+  run: string;
+  role: Role;
+  name: string;
+  brief: string;
+  rung: string;
+  thread?: string;
+  lane?: string;
+  next?: string;
+}
+
+/** One finished dispatch: its record and what to do next. */
+export interface DispatchResult {
+  record: RunRecord;
+  hints: string[];
+}
+
+/** A dispatch that has been admitted and launched. */
+export interface Dispatched {
+  name: string;
+  role: Role;
+  rung: string;
+  dispatchId: string;
+  admittedAt: string;
+}
+
+export interface DispatchStarted {
+  dispatched: Dispatched;
+  hints: string[];
+}
+
+export interface WaitInput {
+  run: string;
+  /** the role names to wait on; default: every uncollected dispatch of the run */
+  names?: string[];
+  /** wait until every one of them has finished, not only the first */
+  all?: boolean;
+}
+
+export interface WaitResult {
+  /** each dispatch that finished, with its record and hints */
+  records: DispatchResult[];
+  /** failover stand-ins launched for a usage limit, not awaited */
+  started: Dispatched[];
+  /** every uncollected role of the run this call did not return, running or finished, stand-ins included */
+  running: string[];
+  hints: string[];
+}
+
+export type Progress = (message: string) => void;
+
+const hintsFor = (run: Run, d: Dispatch, r: RunRecord) =>
+  dispatchHints(r, d.admit.owns, relative(run.dir, d.dir));
+
+const dispatchedOf = (d: Dispatch): Dispatched => ({
+  name: d.admit.name,
+  role: d.admit.role,
+  rung: d.admit.rung,
+  dispatchId: d.admit.dispatchId,
+  admittedAt: d.admit.admittedAt,
+});
+
+/** Refreshes state.md (spec §4.5 notes kept on a git failure); a refresh that fails adds its hint once. */
+async function refresh(run: Run, change: NotesPatch, hints: string[]): Promise<void> {
+  for (const h of (await refreshState(run, change)).hints) if (!hints.includes(h)) hints.push(h);
+}
+
+/** How a dispatch's supervisor is started; tests replace it to make a launch fail. */
+export const launcher = { launch };
+
+/**
+ * Starts an admitted dispatch's supervisor. Admission already left its collect mark, and a launch that
+ * throws keeps it: the dispatch is recorded as lost once its start grace passes, and a `wait` returns that
+ * record, as the error says.
+ */
+function start(d: Dispatch, specPath: string): void {
+  try {
+    launcher.launch(d, specPath);
+  } catch (e) {
+    throw new CatherdError("E_IO_UNEXPECTED", `could not launch ${d.admit.name}: ${errorMessage(e)}`, {
+      fix: "wait(run) returns its record, lost, once its start grace passes; then dispatch it again",
+    });
+  }
+}
+
+/** The watchers `dispatch` started, each until its dispatch is finalized. */
+const watchers = new Set<Promise<void>>();
+
+/** Settles once every watcher has: tests await it so none outlives its test. */
+export async function watchersSettled(): Promise<void> {
+  while (watchers.size > 0) await Promise.all(watchers);
+}
+
+/**
+ * Finalizes a dispatch as soon as it exits and refreshes state.md, as reconcile's watcher does, so its
+ * after-snapshot holds only its own writes and the live list and the spend stay true. It never collects
+ * the record and never fails over: `wait` does both, once. Its errors are logged; `wait` finalizes then.
+ */
+function watch(deps: Deps, run: Run, d: Dispatch): void {
+  const w = (async () => {
+    await waitForFinish(d, { pollMs: deps.pollMs, tickMs: Number.POSITIVE_INFINITY, now: deps.now });
+    await finalizeDispatch(run, d);
+    const { hints } = await refreshState(run);
+    if (hints[0]) log("warn", "dispatch", { run: run.id, name: d.admit.name, hint: hints[0] });
+  })()
+    .catch((e: unknown) =>
+      log("warn", "dispatch", { run: run.id, name: d.admit.name, error: errorMessage(e) }),
+    )
+    .finally(() => watchers.delete(w));
+  watchers.add(w);
+}
+
+/**
+ * Spec §4.4, plan 9 ruling 1: admit and launch one role, start its watcher, then refresh state.md with
+ * `next` (after the launch, so nothing delays it). Returns at once; `wait` collects the record.
+ */
+export async function dispatch(deps: Deps, i: DispatchInput): Promise<DispatchStarted> {
+  const run = findRun(i.run);
+  const { d, specPath } = await admit(deps, run, {
+    role: i.role,
+    name: i.name,
+    brief: i.brief,
+    rung: i.rung,
+    thread: i.thread ?? null,
+    lane: i.lane ?? null,
+    failoverFrom: null,
+  });
+  start(d, specPath);
+  watch(deps, run, d);
+  const hints: string[] = [];
+  await refresh(run, i.next ? { next: i.next } : {}, hints);
+  return { dispatched: dispatchedOf(d), hints };
+}
+
+const NOTHING = "nothing to wait for: every dispatch of this run has been collected; dispatch a role first";
+const nothingFor = (n: string) => `${n} has nothing to collect: result(run, "${n}") reads its last record`;
+const takenElsewhere = (n: string) =>
+  `${n}: its record went to another call (cancel or wait); result(run, "${n}") reads it`;
+const ABORTED = "wait was cancelled: nothing collected";
+
+/** The run's dispatches a `wait` has yet to hand back, read from disk, so a restarted server finds them. */
+const uncollected = (run: Run): Dispatch[] => listDispatches(run).filter((d) => awaitsCollect(d.dir));
+
+/** Every uncollected role of the run, running or finished: what a `wait` still has to return. */
+const pendingNames = (run: Run): string[] => [...new Set(uncollected(run).map((d) => d.admit.name))];
+
+interface Collected {
+  result: DispatchResult;
+  started: Dispatched | null;
+  pause: string | null;
+}
+
+/** The record of a dispatch this wait has collected, with its failover; an unexpected failover error becomes a hint. */
+async function collected(deps: Deps, run: Run, d: Dispatch, record: RunRecord): Promise<Collected> {
+  if (record.status !== "limit")
+    return { result: { record, hints: hintsFor(run, d, record) }, started: null, pause: null };
+  try {
+    return await failover(deps, run, d, record);
+  } catch (e) {
+    return {
+      result: { record, hints: [...hintsFor(run, d, record), `failover: ${errorMessage(e)}`] },
+      started: null,
+      pause: `paused: ${record.backend} usage limit; resume when the user says so`,
+    };
+  }
+}
+
+/**
+ * Plan 9 ruling 1: blocks until one of the named uncollected dispatches (default: every one of the run)
+ * has finished, or every one with `all`; finalizes each (claim-based, so the dispatch's watcher, a cancel,
+ * reconcile or another server finalizes it once), hands each record to this wait only, launches a usage
+ * limit's stand-in without awaiting it, then refreshes state.md, with the pause a limit calls for.
+ * `running` names every uncollected role this call did not return. One dispatch that cannot be finalized
+ * becomes a hint and is dropped; an error that still escapes puts back what this call collected, and so
+ * does an abort, which returns without collecting.
+ */
+export async function wait(
+  deps: Deps,
+  i: WaitInput,
+  onProgress?: Progress,
+  signal?: AbortSignal,
+): Promise<WaitResult> {
+  const run = findRun(i.run);
+  const names = i.names ? [...new Set(i.names)] : null;
+  for (const n of names ?? []) assertId("role name", n);
+  const hints: string[] = [];
+  const targets = uncollected(run).filter((d) => !names || names.includes(d.admit.name));
+  for (const n of names ?? []) if (!targets.some((d) => d.admit.name === n)) hints.push(nothingFor(n));
+  if (targets.length === 0) {
+    const running = pendingNames(run);
+    return { records: [], started: [], running, hints: running.length ? hints : [...hints, NOTHING] };
+  }
+
+  const open = new Map(targets.map((d) => [d.admit.dispatchId, d]));
+  const taken: Dispatch[] = [];
+  const records: DispatchResult[] = [];
+  const started: Dispatched[] = [];
+  let pause: string | null = null;
+  let ticked = Date.now();
+  const putBack = () => {
+    for (const d of taken) putBackCollect(d.dir);
+  };
+  try {
+    for (;;) {
+      if (signal?.aborted) {
+        putBack();
+        return { records: [], started: [], running: pendingNames(run), hints: [ABORTED] };
+      }
+      const recorded = new Set(readRecords(run).records.map((r) => r.dispatchId));
+      for (const d of open.values()) {
+        if (!recorded.has(d.admit.dispatchId) && dispatchState(d, deps.now()) !== "finished") continue;
+        open.delete(d.admit.dispatchId);
+        let record: RunRecord;
+        try {
+          record = await finalizeDispatch(run, d);
+        } catch (e) {
+          // dropped, not retried: one dispatch that cannot be finalized must not hold every later wait
+          if (await tryCollect(d.dir)) {
+            endCollect(d.dir);
+            hints.push(
+              `${d.admit.name}: not finalized: ${errorMessage(e)}; result(run, "${d.admit.name}") reads its record once it has one`,
+            );
+          }
+          continue;
+        }
+        if (signal?.aborted) break;
+        if (!(await tryCollect(d.dir))) {
+          hints.push(takenElsewhere(d.admit.name));
+          continue;
+        }
+        taken.push(d);
+        const c = await collected(deps, run, d, record);
+        records.push(c.result);
+        if (c.started) started.push(c.started);
+        if (c.pause) pause = c.pause;
+      }
+      if (signal?.aborted) continue;
+      if (open.size === 0 || (!i.all && records.length > 0)) break;
+      await Bun.sleep(deps.pollMs);
+      if (onProgress && Date.now() - ticked >= deps.tickMs) {
+        ticked = Date.now();
+        for (const d of open.values()) {
+          const secs = Math.max(0, Math.round((ticked - Date.parse(d.admit.admittedAt)) / 1000));
+          const ev = lastEvent(d);
+          try {
+            onProgress(`${d.admit.name} · ${d.admit.rung} · ${secs}s${ev ? ` · ${ev}` : ""}`);
+          } catch {
+            // a progress report must never stop the wait
+          }
+        }
+      }
+    }
+    await refresh(run, pause ? { next: pause } : {}, hints);
+  } catch (e) {
+    putBack();
+    throw e;
+  }
+  // aborted during the refresh: the result would reach no one, so put back what it collected
+  if (signal?.aborted) {
+    putBack();
+    return { records: [], started: [], running: pendingNames(run), hints: [ABORTED] };
+  }
+  // in the order the roles finished, however the polls happened to see them
+  records.sort((a, b) => Date.parse(a.record.endedAt) - Date.parse(b.record.endedAt));
+  // the leases end only now, with the failovers launched and state.md refreshed: a crash before this
+  // leaves each lease to a dead owner, and the next wait collects that record again
+  for (const d of taken) endCollect(d.dir);
+  return { records, started, running: pendingNames(run), hints };
+}
+
+/**
+ * Whether a dispatch has concrete launch evidence: a live supervisor holding its lock (taken first thing),
+ * launch.json (written by `launch` right after the spawn), proc.json (written by the supervisor once its
+ * worker runs) or exit.json. Not the collect mark or lease: admission writes the mark before any launch.
+ */
+function launched(d: Dispatch): boolean {
+  return (
+    supervisorAlive(d.dir) ||
+    existsSync(launchPath(d.dir)) ||
+    readProc(d.dir) !== null ||
+    readExit(d.dir) !== null
+  );
+}
+
+const recordOf = (run: Run, d: Dispatch): boolean =>
+  readRecords(run).records.some((r) => r.dispatchId === d.admit.dispatchId);
+
+/**
+ * The stand-in's brief (spec §4.5, audit C4). A fresh round reruns its own brief; a fix round hands
+ * over the lane file and the fix brief by path, since the stand-in cannot resume the old thread.
+ */
+function standInBrief(run: Run, d: Dispatch): string {
+  const own = dispatchPaths(d.dir).brief;
+  if (d.admit.thread === null) return readFileSync(own, "utf8");
+  return [
+    `You take over ${d.admit.name} from ${d.admit.rung}, which hit a usage limit. You start on a fresh thread, so read these first:`,
+    ...(d.admit.lane ? [`- the lane file: ${laneFile(run, d.admit.lane)}`] : []),
+    `- the fix brief the previous thread was given: ${own}`,
+    "The work so far is in the tree. Do what the fix brief asks.",
+  ].join("\n");
+}
+
+/**
+ * Spec §4.5: on a limit, launch the rung's stand-in on a fresh thread, through admission again (budget
+ * included), without awaiting it. A stand-in that hits a limit too pauses the run. A stand-in already
+ * launched for this record (by a wait whose collection was put back) is reused, never launched twice.
+ */
+async function failover(deps: Deps, run: Run, d: Dispatch, limited: RunRecord): Promise<Collected> {
+  const hints = hintsFor(run, d, limited);
+  const result = (h: string[]): DispatchResult => ({ record: limited, hints: h });
+  if (d.admit.failoverFrom !== null)
+    return {
+      result: result(hints),
+      started: null,
+      pause: `paused: ${limited.backend} usage limit on ${limited.rung} and on ${d.admit.failoverFrom}`,
+    };
+  const paused = `paused: ${limited.backend} usage limit; resume when the user says so`;
+  const failedOver = (standIn: string, next: Dispatch): Collected => ({
+    result: result([
+      `limit: ${limited.rung} hit a usage limit; failed over to ${standIn}`,
+      // the stand-in's before-snapshot already holds the limited run's writes: surface them here
+      ...hints.filter((h) => /^(violation|git-unavailable):/.test(h)),
+    ]),
+    started: dispatchedOf(next),
+    pause: null,
+  });
+  // tied to this limited dispatch's id: a later dispatch of the same name, rung and limit is not its stand-in
+  const already = listDispatches(run)
+    .filter((x) => x.admit.failoverOf === d.admit.dispatchId)
+    .at(-1);
+  if (already && launched(already)) return failedOver(already.admit.rung, already);
+  if (already && !recordOf(run, already) && dispatchState(already, deps.now()) === "starting") {
+    // Admitted (maybe marked) by a collector that died before launching it: launch it now, never report
+    // it unlaunched. Only the collector holding the limited record launches its stand-in, and we hold that
+    // record now, so that collector is dead. If it died after spawning a supervisor that has not taken the
+    // dispatch's lock yet, the second supervisor started here and that one race for the lock: exactly one
+    // runs the worker, the other exits touching nothing (supervisor.ts), so no worker ever runs twice.
+    start(already, dispatchPaths(already.dir).spec);
+    watch(deps, run, already);
+    return failedOver(already.admit.rung, already);
+  }
+  // else that stand-in never ran and is over (recorded as lost, or past its start grace): admit a new one
+  const standIn = standInFor(deps.profiles.forRepo(run.meta.repo).failover, limited.rung, run.meta.repo);
+  if (!standIn) return { result: result(hints), started: null, pause: paused };
+  if (parseRung(standIn).backend === "claude") {
+    const agent = deps.profiles.agentFor(run.meta.repo, d.admit.role, standIn);
+    return {
+      result: result([
+        ...hints,
+        `failover: run ${d.admit.name} as Agent(subagent_type: "${agent}"), standing in for ${limited.rung}`,
+      ]),
+      started: null,
+      pause: null,
+    };
+  }
+  let next: Awaited<ReturnType<typeof admit>>;
+  try {
+    next = await admit(deps, run, {
+      role: d.admit.role,
+      name: d.admit.name,
+      brief: standInBrief(run, d),
+      rung: standIn,
+      thread: null,
+      lane: d.admit.lane,
+      failoverFrom: limited.rung,
+      failoverOf: d.admit.dispatchId,
+    });
+  } catch (e) {
+    if (!isCatherdError(e)) throw e;
+    return {
+      result: result([...hints, `failover: ${standIn} refused: ${e.code} ${e.message}`]),
+      started: null,
+      pause: paused,
+    };
+  }
+  start(next.d, next.specPath);
+  watch(deps, run, next.d);
+  return failedOver(standIn, next.d);
+}
+
+/** How long an orphaned worker gets between SIGTERM and SIGKILL; tests shorten it. */
+export const orphanLimits = { killGraceMs: KILL_GRACE_MS };
+
+/**
+ * A worker whose supervisor died has no one to read the cancel file: stop its group here (SIGTERM, then
+ * SIGKILL after the grace), each signal guarded by the worker's pid and start time, and write the
+ * exit.json the supervisor would have, with the signal that ended it. A worker catherd cannot identify
+ * (no start time, or a proc.json whose pgid is not the worker's own pid) is never signalled: its pid may
+ * belong to someone else by now, so the dispatch is only recorded as cancelled. A finalizer that sees the
+ * worker gone first reads the cancel file instead (finalize.ts). Nothing happens while the supervisor
+ * lives: it acts on the cancel file itself.
+ */
+async function stopOrphan(deps: Deps, d: Dispatch): Promise<void> {
+  const proc = readProc(d.dir);
+  if (!proc || readExit(d.dir) || isAlive(proc.supervisorPid, proc.supervisorStartTime)) return;
+  const worker = () => isAlive(proc.pid, proc.startTime);
+  // a signal needs the start time read back exactly: a pid whose start time cannot be read now may be reused
+  const surely = () => isSurelyAlive(proc.pid, proc.startTime);
+  if (!worker()) return;
+  let signal: NodeJS.Signals | null = null;
+  if (proc.startTime !== null && (proc.pgid === undefined || proc.pgid === proc.pid)) {
+    if (surely()) {
+      signal = "SIGTERM";
+      killGroup(proc.pid, signal);
+    }
+    const end = Date.now() + orphanLimits.killGraceMs;
+    while (signal && Date.now() < end && worker()) await Bun.sleep(deps.pollMs);
+    if (signal && worker() && surely()) {
+      signal = "SIGKILL";
+      killGroup(proc.pid, signal);
+    }
+  }
+  writeJsonAtomic(dispatchPaths(d.dir).exit, {
+    schema: 1,
+    code: null,
+    signal,
+    reason: "cancelled",
+    endedAt: new Date().toISOString(),
+  });
+}
+
+/**
+ * Spec §4.7: stop a live dispatch (interrupt, SIGTERM, SIGKILL) and record it as cancelled. It collects the
+ * record it returns, so no later `wait` returns it again.
+ */
+export async function cancel(deps: Deps, runId: string, name: string): Promise<DispatchResult> {
+  const run = findRun(runId);
+  assertId("role name", name);
+  const live = liveDispatches(run, deps.now()).find((d) => d.admit.name === name);
+  if (!live)
+    throw new CatherdError("E_RUN_NOT_LIVE", `${name} has no live dispatch`, {
+      fix: "status(run) lists the live ones",
+    });
+  requestCancel(live.dir);
+  await stopOrphan(deps, live);
+  await waitForFinish(live, { pollMs: deps.pollMs, tickMs: Number.POSITIVE_INFINITY, now: deps.now });
+  const record = await finalizeDispatch(run, live);
+  const mine = await tryCollect(live.dir);
+  // cancel returns at once, so its lease ends at once
+  if (mine) endCollect(live.dir);
+  const also = mine ? [] : [`${name}: a wait in flight also returned this record`];
+  const { hints } = await refreshState(run);
+  return { record, hints: [...hintsFor(run, live, record), ...also, ...hints] };
+}

package/src/services/dispatches.ts

@@ -0,0 +1,145 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { z } from "zod";
+import { ACCESS, type RunRecord } from "../domain/record.ts";
+import { ROLES } from "../domain/roles.ts";
+import { dispatchPaths, readExit, supervisorAlive } from "../infra/dispatch-dir.ts";
+import { isAlive } from "../infra/proc.ts";
+import { readVersioned, writeTextAtomic } from "../infra/store.ts";
+import { readRecords, type Run, runPaths } from "./run-store.ts";
+
+/** admit.json: what admission decided. Written last, so a dispatch exists once it does. */
+const AdmitSchema = z.looseObject({
+  schema: z.literal(1),
+  runId: z.string(),
+  dispatchId: z.string(),
+  name: z.string(),
+  role: z.enum(ROLES),
+  lane: z.string().nullable(),
+  owns: z.array(z.string()),
+  rung: z.string(),
+  backend: z.string(),
+  thread: z.string().nullable(),
+  attempt: z.number().int().min(1),
+  failoverFrom: z.string().nullable(),
+  /** the limited dispatch a failover stand-in replaces (plan 9); absent on any other dispatch */
+  failoverOf: z.string().optional(),
+  access: z.enum(ACCESS),
+  isolated: z.boolean(),
+  cliVersion: z.string().nullable(),
+  admittedAt: z.string(),
+  repo: z.string(),
+  /** `git status` fingerprints when admitted, for changedOwned and violations */
+  before: z.record(z.string(), z.string()),
+});
+export type Admit = z.infer<typeof AdmitSchema>;
+
+export interface Dispatch {
+  dir: string;
+  admit: Admit;
+}
+export type DispatchState = "starting" | "running" | "finished";
+
+/** How long an admitted dispatch may go without a launch.json before it counts as never started. */
+export const STARTING_GRACE_MS = 30_000;
+/** The start grace in force; tests shorten it. */
+export const startLimits = { graceMs: STARTING_GRACE_MS };
+
+export const admitPath = (dir: string): string => join(dir, "admit.json");
+export const launchPath = (dir: string): string => join(dir, "launch.json");
+export const roleDir = (run: Run, name: string): string => join(runPaths(run.dir).roles, name);
+
+/** Every admitted dispatch of the run, oldest first; a folder whose admit.json cannot be read is skipped. */
+export function listDispatches(run: Run): Dispatch[] {
+  const roles = runPaths(run.dir).roles;
+  const out: Dispatch[] = [];
+  if (!existsSync(roles)) return out;
+  for (const name of readdirSync(roles, { withFileTypes: true })) {
+    if (!name.isDirectory()) continue;
+    for (const id of readdirSync(join(roles, name.name), { withFileTypes: true })) {
+      if (!id.isDirectory()) continue;
+      const dir = join(roles, name.name, id.name);
+      try {
+        out.push({ dir, admit: readVersioned(admitPath(dir), AdmitSchema, 1) });
+      } catch {
+        // admission never finished writing this folder
+      }
+    }
+  }
+  return out.sort((a, b) => a.admit.dispatchId.localeCompare(b.admit.dispatchId));
+}
+
+interface ProcFile {
+  pid: number;
+  /** the worker's process group; the detached worker leads it */
+  pgid?: number;
+  startTime: string | null;
+  supervisorPid: number;
+  supervisorStartTime: string | null;
+  startedAt: string;
+}
+interface LaunchFile {
+  supervisorPid: number;
+  supervisorStartTime: string | null;
+}
+
+function readJson<T>(file: string): T | null {
+  try {
+    return JSON.parse(readFileSync(file, "utf8")) as T;
+  } catch {
+    return null;
+  }
+}
+
+export const readProc = (dir: string): ProcFile | null => readJson<ProcFile>(dispatchPaths(dir).proc);
+
+/**
+ * Spec §3.3: finished once exit.json exists, or once neither the supervisor nor its worker is alive,
+ * each identified by pid and start time, so a reused pid never keeps a dispatch alive (audit C3).
+ */
+export function dispatchState(d: Dispatch, now = Date.now()): DispatchState {
+  if (readExit(d.dir)) return "finished";
+  // a live supervisor holds its dispatch's lock from its first moment, before launch.json or proc.json
+  if (!readProc(d.dir) && supervisorAlive(d.dir)) return "starting";
+  const proc = readProc(d.dir);
+  if (proc)
+    return isAlive(proc.supervisorPid, proc.supervisorStartTime) || isAlive(proc.pid, proc.startTime)
+      ? "running"
+      : "finished";
+  const launch = readJson<LaunchFile>(launchPath(d.dir));
+  if (launch) return isAlive(launch.supervisorPid, launch.supervisorStartTime) ? "starting" : "finished";
+  return now - Date.parse(d.admit.admittedAt) < startLimits.graceMs ? "starting" : "finished";
+}
+
+export type LiveDispatch = Dispatch & { state: DispatchState };
+
+/** Dispatches that have no record yet, with their state; `live` keeps only the unfinished ones. */
+export function pendingDispatches(
+  run: Run,
+  now = Date.now(),
+  records: RunRecord[] = readRecords(run).records,
+): LiveDispatch[] {
+  const recorded = new Set(records.map((r) => r.dispatchId));
+  return listDispatches(run)
+    .filter((d) => !recorded.has(d.admit.dispatchId))
+    .map((d) => ({ ...d, state: dispatchState(d, now) }));
+}
+
+export const liveDispatches = (run: Run, now = Date.now()): LiveDispatch[] =>
+  pendingDispatches(run, now).filter((d) => d.state !== "finished");
+
+/** Spec §4.1: `roles/<name>/latest` names the newest dispatch of a role. */
+export function setLatest(run: Run, name: string, dispatchId: string): void {
+  writeTextAtomic(join(roleDir(run, name), "latest"), dispatchId);
+}
+
+export function latestDispatch(run: Run, name: string): Dispatch | null {
+  const all = listDispatches(run).filter((d) => d.admit.name === name);
+  let id: string | null = null;
+  try {
+    id = readFileSync(join(roleDir(run, name), "latest"), "utf8").trim();
+  } catch {
+    // no pointer yet
+  }
+  return all.find((d) => d.admit.dispatchId === id) ?? all.at(-1) ?? null;
+}