omp-conductor 0.15.13 → 0.16.0

package/src/status-render.ts

@@ -0,0 +1,445 @@
+/**
+ * The status-rendering half of the fleet operator surface (`fleet.ts`
+ * composes; this owns the rendered text).
+ *
+ * Extracted from `fleet.ts` so the region every issue that surfaces something
+ * an operator reads must touch is a lane of its own: #459, #582, #318, #283
+ * and #484 all render a new line here, and a diff touching only this module
+ * must not have to touch the composition root. Behavior is deliberately
+ * untouched by the move — this is a move, not a rewrite: the module takes a
+ * data structure and returns text, and reads nothing itself.
+ *
+ * What status rendering is allowed to consult is the point of the module. It
+ * does not import `fleet.ts` back, and it owns the structural input it
+ * consumes (`FleetLayers`, `TelegramHealth`, `FleetDaemonProbe`) rather than
+ * reaching into the module that builds them — the same seam #580 used for
+ * `AdmissionDeps`. Adding a field to `fleetLayers` must not break these
+ * functions.
+ */
+
+import { formatZonedMinute } from "./availability.ts";
+import type { DaemonStop } from "./types.ts";
+import { settlementFlagSummary } from "./diff-flags.ts";
+import type { CodeGraphHealth } from "./graph-health.ts";
+import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
+import { formatOrchestratorDown } from "./orchestrator-down.ts";
+import { planUsageLine } from "./usage.ts";
+import { SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
+import type { WorkerPausePhase } from "./worker.ts";
+import { formatRss, rssBytesFromHealthz } from "./host.ts";
+import {
+  formatBaseHealth,
+  formatDispatchSummary,
+  formatFreezes,
+  formatReleaseGrants,
+  formatSalvagedRuns,
+  isPaused,
+  pausedAt,
+  pauseProvenance,
+  type StatusSnapshot,
+} from "./daemon.ts";
+
+// layered status
+// ---------------------------------------------------------------------------
+
+export type DispatchLayer = "running" | "paused" | "stopped";
+export type TicksLayer =
+  | "armed"
+  | "disarmed"
+  | "no-heartbeat-config"
+  | "invalid-heartbeat-config"
+  | "ungated";
+export type PaneLayer = "live" | "missing" | "unknown";
+/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
+export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
+export type HerdrLayer = "active" | "inactive" | "unknown";
+export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
+
+export interface TelegramHealth {
+  kind: TelegramLayer;
+  detail?: string;
+}
+
+export interface FleetLayers {
+  dispatch: DispatchLayer;
+  ticks: TicksLayer;
+  ticksDetail?: string;
+  nextTickAt?: string;
+  pane: PaneLayer;
+  paneDetail?: string;
+  recovery: RecoveryLayer;
+  recoveryDetail?: string;
+  herdr: HerdrLayer;
+  herdrDetail?: string;
+  armedPath?: string;
+  tickConfigPath?: string;
+  paneHaltPath?: string;
+  paused: boolean;
+  daemon: { running: boolean; pid?: number; port?: number };
+}
+
+export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined {
+  if (!graph.configured) return undefined;
+  const indexed = graph.repos.filter((repo) => repo.index === "present").length;
+  let refresh: string = graph.refresh.result;
+  if (graph.refresh.lastSuccessAt !== undefined) {
+    const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt));
+    refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`;
+  }
+  return [
+    `code graph  ${graph.status}  ${indexed}/${graph.repos.length} repos indexed`,
+    `  indexer   ${graph.prerequisites.indexer}`,
+    `  MCP mount ${graph.prerequisites.mcpMount}`,
+    `  timer     ${graph.timer.enabled} / ${graph.timer.active}`,
+    `  refresh   ${refresh}`,
+    ...graph.reasons.map((reason) => `  - ${reason}`),
+  ].join("\n");
+}
+
+
+/**
+ * Project-scoped reading of a living daemon record + `/healthz` body.
+ *
+ * Board and `status` share this so a host running one daemon for several
+ * projects cannot be called healthy for a project it does not serve (#379).
+ * A third interpretation convention is forbidden.
+ */
+export type DaemonProjectHealth =
+  | { kind: "stopped" }
+  | { kind: "ok" }
+  | { kind: "unreachable" }
+  | { kind: "other-project"; serves?: string };
+
+/**
+ * Facts `formatFleetStatus` needs about the living daemon. Pure input — the
+ * caller probes; the formatter never shells out (#379).
+ */
+export type FleetDaemonProbe = {
+  project: DaemonProjectHealth;
+  /** `/healthz` body when the probe is project-ok (rss / overlays). */
+  body?: string;
+  /** systemd ownership of the living record pid; omit when unprobed. */
+  unit?: UnitOwnership;
+};
+
+function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
+  if (probe === undefined) return "unprobed";
+  switch (probe.project.kind) {
+    case "stopped":
+      return "stopped";
+    case "ok":
+      return "ok";
+    case "unreachable":
+      return "unreachable — the process is up but not serving";
+    case "other-project":
+      return probe.project.serves === undefined
+        ? "other-project"
+        : `other-project — serves ${probe.project.serves}`;
+  }
+}
+
+/**
+ * Unit line for a living record. The bare unit name alone used to imply
+ * systemd ownership that was never checked (#379); every branch states the
+ * probe result, and `unknown` is never rendered as owned.
+ */
+function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string {
+  if (unit === undefined) return `  unit      ${SYSTEMD_UNIT}  unprobed`;
+  switch (unit.kind) {
+    case "active":
+      if (unit.pid === recordPid) {
+        return `  unit      ${SYSTEMD_UNIT}  systemd-owned`;
+      }
+      // Live unit, different MainPID — the record pid is not systemd's.
+      return `  unit      ${SYSTEMD_UNIT}  unmanaged — MainPID ${unit.pid}`;
+    case "failed":
+      return `  unit      ${SYSTEMD_UNIT}  unmanaged — unit failed`;
+    case "inactive":
+      return `  unit      ${SYSTEMD_UNIT}  inactive`;
+    case "unknown":
+      return `  unit      ${SYSTEMD_UNIT}  unknown — ${unit.reason}`;
+  }
+}
+
+/**
+ * The newest daemon stop/restart provenance as status lines (#378).
+ *
+ * Rendered under the daemon block whether the daemon is down or has since
+ * been restarted — the whole point is that the debrief survives the restart.
+ * An unattributed record says so explicitly: the external-signal fallback
+ * records what the receiving daemon knew (that it was unattributed, when,
+ * which projects it served with live counts) rather than guessing a caller.
+ */
+function formatLastStop(lastStop: DaemonStop | undefined): string[] {
+  if (lastStop === undefined) return [];
+  const caller = lastStop.unattributed
+    ? "unattributed — no mediated request (external signal)"
+    : `pid ${lastStop.callerPid ?? "?"}` +
+      (lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
+      (lastStop.role === undefined ? "" : ` (${lastStop.role})`);
+  return [
+    `  last stop  ${new Date(lastStop.at).toISOString()}  ${lastStop.controlPath}`,
+    `  caller     ${caller}`,
+    `  scope      ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
+    `  affects    ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
+    `  reason     ${lastStop.reason}`,
+  ];
+}
+
+
+export function formatFleetStatus(
+  s: StatusSnapshot,
+  layers: FleetLayers,
+  daemon: FleetDaemonProbe | undefined = undefined,
+  telegram: TelegramHealth = { kind: "unprobed" },
+  now = Date.now(),
+  codeGraph: CodeGraphHealth = { configured: false },
+  brief: string | undefined = undefined,
+  decisions: string | undefined = undefined,
+  failureClasses: string | undefined = undefined,
+  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
+  intake: string | undefined = undefined,
+  lastStop: DaemonStop | undefined = undefined,
+  siblings: { project: string; live: number }[] = [],
+): string {
+  const tickLine =
+    layers.ticksDetail === undefined
+      ? `ticks     ${layers.ticks}`
+      : `ticks     ${layers.ticks}  (${layers.ticksDetail})`;
+  let nextTickLine: string | undefined;
+  if (layers.nextTickAt !== undefined) {
+    const delta = Date.parse(layers.nextTickAt) - now;
+    const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
+    nextTickLine =
+      `next tick ${layers.nextTickAt}  ` +
+      `(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
+  }
+  const paneLine =
+    layers.paneDetail === undefined
+      ? `pane      ${layers.pane}`
+      : `pane      ${layers.pane}  (${layers.paneDetail})`;
+  const recoveryLine =
+    layers.recoveryDetail === undefined
+      ? `recovery  ${layers.recovery}`
+      : `recovery  ${layers.recovery}  (${layers.recoveryDetail})`;
+  const herdrLine =
+    layers.herdrDetail === undefined
+      ? `herdr     ${layers.herdr}`
+      : `herdr     ${layers.herdr}  (${layers.herdrDetail})`;
+  const telegramLine =
+    telegram.detail === undefined
+      ? `telegram  ${telegram.kind}`
+      : `telegram  ${telegram.kind}  (${telegram.detail})`;
+
+  let daemonBlock: string;
+  if (!layers.daemon.running || layers.daemon.pid === undefined) {
+    daemonBlock = "daemon    not running";
+  } else {
+    // Only trust rss from a project-ok body — a foreign payload's bytes are
+    // not this project's daemon facts (#379).
+    const rss =
+      daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
+    daemonBlock = [
+      "daemon",
+      `  pid       ${layers.daemon.pid}`,
+      `  port      ${layers.daemon.port ?? "?"}`,
+      ...(rss === undefined ? [] : [`  rss       ${formatRss(rss)}`]),
+      `  healthz   ${formatDaemonHealthz(daemon)}`,
+      formatDaemonUnit(daemon?.unit, layers.daemon.pid),
+    ].join("\n");
+  }
+
+  const graphBlock = formatCodeGraphHealth(codeGraph, now);
+
+  // Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
+  // a halt) answers "who stopped the fleet" without opening a file (#185). An
+  // unparseable sentinel — paused but with no datable line 1 — is itself news:
+  // it means a run admitted before an *unknown* pause cannot prove innocence
+  // (#174), so completion mutations fail closed while release gates remain usable.
+  const dispatchLine =
+    layers.dispatch === "paused"
+      ? (() => {
+          const prov = pauseProvenance(s.project);
+          if (prov !== undefined) {
+            const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
+            return `dispatch  paused (source: ${prov.source}${reason})`;
+          }
+          return isPaused(s.project) && pausedAt(s.project) === undefined
+            ? "dispatch  paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
+            : "dispatch  paused";
+        })()
+      : `dispatch  ${layers.dispatch}`;
+
+  return [
+    dispatchLine,
+    tickLine,
+    ...(nextTickLine === undefined ? [] : [nextTickLine]),
+    paneLine,
+    recoveryLine,
+    herdrLine,
+    telegramLine,
+    ...(brief === undefined ? [] : [brief]),
+    ...(decisions === undefined ? [] : [decisions]),
+    ...(failureClasses === undefined ? [] : [failureClasses]),
+    ...(intake === undefined ? [] : [intake]),
+    ...(graphBlock === undefined ? [] : [graphBlock]),
+    daemonBlock,
+    ...formatLastStop(lastStop),
+    "",
+    formatProjectBody(s, workerPhases, now, siblings),
+  ].join("\n");
+}
+
+
+function formatAvailabilityStatus(s: StatusSnapshot): string[] {
+  const availability = s.availability;
+  if (availability === undefined) return [];
+  if (availability.mode === "always") {
+    return ["availability 24-hour interrupts (no weekly window)"];
+  }
+  const mode =
+    availability.nextTransitionAt === undefined || availability.timezone === undefined
+      ? `${availability.mode}; next transition could not be calculated`
+      : `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`;
+  const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", ");
+  return [`availability ${mode}; quiet-hours bypass ${bypass}`];
+}
+
+
+function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
+  const schedule = s.digestSchedule;
+  if (schedule === undefined) return [];
+  if (schedule.mode === "disabled") return ["next digest disabled"];
+  if (schedule.mode === "per-tick") return ["next digest every tick"];
+  if (schedule.mode === "due") return ["next digest due now"];
+  return [
+    schedule.nextAt === undefined
+      ? "next digest could not be calculated"
+      : `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`,
+  ];
+}
+
+/**
+ * The shared-daemon visibility row (#545). When the daemon also serves sibling
+ * projects, tell the reader how many runs each has live, so they can tell "my
+ * fleet is idle" from "the process I am about to stop is busy" — the count is
+ * the point, so a "shared daemon" line that omits it is the bug re-expressed.
+ */
+function formatSiblingLive(siblings: { project: string; live: number }[]): string | undefined {
+  if (siblings.length === 0) return undefined;
+  const counts = siblings.map((s) => `${s.project} (${s.live} live)`).join(", ");
+  return `shared    daemon also serves ${siblings.length} other project(s): ${counts}`;
+}
+
+
+function formatProjectBody(
+  s: StatusSnapshot,
+  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
+  now = Date.now(),
+  siblings: { project: string; live: number }[] = [],
+): string {
+  const siblingLine = formatSiblingLive(siblings);
+  const lines = [
+    `project   ${s.project}${s.paused ? "  (PAUSED)" : ""}`,
+    ...(s.pauseReason === undefined ? [] : [`paused    ${s.pauseReason}`]),
+    `config    ${s.configPath}`,
+    `state     ${s.stateDir}`,
+    ...(siblingLine === undefined ? [] : [siblingLine]),
+    // The orchestrator-down degrade row: first-class in the body, present only
+    // while the incident is open, so recovery drops it (#288).
+    ...(s.orchestratorDown === undefined
+      ? []
+      : formatOrchestratorDown(s.orchestratorDown, now)),
+    ...formatAvailabilityStatus(s),
+    ...formatDigestScheduleStatus(s),
+    "",
+    "caps",
+    `  workers            ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
+    `  issues today       ${s.runsToday}`,
+    s.caps.dailySpendUsd === null
+      ? `  spend today        $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
+      : `  spend today        $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
+    // Its own row beside the spend row, never folded into it: they are two
+    // independent controls and an operator has to see which one stopped the
+    // fleet (#110).
+    `  plan usage         ${planUsageLine(s.planUsage)}`,
+    // The tracker's API budget, when the renderer could read it. Absent on a
+    // broken `gh`: one missing row, never a broken report (#188). Next to it,
+    // the refusals the tracker actually observed, so a budget that looks
+    // healthy next to every call being refused is still visible (#198).
+    ...(s.github === undefined
+      ? []
+      : [
+          `  github             graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
+            (s.ghRefusals === undefined || s.ghRefusals.count === 0
+              ? ""
+              : ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
+        ]),
+    // The daemon's own observed github traffic today (#198): a separate row
+    // from the polled budget above, so one missing `status` read never hides
+    // the other. `onCall` counts every spawn including ones that end 304, so
+    // `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
+    // difference, and the polled budget row stays the authority (#203).
+    `  github calls       daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
+      ((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
+        ? ""
+        : ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
+    // Label-projection ops the tracker has not applied yet (#201): GitHub is
+    // behind what the store decided, and the operator can see the lag instead
+    // of discovering it as a stale label or a missing one.
+    ...(s.labelOps === undefined
+      ? []
+      : [
+          `  labels             projection ${s.labelOps.pending} pending (oldest ${
+            s.labelOps.oldestAgeMs >= 60_000
+              ? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
+              : `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
+          })`,
+        ]),
+    `  new worker turns   ${s.caps.workerMaxTurns}`,
+    ...s.turnOverrides.map(
+      ({ issue, maxTurns }) => `  turn override      #${issue} → ${maxTurns} (next attempt)`,
+    ),
+    `  worker wall clock  ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
+    `  failed attempts    ${s.caps.maxAttemptsPerIssue}`,
+    `  continuations      ${s.caps.maxContinuationsPerIssue}`,
+    "",
+    ...formatReleaseGrants(s.releaseGrants),
+    "",
+    formatDispatchSummary(s.dispatch),
+    "",
+  ];
+  if (s.activeRuns.length === 0) {
+    lines.push("active runs  (none)");
+  } else {
+    lines.push("active runs");
+    for (const r of s.activeRuns) {
+      const phase = workerPhases.get(r.issue);
+      const state = phase === "pausing" || phase === "paused" ? phase : r.state;
+      lines.push(
+        `  #${r.issue}  ${r.repo}  ${state}  attempt ${r.attempt}  ` +
+          `${r.turns}/${r.maxTurns} turns  ${r.spendUsd.toFixed(2)}  ${r.branch}` +
+          (r.prUrl ? `  ${r.prUrl}` : ""),
+      );
+      // The orchestrator's Duty 1 reads this command, and a flagged run's
+      // escalation is deduplicated after one delivery — so this is where a
+      // flagged PR stays visible for as long as it is still open (#128).
+      const flagged = settlementFlagSummary(r.settlementFlags);
+      if (flagged !== undefined) lines.push(`      ${flagged}`);
+    }
+  }
+  lines.push(...formatBaseHealth(s.baseHealth));
+  lines.push(...formatFreezes(s.freezes));
+  lines.push(...formatSalvagedRuns(s.salvagedRuns));
+  lines.push(...formatOpenReports(s.openReports));
+  lines.push(...formatDigestBacklog(s.digestBacklog));
+  if (s.liveWorkers > 0) {
+    lines.push(
+      "",
+      `deploy    ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
+        `hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
+    );
+  }
+  return lines.join("\n");
+}
+

package/src/stop-provenance.ts

@@ -37,6 +37,59 @@ export interface StopProvenanceSpec {
  * entry for drain-style restarts whose signal may never arrive, and at the
  * lifecycle chokepoint immediately before signalling for every mediated stop).
  */
+/**
+ * One configured project's live workload at a stop/restart refusal window.
+ */
+export interface LiveWorkload {
+  project: string;
+  /** Live (claimed/running) run count for this project. */
+  live: number;
+  /** The issues currently live, so a refusal can name exactly what would be orphaned. */
+  issues: number[];
+}
+
+/**
+ * Every configured project's live workload. The refusal half of #545: the
+ * shared daemon serves all projects, so a stop/restart must account for every
+ * live run — not just the one the request happens to name — and it has to name
+ * the issues, not just a count, so an operator can see what `--force` would
+ * orphan. Read-only over the store; never writes.
+ */
+export function liveWorkload(
+  store: { liveRuns(project: string): readonly { issue: number }[] },
+): LiveWorkload[] {
+  const cfg = loadConfig();
+  return cfg.projects.map((project) => {
+    const runs = store.liveRuns(project.name);
+    return { project: project.name, live: runs.length, issues: runs.map((r) => r.issue) };
+  });
+}
+
+/**
+ * The refusal gate for a stop/restart of the shared daemon (#545).
+ *
+ * Refuses while ANY configured project — the named one and its siblings alike
+ * — has a live run, since the process is global and stopping it orphans every
+ * project's workers. Throws with the live project(s) and issue numbers so the
+ * operator sees the whole blast radius, never a project-local count.
+ *
+ * `override` (an explicit `--force`) is the escape hatch that keeps a wedged
+ * daemon stoppable: the caller skips this gate on `--force` and records the
+ * override in the stop provenance rather than silencing it.
+ */
+export function refuseBusyDaemon(
+  workload: LiveWorkload[],
+  o: { override: boolean; verb: "stop" | "restart" },
+): void {
+  const busy = workload.filter((w) => w.live > 0);
+  if (busy.length === 0 || o.override) return;
+  const live = busy.map((w) => `project ${w.project}: #${w.issues.join(", #")}`).join("; ");
+  throw new Error(
+    `refusing to ${o.verb} the shared daemon while work is live — ${live}. ` +
+      `Re-run with --force to ${o.verb} anyway (the override is recorded in the daemon-stop provenance).`,
+  );
+}
+
 export function buildStopProvenance(spec: StopProvenanceSpec): DaemonStopDraft {
   const cfg = loadConfig();
   const store = openStore(dbPath());