omp-conductor 0.15.11 → 0.15.13

package/src/model-fallback.ts

@@ -0,0 +1,177 @@
+/**
+ * The model-failover chain (#286): which model the next attempt of one run
+ * chain dispatches on, and how the choice is worded so a merged branch built
+ * on a different model stays attributable months later.
+ *
+ * Pure and synchronous: the daemon reads the run rows from the store, hands
+ * them here, and does what the returned choice says. Nothing in this module
+ * reaches a store or a provider — the same split as `failure-class.ts`, kept
+ * here (and NOT there) because `failure-class.ts` owns the *verdict* while
+ * this owns the *response to a run of provider verdicts*.
+ *
+ * The trigger is deliberately counted from the store, never from the
+ * harness's `modelFallbackMessage`: that message means the harness already
+ * downgraded internally and is evidence about a run, not a decision; keying
+ * on it would fire the feature when nothing needed switching and stay silent
+ * when a provider returns 402 cleanly.
+ */
+
+import type { FailureClass, RunRecord } from "./types.ts";
+
+/**
+ * Failure classes that move a chain onto its next model — and ONLY these: a
+ * cap kill means the slice was too big, and switching models in response
+ * would hide a decomposition problem behind a provider one (#286, #490).
+ */
+export const FAILOVER_CLASSES: readonly FailureClass[] = [
+  "provider-transient",
+  "provider-credit",
+];
+
+/** The default for a project's `modelFallbackThreshold` when it is absent or
+ *  unusable. Two consecutive provider-class failures, then the next attempt
+ *  moves to the next model. */
+export const DEFAULT_MODEL_FALLBACK_THRESHOLD = 2;
+
+/** True for the classes {@link FAILOVER_CLASSES} names, so a non-provider
+ *  failure can never push a chain sideways. */
+export function isProviderFailureClass(cls: string | undefined): cls is FailureClass {
+  return cls !== undefined && (FAILOVER_CLASSES as readonly string[]).includes(cls);
+}
+
+/** Consecutive provider-class failures at the head of one run chain. */
+export interface ProviderFailureFacts {
+  /** How many terminal runs at the head of the chain failed as a provider. */
+  streak: number;
+  /** Those runs' classes, newest first — the set is either one class or a mix
+   *  of `provider-transient` and `provider-credit`. */
+  classes: FailureClass[];
+  /**
+   * The model the most recent failure dispatched on, when its row recorded
+   * one. Undefined when the rows predate the column or the project has no
+   * chain — the log then falls back to the configured primary, or to "harness
+   * default" for a project with no `workerModel`.
+   */
+  previousModel: string | undefined;
+}
+
+/**
+ * Counts the consecutive provider-class failures at the head of a run chain.
+ * `runs` is in store order (oldest first, per `Store.runsForIssue`); the walk
+ * starts at the newest row and stops at the first row that is not a
+ * provider-class failure — a success, a non-provider failure, an unclassified
+ * row, a live row — so "sticky per chain, never global" falls out: a fresh
+ * issue has no rows and starts on the primary, and one issue's bad luck can
+ * never move another issue's chain.
+ */
+export function providerFailureFacts(runs: readonly RunRecord[]): ProviderFailureFacts {
+  const facts: ProviderFailureFacts = { streak: 0, classes: [], previousModel: undefined };
+  for (let i = runs.length - 1; i >= 0; i--) {
+    const run = runs[i]!;
+    if (!isProviderFailureClass(run.failureClass)) break;
+    facts.streak += 1;
+    facts.classes.push(run.failureClass);
+    if (facts.previousModel === undefined && run.model !== undefined) {
+      facts.previousModel = run.model;
+    }
+  }
+  return facts;
+}
+
+/** The model the next attempt should dispatch on, and whether that is a
+ *  failover at all. */
+export interface ModelChoice {
+  /** Model to dispatch on; `undefined` leaves the harness default, exactly as
+   *  an unconfigured project dispatches today. */
+  model: string | undefined;
+  /** True when the choice came from the fallback chain — the failover has
+   *  fired and the tick report must say so. */
+  fallback: boolean;
+}
+
+/**
+ * Where one run chain's next attempt dispatches, given the chain configuration
+ * and the store-derived failure streak.
+ *
+ * With no chain (empty or absent `modelFallbacks`) the primary model is
+ * returned unchanged and `fallback` stays false — today's dispatch byte for
+ * byte. Once the streak reaches the threshold the chain advances one slot per
+ * extra failure, clamped to the last model: the chain is exhausted there, and
+ * the existing escalation path (the provider-transient strike cap) settles it,
+ * naming every model tried.
+ */
+export function resolveDispatchModel(args: {
+  workerModel: string | undefined;
+  modelFallbacks: readonly string[] | undefined;
+  threshold: number;
+  streak: number;
+}): ModelChoice {
+  const { workerModel, modelFallbacks, threshold, streak } = args;
+  if (modelFallbacks === undefined || modelFallbacks.length === 0) {
+    return { model: workerModel, fallback: false };
+  }
+  if (streak < threshold) return { model: workerModel, fallback: false };
+  const index = Math.min(streak - threshold, modelFallbacks.length - 1);
+  return { model: modelFallbacks[index] ?? workerModel, fallback: true };
+}
+
+/**
+ * The clause the tick report annexes to a failover dispatch:
+ * `on <fallback> after N provider-transient failures on <previous>`, the shape
+ * the issue asked for (`#123 attempt 3 on <fallback> after 2
+ * provider-transient failures on <primary>`). `undefined` when no failover
+ * fired, so the ordinary dispatch log line is untouched.
+ */
+export function fallbackClause(
+  choice: ModelChoice,
+  facts: ProviderFailureFacts,
+  workerModel: string | undefined,
+): string | undefined {
+  if (!choice.fallback) return undefined;
+  const classWord = new Set(facts.classes).size === 1 ? facts.classes[0]! : "provider";
+  const countWord = facts.streak === 1 ? "failure" : "failures";
+  const previous = facts.previousModel ?? workerModel ?? "harness default";
+  return `on ${choice.model} after ${facts.streak} ${classWord} ${countWord} on ${previous}`;
+}
+
+/** One recorded model dispatch in a chain: which model, on which attempt. */
+export interface ModelTried {
+  model: string;
+  attempt: number;
+}
+
+/** Every recorded model of a chain, in chain order — what the exhaustion
+ *  escalation names when it says "every model tried". Rows without a recorded
+ *  model (no chain configured, or written before the column existed) are
+ *  absent: an unnamed primary is the harness default, which the escalation
+ *  wording covers without pretending to know its name. */
+export function modelsTried(runs: readonly RunRecord[]): ModelTried[] {
+  const tried: ModelTried[] = [];
+  for (const run of runs) {
+    if (run.model === undefined) continue;
+    tried.push({ model: run.model, attempt: run.attempt });
+  }
+  return tried;
+}
+
+/**
+ * `"acme/a (attempts 1-2), acme/b (attempt 3)"` — consecutive attempts
+ * grouped under their model, in chain order.
+ */
+export function formatModelsTried(tried: readonly ModelTried[]): string {
+  const groups: { model: string; attempts: number[] }[] = [];
+  for (const { model, attempt } of tried) {
+    const last = groups[groups.length - 1];
+    if (last !== undefined && last.model === model) last.attempts.push(attempt);
+    else groups.push({ model, attempts: [attempt] });
+  }
+  return groups
+    .map(({ model, attempts }) => {
+      const span =
+        attempts.length === 1
+          ? `attempt ${attempts[0]}`
+          : `attempts ${attempts[0]}-${attempts[attempts.length - 1]}`;
+      return `${model} (${span})`;
+    })
+    .join(", ");
+}

package/src/omp.ts

@@ -19,14 +19,15 @@ import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 
 import { readOnlySession, worktreeConfinement } from "./confinement.ts";
-import { releasePolicyTripwire, type ReleaseBlockContext } from "./release-policy.ts";
+import { releasePolicyTripwire, type GateShape, type ReleaseBlockContext } from "./release-policy.ts";
 import type {
   HostToParent,
   ParentToHost,
   SessionHostSpec,
 } from "./session-host.ts";
 import { decodeFrames, encodeFrame } from "./session-host.ts";
-import type { ReleaseShape, ResolvedGrants, SessionRole } from "./types.ts";
+import type { ResolvedGrants, SessionRole } from "./types.ts";
+import { SESSION_ROLE_ENV } from "./types.ts";
 import { conductorVerbs } from "./verbs/client.ts";
 
 const OMP_PACKAGE = "@oh-my-pi/pi-coding-agent";
@@ -43,6 +44,14 @@ export interface AgentSessionLike {
    * `"*"` for every event. The payload is the harness's own event union, which
    * this package cannot name without the peer dependency, so it arrives as
    * `unknown` and each caller narrows the two or three fields it reads.
+   *
+   * One event is not the harness's at all: `"session_exit"` fires when the
+   * underlying session *process* has terminated, carrying
+   * `{ type: "session_exit", code?: number | null }`. It is the single real
+   * terminal signal — a crashed or killed session is never confused with one
+   * that merely stopped streaming (`"agent_end"`). A supervisor may receive it
+   * after its own `dispose()`, and must decide what an exit means from its own
+   * state rather than assuming every exit is a crash.
    */
   on(event: string, cb: (e: unknown) => void): void;
   abort(): void;
@@ -158,7 +167,7 @@ export async function createLocalSession(opts: {
   /** Install the release/deploy tool-call gate with these per-shape grants. */
   releaseGrants?: ResolvedGrants;
   /** Durable audit callback invoked only when that gate rejects a call. */
-  onReleaseBlocked?: (shape: ReleaseShape, context: ReleaseBlockContext) => void;
+  onReleaseBlocked?: (shape: GateShape, context: ReleaseBlockContext) => void;
   /**
    * The conductor verb socket this session's mutation tools call (#126).
    *
@@ -414,7 +423,7 @@ export interface CreateSessionOptions {
   resume?: boolean;
   role: SessionRole;
   releaseGrants?: ResolvedGrants;
-  onReleaseBlocked?: (shape: ReleaseShape, context: ReleaseBlockContext) => void;
+  onReleaseBlocked?: (shape: GateShape, context: ReleaseBlockContext) => void;
 
   /**
    * Where the control socket is bound. The daemon puts it beside the run's own
@@ -559,6 +568,11 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
   const argv = [process.execPath, opts.hostModule ?? SESSION_HOST, JSON.stringify(spec)];
   const child = Bun.spawn(argv, {
     cwd: opts.cwd,
+    // Stamp the session's own role on the child, so a process the agent runs —
+    // `omp-conductor report` from its sandbox — can tell a worker session from
+    // the operator's shell. Direct CLI runs outside a spawned session inherit
+    // nothing and stay the orchestrator surface.
+    env: opts.role === undefined ? undefined : { ...process.env, [SESSION_ROLE_ENV]: opts.role },
     stdin: "ignore",
     // The harness writes progress to stdout; both streams are the daemon's log,
     // never the protocol. The protocol has its own socket precisely so a chatty
@@ -617,6 +631,67 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
     return stderrTail.trim();
   };
 
+  /**
+   * The child's own start failure, if it got one out over the socket before it
+   * died — the socket counterpart of the pipe drain.
+   *
+   * A child that cannot start can connect and write `start-error` in the same
+   * event-loop turn its exit is dispatched in: the connect already completed
+   * in the kernel, the accept is still queued, and closing the listener in
+   * that window drops the frame with the reason. So every exit route waits
+   * this out BEFORE {@link cleanup} closes the server, then prefers the frame
+   * over a bare exit code. Bounded like the pipes: a dead child that never
+   * connected has nothing on the wire, and 2s only ever delays an
+   * already-failed startup.
+   *
+   * Memoized: the pre-connect catch and the exit handler both arrive here, and
+   * both must read the SAME socket — two independent readers would race on the
+   * first `destroy()` and one of them would lose the frame.
+   */
+  let startErrorReading: Promise<string | undefined> | undefined;
+  const startErrorFromSocket = (): Promise<string | undefined> => {
+    startErrorReading ??= (async (): Promise<string | undefined> => {
+      const socket = await Promise.race([
+        attached.catch(() => undefined),
+        new Promise<undefined>((resolve) => {
+          setTimeout(() => resolve(undefined), DRAIN_GRACE_MS).unref?.();
+        }),
+      ]);
+      if (socket === undefined) return undefined;
+      try {
+        return await new Promise<string | undefined>((resolve) => {
+          let buffer = "";
+          let settled = false;
+          const finish = (message: string | undefined): void => {
+            if (settled) return;
+            settled = true;
+            clearTimeout(timer);
+            resolve(message);
+          };
+          const timer = setTimeout(() => finish(undefined), DRAIN_GRACE_MS);
+          timer.unref?.();
+          socket.on("data", (chunk: Buffer) => {
+            buffer += chunk.toString("utf8");
+            const { frames, rest } = decodeFrames(buffer);
+            buffer = rest;
+            for (const frame of frames) {
+              const message = frame as { t?: unknown; message?: unknown };
+              if (message.t === "start-error" && typeof message.message === "string") {
+                finish(message.message);
+                return;
+              }
+            }
+          });
+          socket.once("error", () => finish(undefined));
+          socket.once("close", () => finish(undefined));
+        });
+      } finally {
+        socket.destroy();
+      }
+    })();
+    return startErrorReading;
+  };
+
   const cleanup = (): void => {
     server.close();
     rmSync(socketPath, { force: true });
@@ -647,17 +722,39 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
     pending.clear();
   };
 
+  // The one terminal event this proxy owns outright: the session process is
+  // gone, whether by crash, kill, or a clean dispose of its own. Delivered to
+  // subscribers *before* any failure is surfaced, so a supervisor learning of
+  // the death can also read its exit code off the same event. `disposing` is
+  // the proxy's own intent (its disposer asked the child to stop), not the
+  // subscriber's, so the event is emitted in both branches and each observer
+  // decides what an exit means from its own state.
+  const emitSessionExit = (code: number | null): void => {
+    const type = "session_exit";
+    const event = { type, code };
+    for (const cb of handlers.get(type) ?? []) cb(event);
+    for (const cb of handlers.get("*") ?? []) cb(event);
+  };
+
   void child.exited.then(async (code) => {
     onExit();
+    const exitCode = typeof code === "number" ? code : null;
+    emitSessionExit(exitCode);
+    // A child that cannot start writes `start-error` over the socket, not the
+    // pipes — and its exit can be dispatched before the accept of a connection
+    // that already completed in the kernel. Let the socket settle before the
+    // listener closes, so the child's own words survive the ordering.
+    if (disposing) {
+      cleanup();
+      return;
+    }
+    const startError = await startErrorFromSocket();
     cleanup();
-    // A child that exits during teardown exited because we asked it to. Only an
-    // exit while the session is supposed to be live is a failure — and it has
-    // to reach whoever is awaiting a prompt, or the dispatcher waits forever
-    // for a turn from a process that is gone.
-    if (disposing) return;
     // The tail first, so the message carries the child's own words.
     await settledTail();
-    fail(`omp-conductor session child exited ${String(code)} before the session ended`);
+    fail(
+      startError ?? `omp-conductor session child exited ${String(code)} before the session ended`,
+    );
   });
 
   let socket: Socket;
@@ -681,15 +778,20 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
     ]);
   } catch (err) {
     child.kill("SIGKILL");
+    // The child's exit can win the race above against the accept of a
+    // connection that completed in the kernel — then its `start-error` frame
+    // is still queued, and closing the listener now would throw it away.
+    // Read the socket first (bounded, like the pipes), and prefer its words
+    // over a bare exit code.
+    const startError = await startErrorFromSocket();
     cleanup();
     // Killed first, so the pipes are already closing, and only then read. This is
     // the route a child that dies before connecting takes — a missing peer
     // dependency, say — and reading `stderrTail` synchronously here raced the
     // drain loops and reported a bare exit code instead of the reason.
     const tail = await settledTail();
-    throw new Error(
-      `${err instanceof Error ? err.message : String(err)}${tail === "" ? "" : `\nchild output:\n${tail}`}`,
-    );
+    const reason = startError ?? (err instanceof Error ? err.message : String(err));
+    throw new Error(`${reason}${tail === "" ? "" : `\nchild output:\n${tail}`}`);
   }
 
   let buffer = "";

package/src/orchestrator-down.ts

@@ -0,0 +1,231 @@
+/**
+ * The orchestrator-down incident: how the daemon learns — and tells its
+ * operator — that the embedded orchestrator session is gone.
+ *
+ * A failed orchestrator used to degrade quietly: the daemon logged a warning
+ * and kept running, but the tier-1 escalations that then fell back to issue
+ * comments were exactly the "nobody reads it until morning" path the
+ * orchestrator exists to avoid. Nothing paged, and `status` never said the
+ * operator channel was degraded (the pending issue is #288).
+ *
+ * This module makes three facts durable and observable:
+ *
+ *  1. **The incident.** Start failure (no handle) and unexpected session death
+ *     (a handle whose `alive()` is false) each open one durable
+ *     `orchestrator-incident` row, re-derived across restarts so a daemon
+ *     restarted while still down rediscovers it. The row survives and is
+ *     removed only on recovery.
+ *  2. **One page per incident.** Open, and every subsequent down tick, page a
+ *     tier-2 urgent escalation whose key is anchored on the incident's `since`
+ *     timestamp — the notification ledger pages once, never per tick, and a
+ *     *new* incident (a flapping orchestrator recovering then dying again) gets
+ *     a new anchor and pages again.
+ *  3. **The degrade row.** While down, `status` names the mode, the since-
+ *     moment and how many tier-1 escalations were diverted to issue comments.
+ *     Recovery closes the row and pages one closing notice on the same anchor,
+ *     with downtime and the diverted count.
+ *
+ * The session liveness driving this is event-driven (the proxy's `session_exit`
+ * terminal event), never a timer: a wedged-but-alive session stays "alive"
+ * here and belongs to the `.conductor-stalled` watchdog, which this module
+ * deliberately does not touch.
+ */
+
+import type { OrchestratorHandle } from "./orchestrator.ts";
+import type {
+  Escalation,
+  OrchestratorDownMode,
+  OrchestratorIncident,
+  ProjectConfig,
+  Store,
+} from "./types.ts";
+
+/** Fleet-scoped pages carry issue `0` — there is no tracker issue for this. */
+const NO_ISSUE = 0;
+
+export interface OrchestratorDownDeps {
+  project: ProjectConfig;
+  store: Store;
+  /** The embedded session handle; `undefined` when it failed to start. */
+  orchestrator?: OrchestratorHandle;
+  /**
+   * The daemon's escalation transport. Errors are caught here (the call is
+   * best-effort and ledger-deduped), so a transport that is itself down never
+   * takes the tick down with it.
+   */
+  escalate(e: Escalation): Promise<void>;
+  /**
+   * Start-failure cause, only meaningful when a new incident is opened with no
+   * handle. Carried so the page and `status` tell the operator *why*, not just
+   * that the orchestrator is gone.
+   */
+  startCause?: string;
+  now?(): number;
+  log?(msg: string): void;
+}
+
+function iso(at: number): string {
+  return new Date(at).toISOString();
+}
+
+/** Compact, human-scale duration for downtime and ages. */
+export function formatDownDuration(ms: number): string {
+  if (ms < 60_000) return `${Math.max(0, Math.round(ms / 1_000))}s`;
+  if (ms < 3_600_000) return `${Math.round(ms / 60_000)}m`;
+  if (ms < 86_400_000) return `${Math.round(ms / 3_600_000)}h`;
+  return `${Math.round(ms / 86_400_000)}d`;
+}
+
+function warn(msg: string): void {
+  process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`);
+}
+
+function errText(e: unknown): string {
+  return e instanceof Error ? e.message : String(e);
+}
+
+/**
+ * Best-effort page, mirroring the daemon's `safeEscalate`: a transport failure
+ * is logged, never thrown. The recipient (the daemon's escalator) owns the
+ * notification-ledger dedupe, which is what makes "page once per incident" a
+ * property of the store rather than of this loop.
+ */
+async function deliver(deps: OrchestratorDownDeps, e: Escalation): Promise<boolean> {
+  try {
+    await deps.escalate(e);
+    return true;
+  } catch (err) {
+    (deps.log ?? warn)(`orchestrator-down escalation could not be delivered: ${errText(err)}`);
+    return false;
+  }
+}
+
+/**
+ * The down page. Keyed on the incident's `since` so a still-open incident never
+ * pages again (the ledger dedupes on the summary), while each *new* incident —
+ * a fresh `since` — pages once.
+ */
+export function downEscalation(project: string, incident: OrchestratorIncident): Escalation {
+  return {
+    tier: 2,
+    category: "confirmed-failure",
+    urgent: true,
+    project,
+    issue: NO_ISSUE,
+    summary:
+      `Orchestrator down since ${iso(incident.since)} — tier-1 escalations diverting to issue comments (${project})`,
+    detail: [
+      `Mode: ${incident.mode}.`,
+      ...(incident.cause === undefined ? [] : [`Cause: ${incident.cause}.`]),
+      "",
+      "Tier-1 escalations still land — as issue comments, the path this orchestrator",
+      "exists to avoid. Dispatch, settlement and release are unaffected; the operator",
+      "channel is degraded. Recovery sends one closing notice on this same incident.",
+    ].join("\n"),
+  };
+}
+
+/** The closing notice, referencing the same incident anchor. */
+export function recoveryEscalation(
+  project: string,
+  closed: OrchestratorIncident,
+  now: number,
+): Escalation {
+  const downtime = formatDownDuration(Math.max(0, now - closed.since));
+  return {
+    tier: 2,
+    category: "confirmed-failure",
+    urgent: true,
+    project,
+    issue: NO_ISSUE,
+    summary:
+      `Orchestrator recovered for ${project} ` +
+      `(down since ${iso(closed.since)}) — tier-1 escalations resume as injected prompts`,
+    detail: [
+      `Mode: ${closed.mode}. Downtime: ${downtime}.`,
+      `Tier-1 escalations diverted to issue comments while down: ${closed.diverted}.`,
+      "",
+      "The orchestrator is back and ticking. The .conductor-stalled wedge watchdog is",
+      "unchanged and still covers a session that stops draining its queue.",
+    ].join("\n"),
+  };
+}
+
+/**
+ * Reconcile the embedded orchestrator's liveness against the durable incident.
+ * Idempotent, so it is safe to call at startup (after the escalator exists) and
+ * on every tick: opening, paging, divert-counting and recovering all converge
+ * on the store.
+ *
+ * External orchestrators return early — their down state is a human-owned pane,
+ * not conductor's to page, and the stall watchdog still covers their wedge.
+ */
+export async function reconcileOrchestratorDown(deps: OrchestratorDownDeps): Promise<void> {
+  const { project, store } = deps;
+  const now = deps.now ?? Date.now;
+  const log = deps.log ?? warn;
+  if (project.escalation.orchestrator === "external") return;
+
+  const handle = deps.orchestrator;
+  const up = handle !== undefined && handle.alive();
+  const existing = store.orchestratorIncident(project.name);
+
+  if (up) {
+    if (existing === undefined) return;
+    // A session is (re)started and alive: close the incident from before and
+    // page one closing notice on the same anchor.
+    const closed = store.closeOrchestratorIncident(project.name, now());
+    if (closed === undefined) return; // raced with another daemon
+    const delivered = await deliver(deps, recoveryEscalation(project.name, closed, now()));
+    log(
+      `orchestrator recovered — closed incident from ${iso(closed.since)} ` +
+        `(${closed.diverted} diverted)${delivered ? "" : " (page unsent)"}`,
+    );
+    return;
+  }
+
+  // Down. Open once (the store's INSERT OR IGNORE is the dedupe), then page —
+  // the page itself is ledger-deduped so later ticks re-derive the world
+  // without paging again.
+  if (existing === undefined) {
+    const mode: OrchestratorDownMode = handle === undefined ? "start-failed" : "crashed";
+    if (
+      !store.openOrchestratorIncident({
+        project: project.name,
+        mode,
+        ...(mode === "start-failed"
+          ? deps.startCause === undefined
+            ? {}
+            : { cause: deps.startCause }
+          : { cause: "session died after a healthy start" }),
+        since: now(),
+      })
+    ) {
+      return; // raced with another daemon opening it
+    }
+  }
+  const incident = store.orchestratorIncident(project.name);
+  if (incident === undefined) return;
+  await deliver(deps, downEscalation(project.name, incident));
+  log(
+    `ERROR: orchestrator ${incident.mode} down since ${iso(incident.since)} — ` +
+      `tier-1 escalations diverting to issue comments`,
+  );
+}
+
+/**
+ * The first-class degrade row for `status`: mode, since-when, and how many
+ * tier-1 escalations have been diverted to issue comments. Absent entirely
+ * (empty array) when the orchestrator is healthy, so recovery drops the row.
+ */
+export function formatOrchestratorDown(row: OrchestratorIncident, now = Date.now()): string[] {
+  return [
+    `orchestrator  DEGRADED  ${row.mode} since ${iso(row.since)} ` +
+      `(${formatDownDuration(Math.max(0, now - row.since))} ago)`,
+    `  diverting  tier-1 escalations to issue comments` +
+      (row.diverted === 0
+        ? " (none yet)"
+        : ` — ${row.diverted} escalated issue${row.diverted === 1 ? "" : "s"} diverted`),
+    ...(row.cause === undefined ? [] : [`  cause      ${row.cause}`]),
+  ];
+}