omp-conductor 0.18.0 → 0.18.2

package/src/status-render.ts

@@ -18,13 +18,13 @@
  */
 
 import { formatZonedMinute } from "./availability.ts";
-import type { DaemonStop } from "./types.ts";
+import type { DaemonStop, GroomingRecord } from "./types.ts";
 import { settlementFlagSummary } from "./diff-flags.ts";
 import type { CodeGraphHealth } from "./graph-health.ts";
 import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
 import { formatDownDuration, formatOrchestratorDown } from "./orchestrator-down.ts";
 import { planUsageLine } from "./usage.ts";
-import { SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
+import { HEALTH_TIMEOUT_MS, SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
 import type { WorkerPausePhase } from "./worker.ts";
 import { formatRss, rssBytesFromHealthz } from "./host.ts";
 import {
@@ -90,6 +90,14 @@ export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()):
     `code graph  ${graph.status}  ${indexed}/${graph.repos.length} repos indexed`,
     `  indexer   ${graph.prerequisites.indexer}`,
     `  MCP mount ${graph.prerequisites.mcpMount}`,
+    // The runtime half of the mount finding (#726): what dispatched sessions
+    // actually held in their registries. Rendered only when a run recorded
+    // it — "observed" is never claimed without evidence.
+    ...(graph.session.recorded === 0
+      ? []
+      : [
+          `  session   observed in ${graph.session.recorded} run(s): present ${graph.session.present} / absent ${graph.session.absent}`,
+        ]),
     `  timer     ${graph.timer.enabled} / ${graph.timer.active}`,
     `  refresh   ${refresh}`,
     ...graph.reasons.map((reason) => `  - ${reason}`),
@@ -108,6 +116,7 @@ export type DaemonProjectHealth =
   | { kind: "stopped" }
   | { kind: "ok" }
   | { kind: "unreachable" }
+  | { kind: "unresponsive" }
   | { kind: "other-project"; serves?: string };
 
 /**
@@ -122,6 +131,14 @@ export type FleetDaemonProbe = {
   unit?: UnitOwnership;
 };
 
+/**
+ * #685: the healthz timeout is a probe outcome, never a death verdict — the
+ * same discipline the indexer probe uses when it reports `probe timed out`
+ * instead of declaring the indexer missing.
+ */
+const UNRESPONSIVE_HEALTHZ =
+  `unresponsive (healthz timed out after ${HEALTH_TIMEOUT_MS / 1000}s)`;
+
 function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
   if (probe === undefined) return "unprobed";
   switch (probe.project.kind) {
@@ -129,6 +146,8 @@ function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
       return "stopped";
     case "ok":
       return "ok";
+    case "unresponsive":
+      return UNRESPONSIVE_HEALTHZ;
     case "unreachable":
       return "unreachable — the process is up but not serving";
     case "other-project":
@@ -187,6 +206,141 @@ function formatLastStop(lastStop: DaemonStop | undefined): string[] {
 }
 
 
+// durable to-spec grooming lifecycle (#809)
+// ---------------------------------------------------------------------------
+
+/**
+ * The durable mechanical-hold reasons admission's reconcile writes as blocked
+ * rows (#735) — the lane/dependency holds that clear by themselves. Mirrors
+ * `BLOCKED_GROOMING_HOLDS` in `store.ts`; the status reads the stored reason
+ * strings so both sides of the contract stay on the store vocabulary.
+ */
+const MECHANICAL_GROOMING_REASONS: Record<string, true> = {
+  "file-lane": true,
+  "depends-on": true,
+};
+
+/**
+ * The to-spec refusal classes persisted as blocked rows (#772) — a result
+ * that failed validation is a mechanical block, never a verdict. Mirrors the
+ * failure kinds of `ToSpecFailure` in `to-spec.ts`.
+ */
+const REFUSED_GROOMING_REASONS: Record<string, true> = {
+  malformed: true,
+  "missing-source": true,
+  "stale-source": true,
+};
+
+/** The durable in-flight launch marker (#777) — a batch is running right now.
+ *  Mirrors `TO_SPEC_IN_FLIGHT_REASON` in `orchestrator-tick.ts`. */
+const GROOMING_IN_FLIGHT_REASON = "in-flight";
+
+/**
+ * One project's durable grooming state as status lines, or nothing when there
+ * is nothing to report (#809).
+ *
+ * Renders the state already owned by the `grooming_verdicts` table plus the
+ * dispatch snapshot's parked count — each category below is derived from the
+ * project-scoped durable rows the caller passes, never from parsing evidence
+ * strings and never from a second cache:
+ *
+ *  - `awaiting to-spec` — the routable queue candidates the last dispatch
+ *    pass saw that no durable row covers: `routed − rows`, clamped at zero so
+ *    stale rows (candidates since removed from the queue) cannot push it
+ *    negative. `routed` is the same denominator the tick's grooming trigger
+ *    reads (`summary.routed >= groomBelow`), and the fleet's queue-digest
+ *    arithmetic already treats durable rows as current-queue facts
+ *    (`claimable = routed − knownBlocked`), so this is the operator-facing
+ *    half of the same subtraction. Parent/epic exclusions and backlog
+ *    candidates that never entered the queue are tracker-side and not
+ *    store-knowable; the row lines below carry the durable results either
+ *    way.
+ *  - `promotable` / `considered` / `blocked` — completed to-spec results
+ *    (`blocked` rows whose reason is a groomer verdict or a product-judgement
+ *    label, e.g. `needs-product-decision`).
+ *  - `mechanically blocked` — admission's lane/dependency holds.
+ *  - `refused` — to-spec results that failed validation (`malformed`,
+ *    `missing-source`, `stale-source`), told apart from the holds so an
+ *    operator sees whether the runway cannot move or a result cannot be
+ *    trusted.
+ *  - `in-flight` — a launched batch is running (#777).
+ *  - `operator-parked` — the dispatch snapshot's parked count (#507).
+ *
+ * Rendering is read-only: nothing here reconciles, promotes, retries, or
+ * mutates grooming records. Empty categories are omitted — zero-value rows
+ * would be noise, and an absent block means a fleet with no grooming state.
+ */
+export interface GroomingStatusInput {
+  /** Project-scoped durable rows (`Store.groomingVerdicts`), issue-ascending. */
+  records: readonly GroomingRecord[];
+  /** Routable queue candidates from the last dispatch pass
+   *  (`DispatchSummary.routed`). */
+  routed: number;
+  /** Operator-parked candidates from the last dispatch pass
+   *  (`DispatchSummary.parked`). */
+  parked: number;
+}
+
+/** How many issue identifiers one category line may carry — enough to act on,
+ *  never a queue dump. Mirrors the dispatch hold sampling. */
+const GROOMING_STATUS_SAMPLE = 5;
+
+/** One category line: label, count, and a bounded sample of issue identifiers
+ *  with their durable reasons where the reason differs from the line label
+ *  (e.g. `#13 already-done` under `considered`, `#19 malformed` under
+ *  `refused`). Absent at zero. */
+function groomingStatusLine(
+  label: string,
+  rows: readonly GroomingRecord[],
+): string | undefined {
+  if (rows.length === 0) return undefined;
+  const sample = rows
+    .slice(0, GROOMING_STATUS_SAMPLE)
+    .map((row) => (row.reason === label ? `#${row.issue}` : `#${row.issue} ${row.reason}`));
+  const more = rows.length > GROOMING_STATUS_SAMPLE ? ", …" : "";
+  return `  ${label.padEnd(21)}${rows.length}  ${sample.join(", ")}${more}`;
+}
+
+export function formatGroomingStatus(input: GroomingStatusInput): string | undefined {
+  const { records, routed, parked } = input;
+  const promotable = records.filter((row) => row.verdict === "promotable");
+  const considered = records.filter((row) => row.verdict === "considered");
+  const blockedResults = records.filter(
+    (row) =>
+      row.verdict === "blocked" &&
+      MECHANICAL_GROOMING_REASONS[row.reason] !== true &&
+      REFUSED_GROOMING_REASONS[row.reason] !== true &&
+      row.reason !== GROOMING_IN_FLIGHT_REASON,
+  );
+  const mechanical = records.filter(
+    (row) => row.verdict === "blocked" && MECHANICAL_GROOMING_REASONS[row.reason] === true,
+  );
+  const refused = records.filter(
+    (row) => row.verdict === "blocked" && REFUSED_GROOMING_REASONS[row.reason] === true,
+  );
+  const inFlight = records.filter(
+    (row) => row.verdict === "blocked" && row.reason === GROOMING_IN_FLIGHT_REASON,
+  );
+  const awaiting = Math.max(0, routed - records.length);
+  const lines: string[] = [];
+  if (awaiting > 0) {
+    lines.push(`  ${"awaiting to-spec".padEnd(21)}${awaiting} (of ${routed} routable)`);
+  }
+  for (const line of [
+    groomingStatusLine("promotable", promotable),
+    groomingStatusLine("considered", considered),
+    groomingStatusLine("blocked", blockedResults),
+    groomingStatusLine("mechanically blocked", mechanical),
+    groomingStatusLine("refused", refused),
+    groomingStatusLine("in-flight", inFlight),
+  ]) {
+    if (line !== undefined) lines.push(line);
+  }
+  if (parked > 0) lines.push(`  ${"operator-parked".padEnd(21)}${parked}`);
+  if (lines.length === 0) return undefined;
+  return ["grooming (to-spec)", ...lines].join("\n");
+}
+
 export function formatFleetStatus(
   s: StatusSnapshot,
   layers: FleetLayers,
@@ -201,6 +355,7 @@ export function formatFleetStatus(
   intake: string | undefined = undefined,
   lastStop: DaemonStop | undefined = undefined,
   siblings: { project: string; live: number }[] = [],
+  grooming: string | undefined = undefined,
 ): string {
   const tickLine =
     layers.ticksDetail === undefined
@@ -239,8 +394,13 @@ export function formatFleetStatus(
     // not this project's daemon facts (#379).
     const rss =
       daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
+    // Three states, never two (#685): `not running` is reserved for a pid
+    // that is actually gone, and a timed-out `/healthz` is its own state —
+    // a healthy fleet that answers slowly must not read as stopped.
+    const state =
+      daemon?.project.kind === "unresponsive" ? UNRESPONSIVE_HEALTHZ : "running";
     daemonBlock = [
-      "daemon",
+      `daemon    ${state}`,
       `  pid       ${layers.daemon.pid}`,
       `  port      ${layers.daemon.port ?? "?"}`,
       ...(rss === undefined ? [] : [`  rss       ${formatRss(rss)}`]),
@@ -282,6 +442,7 @@ export function formatFleetStatus(
     ...(decisions === undefined ? [] : [decisions]),
     ...(failureClasses === undefined ? [] : [failureClasses]),
     ...(intake === undefined ? [] : [intake]),
+    ...(grooming === undefined ? [] : [grooming]),
     ...(graphBlock === undefined ? [] : [graphBlock]),
     daemonBlock,
     ...formatLastStop(lastStop),
@@ -319,6 +480,31 @@ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
   ];
 }
 
+/** The reporting row: which categories interrupt the operator's phone, and
+ *  where everything else goes. It reads the snapshot's effective summary —
+ *  the legacy preset name is a label, never the source of truth: an explicit
+ *  policy without `scopePreset` renders truthfully from `interruptOn` and the
+ *  digest cadence, so a routine outcome under `decisions` explains itself as
+ *  digest-only instead of reading like a broken Telegram (#633). */
+function formatReportingStatus(s: StatusSnapshot): string[] {
+  const r = s.reporting;
+  if (r === undefined) return [];
+  const interrupt =
+    r.interruptOn.length === 0 ? "nothing interrupts" : `${r.interruptOn.join(", ")} interrupt`;
+  const label = r.scopePreset === undefined ? "reporting —" : `reporting ${r.scopePreset} —`;
+  let consequence: string;
+  if (r.interruptOn.includes("material")) {
+    consequence = "no outcome waits for the digest";
+  } else if (r.digest.cadence === "daily") {
+    consequence = "material outcomes wait for the daily digest";
+  } else if (r.digest.cadence === "none") {
+    consequence = "material outcomes are held; no digest is configured";
+  } else {
+    consequence = "material outcomes wait for the per-tick digest";
+  }
+  return [`${label} ${interrupt}; ${consequence}`];
+}
+
 /**
  * The shared-daemon visibility row (#545). When the daemon also serves sibling
  * projects, tell the reader how many runs each has live, so they can tell "my
@@ -342,6 +528,14 @@ function formatProjectBody(
   const lines = [
     `project   ${s.project}${s.paused ? "  (PAUSED)" : ""}`,
     ...(s.pauseReason === undefined ? [] : [`paused    ${s.pauseReason}`]),
+    // Structured-exposure half of the drain surface (#484): a plain row for
+    // the renderer, so the active drain is visible in `status` before the
+    // human-wording work of the later #484 child lands.
+    ...(s.drain === undefined
+      ? []
+      : [
+          `drain     active — expires ${new Date(s.drain.expiresAt).toISOString()}, ${s.drain.remainingRuns} run(s) remaining`,
+        ]),
     `config    ${s.configPath}`,
     `state     ${s.stateDir}`,
     ...(siblingLine === undefined ? [] : [siblingLine]),
@@ -350,6 +544,7 @@ function formatProjectBody(
     ...(s.orchestratorDown === undefined
       ? []
       : formatOrchestratorDown(s.orchestratorDown, now)),
+    ...formatReportingStatus(s),
     ...formatAvailabilityStatus(s),
     ...formatDigestScheduleStatus(s),
     "",
@@ -433,20 +628,29 @@ function formatProjectBody(
         : round !== undefined
           ? `review-revision ${round}`
           : r.state;
-      // Turn rate (#730): a stalled run and a fast one used to render
-      // identically as a bare turn count. The rate is the lifetime average
-      // over the elapsed shown — the snapshot carries no checkpoint from
-      // which a recent-interval rate could be derived — so it is labelled
-      // `avg`, and `elapsed` is elapsed since claim, never "remaining wall
-      // clock": a paused worker banks its budget, which the snapshot cannot
-      // know. A paused run gets neither number, because its elapsed includes
-      // banked pause time and its state already says `paused`.
+      // Turn rate and cap projection (#730/#767): a stalled run and a fast one
+      // used to render identically as a bare turn count. The rate is the
+      // lifetime average over the elapsed shown — the snapshot carries no
+      // checkpoint from which a recent-interval rate could be derived — so it
+      // is labelled `avg`, and `elapsed` is elapsed since claim, never
+      // "remaining wall clock": a paused worker banks its budget, which the
+      // snapshot cannot know. #767 adds the projection #518 asked for: the
+      // same claim-elapsed rate held forward to the wall-clock budget
+      // (`caps.workerWallClockMs`, rendered in the caps block) as the turn
+      // count the run would reach at that cap — trajectory math over
+      // elapsed-so-far, so reading it against the run's `maxTurns` tells
+      // which cap fires first. A paused run gets neither number, because its
+      // elapsed includes banked pause time and its state already says
+      // `paused`.
       const elapsedMs = Math.max(0, now - r.startedAt);
       const progress = paused
         ? ""
         : `  ${formatDownDuration(elapsedMs)} elapsed  ${(
             (r.turns * 60_000) / Math.max(elapsedMs, 1_000)
-          ).toFixed(1)} turns/min avg`;
+          ).toFixed(1)} turns/min avg` +
+          `  projects ${Math.round(
+            (r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000),
+          )} turns at cap`;
       lines.push(
         `  #${r.issue}  ${r.repo}  ${state}  attempt ${r.attempt}  ` +
           `${r.turns}/${r.maxTurns} turns${progress}  ${r.spendUsd.toFixed(2)}  ${r.branch}` +