omp-conductor 0.18.2 → 0.19.0

package/src/status-render.ts

@@ -18,7 +18,8 @@
  */
 
 import { formatZonedMinute } from "./availability.ts";
-import type { DaemonStop, GroomingRecord } from "./types.ts";
+import { spendTelemetryDetail, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
+import type { DaemonStop, GroomingRecord, InstallSurfaceObservation, RunRecord } from "./types.ts";
 import { settlementFlagSummary } from "./diff-flags.ts";
 import type { CodeGraphHealth } from "./graph-health.ts";
 import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
@@ -37,7 +38,9 @@ import {
   pauseProvenance,
   type StatusSnapshot,
 } from "./daemon.ts";
+import { pidAlive } from "./escalate.ts";
 import { formatQuarantinedRuns, formatSalvagedRuns } from "./settlement.ts";
+import { formatVerbLedger } from "./verbs/ledger.ts";
 
 // layered status
 // ---------------------------------------------------------------------------
@@ -52,6 +55,16 @@ export type TicksLayer =
 export type PaneLayer = "live" | "missing" | "unknown";
 /** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
 export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
+/**
+ * A token count a human reads at a glance: `102.1k`, not `102105`.
+ *
+ * Thousands only — a run's output tokens are the one count here, and they land
+ * between thousands and low millions, where a single unit reads unambiguously.
+ */
+function formatCount(n: number): string {
+  return n < 1_000 ? String(n) : `${(n / 1_000).toFixed(1)}k`;
+}
+
 export type HerdrLayer = "active" | "inactive" | "unknown";
 export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
 
@@ -139,6 +152,14 @@ export type FleetDaemonProbe = {
 const UNRESPONSIVE_HEALTHZ =
   `unresponsive (healthz timed out after ${HEALTH_TIMEOUT_MS / 1000}s)`;
 
+/** One line of a possibly-multi-line record field, bounded for a status row:
+ *  the evidence and disposition an adjudicator writes are prose, and status is
+ *  a scannable board — the full text lives in the durable row and the ledger. */
+function firstLine(text: string, limit = 120): string {
+  const line = text.split("\n").find((candidate) => candidate.trim().length > 0)?.trim() ?? "";
+  return line.length <= limit ? line : `${line.slice(0, limit - 1)}…`;
+}
+
 function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
   if (probe === undefined) return "unprobed";
   switch (probe.project.kind) {
@@ -341,6 +362,106 @@ export function formatGroomingStatus(input: GroomingStatusInput): string | undef
   return ["grooming (to-spec)", ...lines].join("\n");
 }
 
+/**
+ * The one install-surface fault the cheap surfaces are allowed to name (#919):
+ * two different releases live on one host at the same time.
+ *
+ * Deliberately narrower than `doctor`'s finding. An absent surface, a `local:`
+ * herdr link and a pin whose release could not be verified are all real
+ * observations, and all of them are *steady states* someone chose — a warning
+ * repeated every fifteen minutes would train an operator to ignore the row that
+ * matters. A CLI and an omp plugin on different versions is neither steady nor
+ * chosen: it is the 2026-08-22 shape, where the plugin sat on a withdrawn
+ * 0.18.1 beside a 0.18.0 daemon for a day with nothing saying so.
+ *
+ * Pure, so the tick and `status` read the same recorded row and agree by
+ * construction rather than by two copies of a rule.
+ */
+export function installSurfaceMismatch(
+  observation: InstallSurfaceObservation | undefined,
+): string | undefined {
+  if (observation === undefined) return undefined;
+  const { cliVersion, ompVersion } = observation;
+  if (ompVersion === undefined || ompVersion === cliVersion) return undefined;
+  return (
+    `installed surfaces disagree: omp plugin ${ompVersion}, CLI/daemon ${cliVersion}` +
+    `${observation.herdrSource === undefined ? "" : `, herdr ${observation.herdrSource}`}` +
+    ` — run \`omp-conductor upgrade --to ${cliVersion}\` so every surface carries one identity`
+  );
+}
+
+/** The `status` row for the recorded observation (#919): nothing on agreement,
+ *  the mismatch when there is one, and an honest "not observed yet" before the
+ *  first dispatch pass has looked. Never a probe — the whole point of the
+ *  recorded row is that reading it spawns nothing. */
+export function installSurfaceStatusLine(
+  observation: InstallSurfaceObservation | undefined,
+): string | undefined {
+  if (observation === undefined) return "surfaces  not observed yet (the next dispatch pass records them)";
+  const mismatch = installSurfaceMismatch(observation);
+  return mismatch === undefined ? undefined : `surfaces  ${mismatch}`;
+}
+
+/**
+ * The `status` row for the recorded `omp-telegram` versions (#961).
+ *
+ * Same discipline as the row above: rendered from what the dispatch pass
+ * already recorded, so reading it spawns nothing and touches no registry — and
+ * silent on agreement, because a row that always prints is a row nobody reads.
+ *
+ * Deliberately its own line rather than a clause on the `telegram` row. That
+ * row's severity ladder carries one remedy at a time (token, then inbound, then
+ * approval surface, then profile), and a stale install is a different kind of
+ * fact about a different thing: the bot can be perfectly healthy while the
+ * plugin driving it predates the contract the brief mandates. Folding it in
+ * would either bury it under a louder rung or displace one.
+ */
+export function telegramPluginStatusLine(
+  observation: InstallSurfaceObservation | undefined,
+): string | undefined {
+  if (observation === undefined) return undefined;
+  const { telegramInstalled, telegramDaemon, telegramPublished } = observation;
+  // Nothing recorded at all: this pass predates the read, or none answered.
+  // Silent rather than "unverified" — a fleet that does not use Telegram must
+  // not grow a permanent row about a plugin it does not have.
+  if (telegramInstalled === undefined) return undefined;
+  if (telegramPublished !== undefined && telegramPublished !== telegramInstalled) {
+    return (
+      `tg-plugin omp-telegram ${telegramInstalled} installed, ${telegramPublished} published` +
+      `${telegramDaemon === undefined || telegramDaemon === telegramInstalled ? "" : `, daemon ${telegramDaemon}`}` +
+      " — install it and restart its daemon (#882's target ladder ships in the newer one)"
+    );
+  }
+  if (telegramDaemon !== undefined && telegramDaemon !== telegramInstalled) {
+    return (
+      `tg-plugin the running omp-telegram daemon is ${telegramDaemon}, installed is ${telegramInstalled}` +
+      " — restart it so it serves the installed code"
+    );
+  }
+  return undefined;
+}
+
+/**
+ * The `status` row qualifying the spend figure above it (#970).
+ *
+ * Silent when telemetry is healthy, for the same reason the install-surface row
+ * is: a row that always prints is a row nobody reads. Loud when the cap's input
+ * is going absent, because `spend today $0.64 / $35.00` reads as headroom and
+ * the only surface that said otherwise was an on-demand `doctor` run — the exact
+ * gap #919 closed for install surfaces, on a control that stops the fleet
+ * spending money.
+ *
+ * The wording is the shared one, so this row and `doctor`'s finding cannot state
+ * the same fact differently.
+ */
+export function spendTelemetryStatusLine(
+  verdict: SpendTelemetryVerdict | undefined,
+): string | undefined {
+  if (verdict === undefined) return undefined;
+  const detail = spendTelemetryDetail(verdict);
+  return detail === undefined ? undefined : `  spend telemetry    ${detail}`;
+}
+
 export function formatFleetStatus(
   s: StatusSnapshot,
   layers: FleetLayers,
@@ -422,7 +543,19 @@ export function formatFleetStatus(
           const prov = pauseProvenance(s.project);
           if (prov !== undefined) {
             const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
-            return `dispatch  paused (source: ${prov.source}${reason})`;
+            // A fence that declared an owning process is judged against it
+            // (#938): a setup transaction's fence outliving the setup is a
+            // stalled fleet that reads exactly like a deliberate hold. Never
+            // cleared here — status reports, and the operator decides.
+            const owner =
+              prov.owner === undefined
+                ? ""
+                : pidAlive(prov.owner)
+                  ? `, held by a live process (pid ${prov.owner})`
+                  : `, ABANDONED — its process (pid ${prov.owner}) is gone; clear it with \`omp-conductor resume${
+                      s.project === undefined ? "" : ` --project ${s.project}`
+                    }\``;
+            return `dispatch  paused (source: ${prov.source}${reason}${owner})`;
           }
           return isPaused(s.project) && pausedAt(s.project) === undefined
             ? "dispatch  paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)"
@@ -544,6 +677,15 @@ function formatProjectBody(
     ...(s.orchestratorDown === undefined
       ? []
       : formatOrchestratorDown(s.orchestratorDown, now)),
+    // The recorded install-surface observation (#919): silent on agreement,
+    // loud on a proven mismatch, honest before the first pass has looked.
+    ...(installSurfaceStatusLine(s.installSurfaces) === undefined
+      ? []
+      : [installSurfaceStatusLine(s.installSurfaces) as string]),
+    // Same recorded pass, same silence-on-agreement rule (#961).
+    ...(telegramPluginStatusLine(s.installSurfaces) === undefined
+      ? []
+      : [telegramPluginStatusLine(s.installSurfaces) as string]),
     ...formatReportingStatus(s),
     ...formatAvailabilityStatus(s),
     ...formatDigestScheduleStatus(s),
@@ -551,9 +693,26 @@ function formatProjectBody(
     "caps",
     `  workers            ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
     `  issues today       ${s.runsToday}`,
+    // "estimated" is not hedging (#851): this figure is OMP's own local price
+    // for the tokens the session recorded, not the provider's bill. On
+    // 2026-08-21 the local estimate read $25.16 while roughly $16 of provider
+    // credit actually moved, so a row that called it "spend" was asserting
+    // something conductor cannot see. The reservation is shown beside it
+    // because that, not spend-to-date, is what the next admission subtracts.
     s.caps.dailySpendUsd === null
-      ? `  spend today        $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
-      : `  spend today        $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
+      ? `  spend today        $${s.spendTodayUsd.toFixed(2)} estimated (no daily cap)`
+      : `  spend today        $${s.spendTodayUsd.toFixed(2)} estimated / $${s.caps.dailySpendUsd.toFixed(2)}` +
+        (s.reservedSpendUsd === undefined || s.reservedSpendUsd === 0
+          ? ""
+          : ` (+$${s.reservedSpendUsd.toFixed(2)} reserved by runs in flight)`),
+    // What that figure is built on (#970). Silent when telemetry is healthy;
+    // its own row when it is not, because the number above reads as headroom
+    // and the cap subtracts exactly it. Measured 2026-08-22: 9 of 12 working
+    // runs reported $0.00 and the only surface that could have said so was an
+    // on-demand `doctor` run — whose predicate would not have fired either.
+    ...(spendTelemetryStatusLine(s.spendTelemetry) === undefined
+      ? []
+      : [spendTelemetryStatusLine(s.spendTelemetry) as string]),
     // Its own row beside the spend row, never folded into it: they are two
     // independent controls and an operator has to see which one stopped the
     // fleet (#110).
@@ -611,11 +770,93 @@ function formatProjectBody(
     formatDispatchSummary(s.dispatch),
     "",
   ];
+  // The two lifecycle kinds are rendered as two blocks (#898). An operator
+  // looking at "a branch holding a file" has to be able to tell "a worker is
+  // editing this right now" from "durable work nobody is touching, waiting on
+  // review, merge or recovery" — the same distinction #899 made load-bearing in
+  // admission, where only the first kind occupies files. One block for both
+  // read as though every preserved branch had a live editor, which is exactly
+  // the misreading that let a settled artifact look like a busy worker.
+  //
+  // Membership comes from the snapshot's `leasedRunIds` (`Store.leasedRuns`),
+  // never from a state check here: a dispatched review revision is a lease on a
+  // row whose state still says `pushed-green`, so a renderer that classified by
+  // state alone would call a live rewrite "preserved".
+  //
+  // Membership is the UNION of the run's own live state and the recorded ids,
+  // never the ids alone. A live worker is a lease by definition, so the field
+  // exists only to ADD the case a state read cannot see — a dispatched revision
+  // on a row that still says `pushed-green`. Reading it as the whole truth would
+  // let an absent or stale field print a live worker under "worker-free", which
+  // is the exact misreading this block exists to stop, and the union can never
+  // over-report either: `leasedRuns` is itself live rows plus those revisions.
+  //
+  // The two state names are spelled here rather than importing `LIVE_STATES`:
+  // this module is a pure renderer and does not pull in the store (and its
+  // sqlite binding) to read two string literals.
+  const leased = new Set(s.leasedRunIds ?? []);
+  const isLease = (r: RunRecord): boolean =>
+    r.state === "claimed" || r.state === "running" || leased.has(r.id);
+  const leases = s.activeRuns.filter((r) => isLease(r));
+  const preserved = s.activeRuns.filter((r) => !isLease(r));
+  // The per-run annotations both blocks carry: they are facts about the run,
+  // not about which lifecycle kind it is, and a flagged or merge-blocked
+  // artifact is if anything MORE interesting once nobody is editing it.
+  const annotations = (r: RunRecord): string[] => {
+    const out: string[] = [];
+    // The orchestrator's Duty 1 reads this command, and a flagged run's
+    // escalation is deduplicated after one delivery — so this is where a
+    // flagged PR stays visible for as long as it is still open (#128).
+    const flagged = settlementFlagSummary(r.settlementFlags);
+    if (flagged !== undefined) out.push(`      ${flagged}`);
+    // The exact-head merge blocker (#888): a green PR whose head carries
+    // unresolved durable review evidence would otherwise read as merge-ready
+    // while `conductor_pr_merge` refuses it. Same rows the verb reads,
+    // rendered where Duty 1 already looks.
+    // The review-ceiling adjudication for this run's exact head (#874). Matched
+    // on PR + head, never on the run: a verdict describes the diff it read, so a
+    // run that has since pushed a corrected head must not appear to carry the
+    // older head's decision.
+    const adjudication =
+      r.prUrl === undefined || r.headSha === undefined
+        ? undefined
+        : s.reviewAdjudications?.find(
+            (a) => a.prUrl === r.prUrl && a.headSha.toLowerCase() === r.headSha?.toLowerCase(),
+          );
+    if (adjudication !== undefined) {
+      // The role asked for AND the model that actually ran it: those differ
+      // exactly when something is misconfigured, and then the state is
+      // `unavailable-model` and the difference is the whole finding (#875).
+      const launched =
+        adjudication.provenance === undefined
+          ? `role ${adjudication.role} (not launched)`
+          : `role ${adjudication.role} → ${adjudication.provenance.model}${
+              adjudication.provenance.provider === undefined ? "" : ` (${adjudication.provenance.provider})`
+            }`;
+      out.push(
+        `      adjudication ${adjudication.state}  ${launched}` +
+          (adjudication.evidence === undefined ? "" : `  ${firstLine(adjudication.evidence)}`) +
+          (adjudication.disposition === undefined ? "" : `  → ${firstLine(adjudication.disposition)}`),
+      );
+    }
+    const blocked = s.mergeBlockers?.[r.id];
+    if (blocked !== undefined)
+      out.push(
+        `      merge blocked: review round ${blocked.round} ${
+          blocked.state === "pending"
+            ? "queued at this head"
+            : blocked.state === "crashed"
+              ? "dispatched and unfinished at this head"
+              : "failed at this head"
+        } — unresolved findings; push a corrected head, or record a conductor_pr_review_clear for this exact head, before merge`,
+      );
+    return out;
+  };
   if (s.activeRuns.length === 0) {
-    lines.push("active runs  (none)");
+    lines.push("mutation leases  (none)");
   } else {
-    lines.push("active runs");
-    for (const r of s.activeRuns) {
+    lines.push(leases.length === 0 ? "mutation leases  (none)" : "mutation leases");
+    for (const r of leases) {
       const phase = workerPhases.get(r.issue);
       // A run in a live review round reads `review-revision N`, distinct from
       // a failure and from an ordinary continuation, with the round number
@@ -626,7 +867,7 @@ function formatProjectBody(
       const state = paused
         ? phase
         : round !== undefined
-          ? `review-revision ${round}`
+          ? `review-revision ${round.round}`
           : r.state;
       // Turn rate and cap projection (#730/#767): a stalled run and a fast one
       // used to render identically as a bare turn count. The rate is the
@@ -643,24 +884,103 @@ function formatProjectBody(
       // elapsed includes banked pause time and its state already says
       // `paused`.
       const elapsedMs = Math.max(0, now - r.startedAt);
+      // A review revision resumes the SAME run and session, so `turns` and
+      // `startedAt` are cumulative over the whole attempt — and reading them as
+      // this round's is how a revision recorded two minutes ago was read as a
+      // worker stuck for two hours (#802, #797 on 2026-08-19). Nothing is reset:
+      // the attempt totals are the cap evidence and stay exactly as recorded.
+      // They are labelled `cumulative` instead, and the round's own age is shown
+      // beside them from the durable dispatch instant.
+      //
+      // The rate and cap projection are deliberately dropped for a live round:
+      // both divide cumulative turns by cumulative elapsed, so neither describes
+      // the phase the line is naming, and a number attributed to the wrong phase
+      // is worse than no number. The attempt's own cap remains visible as
+      // `N/M turns cumulative`.
       const progress = paused
         ? ""
-        : `  ${formatDownDuration(elapsedMs)} elapsed  ${(
-            (r.turns * 60_000) / Math.max(elapsedMs, 1_000)
-          ).toFixed(1)} turns/min avg` +
-          `  projects ${Math.round(
-            (r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000),
-          )} turns at cap`;
+        : round !== undefined
+          ? ` cumulative  ${formatDownDuration(elapsedMs)} elapsed cumulative  round ${
+              round.round
+            } dispatched ${formatDownDuration(Math.max(0, now - round.dispatchedAt))} ago`
+          : `  ${formatDownDuration(elapsedMs)} elapsed  ${(
+              (r.turns * 60_000) / Math.max(elapsedMs, 1_000)
+            ).toFixed(1)} turns/min avg` +
+            `  projects ${Math.round(
+              (r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000),
+            )} turns at cap`;
       lines.push(
         `  #${r.issue}  ${r.repo}  ${state}  attempt ${r.attempt}  ` +
           `${r.turns}/${r.maxTurns} turns${progress}  ${r.spendUsd.toFixed(2)}  ${r.branch}` +
           (r.prUrl ? `  ${r.prUrl}` : ""),
       );
-      // The orchestrator's Duty 1 reads this command, and a flagged run's
-      // escalation is deduplicated after one delivery — so this is where a
-      // flagged PR stays visible for as long as it is still open (#128).
-      const flagged = settlementFlagSummary(r.settlementFlags);
-      if (flagged !== undefined) lines.push(`      ${flagged}`);
+      // What the lease actually holds, named where the operator is already
+      // looking: the run this is (an issue can have several attempts, and only
+      // one of them holds the lease), how long it has held it, the read the
+      // interlock uses to prove occupancy, and the files it occupies.
+      //
+      // The source is not a guess: `probeRunLane` reads a live checkout when the
+      // row has one and the mirror branch when it does not, which is exactly
+      // this condition — a dispatched revision on a settled row has no worktree,
+      // and its lane comes from the branch.
+      //
+      // The file list is the durable declaration admission persisted at dispatch
+      // (#744) — the half that occupies before anything is written. What the run
+      // has additionally touched is a git read, deliberately not taken here:
+      // status is a recorded answer, never a subprocess per render (#919). A
+      // lease with no declaration says so, because "no files listed" and "no
+      // declaration made" invite different actions.
+      lines.push(
+        `      lease  run ${r.id.slice(0, 8)}  ${formatDownDuration(Math.max(0, now - r.startedAt))} held  ` +
+          `${r.worktree === "" ? "branch" : "worktree"}  ` +
+          (r.lane === undefined || r.lane.files.length === 0
+            ? "no declared lane (probe-only occupancy)"
+            : `occupies ${r.lane.files.join(", ")}`),
+      );
+      // Whether the operator can watch this worker, named rather than absent
+      // (#841). A live run with no pane is a fleet running blind, and silence
+      // reads identical to "there is nothing to see" — so degraded says so and
+      // carries the reason the launch or the last reconcile recorded.
+      lines.push(
+        r.paneId === undefined
+          ? `      pane   degraded${r.paneUnavailable === undefined ? "" : ` — ${r.paneUnavailable}`}`
+          : `      pane   ${r.paneId}${r.paneLabel === undefined ? "" : `  ${r.paneLabel}`}`,
+      );
+      // The fact that explains an expensive-in-time, cheap-in-dollars run
+      // (#518). Shown only when the share is disproportionate, because a run
+      // reasoning normally needs no annotation and a line printed for every run
+      // is a line nobody reads. The threshold is deliberately high: at 80% the
+      // turn cap is already unreachable and the wall clock is the only ceiling
+      // that can fire, which is the operator's actual decision.
+      if (r.outputTokens !== undefined && r.outputTokens > 0 && r.reasoningTokens !== undefined) {
+        const share = r.reasoningTokens / r.outputTokens;
+        if (share >= 0.8) {
+          lines.push(
+            `      tokens ${Math.round(share * 100)}% of ${formatCount(r.outputTokens)} output tokens ` +
+              `were reasoning — deliberating, not stalled; wall clock is the binding cap`,
+          );
+        }
+      }
+      lines.push(...annotations(r));
+    }
+  }
+  // Preserved artifacts: durable work with no worker and no revision behind it.
+  // Rendered without a turn rate or a cap projection on purpose — both describe
+  // a session in progress, and printing them beside work nobody is running is
+  // how a settled artifact came to read as a busy one. What matters here is how
+  // long it has been waiting and on what: a PR to review and merge, or (with no
+  // PR) a branch `conductor_pr_recover` can still publish.
+  if (preserved.length > 0) {
+    lines.push("preserved artifacts  (worker-free — nothing is editing these)");
+    for (const r of preserved) {
+      const since = r.endedAt ?? r.startedAt;
+      lines.push(
+        `  #${r.issue}  ${r.repo}  ${r.state}  attempt ${r.attempt}  ` +
+          `${formatDownDuration(Math.max(0, now - since))} waiting  ${r.turns} turns spent  ` +
+          `${r.spendUsd.toFixed(2)}  ${r.branch}` +
+          (r.prUrl === undefined ? "  no PR — recoverable with conductor_pr_recover" : `  ${r.prUrl}`),
+      );
+      lines.push(...annotations(r));
     }
   }
   lines.push(...formatBaseHealth(s.baseHealth));
@@ -669,6 +989,15 @@ function formatProjectBody(
   lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
   lines.push(...formatOpenReports(s.openReports));
   lines.push(...formatDigestBacklog(s.digestBacklog));
+  // The mediated-verb ledger (#972). Absent from this renderer since the block
+  // was written (#133) — it was wired into `daemon.ts`'s copy, which was
+  // already not the live renderer, so the read below was paid for on every
+  // status call and dropped. It belongs here: the count is how an operator sees
+  // that a mediated act was refused without going to the sqlite file, and
+  // #968's refusal class (41% of this fleet's refusals) went unnoticed for a
+  // fortnight precisely because nothing on this surface said so. Self-silencing
+  // on an empty ledger, so a fresh fleet gains no row.
+  lines.push(...formatVerbLedger(s.verbLedger));
   if (s.liveWorkers > 0) {
     lines.push(
       "",