omp-conductor 0.19.2 → 0.19.3

package/src/status-render.ts

@@ -26,7 +26,7 @@ import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
 import { formatDownDuration, formatOrchestratorDown } from "./orchestrator-down.ts";
 import { planUsageLine } from "./usage.ts";
 import { HEALTH_TIMEOUT_MS, SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
-import type { WorkerPausePhase } from "./worker.ts";
+import type { WorkerPauseView } from "./fleet.ts";
 import { formatRss, rssBytesFromHealthz } from "./host.ts";
 import {
   formatBaseHealth,
@@ -269,7 +269,8 @@ const GROOMING_IN_FLIGHT_REASON = "in-flight";
  *    pass saw that no durable row covers: `routed − rows`, clamped at zero so
  *    stale rows (candidates since removed from the queue) cannot push it
  *    negative. `routed` is the same denominator the tick's grooming trigger
- *    reads (`summary.routed >= groomBelow`), and the fleet's queue-digest
+ *    reads (`summary.routed >= groomBelow`; under `groomBelow: "always"`,
+ *    #988, the duty never clears on volume), and the fleet's queue-digest
  *    arithmetic already treats durable rows as current-queue facts
  *    (`claimable = routed − knownBlocked`), so this is the operator-facing
  *    half of the same subtraction. Parent/epic exclusions and backlog
@@ -472,7 +473,7 @@ export function formatFleetStatus(
   brief: string | undefined = undefined,
   decisions: string | undefined = undefined,
   failureClasses: string | undefined = undefined,
-  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
+  workerPhases: ReadonlyMap<number, WorkerPauseView> = new Map(),
   intake: string | undefined = undefined,
   lastStop: DaemonStop | undefined = undefined,
   siblings: { project: string; live: number }[] = [],
@@ -653,7 +654,7 @@ function formatSiblingLive(siblings: { project: string; live: number }[]): strin
 
 function formatProjectBody(
   s: StatusSnapshot,
-  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
+  workerPhases: ReadonlyMap<number, WorkerPauseView> = new Map(),
   now = Date.now(),
   siblings: { project: string; live: number }[] = [],
 ): string {
@@ -702,6 +703,11 @@ function formatProjectBody(
     s.caps.dailySpendUsd === null
       ? `  spend today        $${s.spendTodayUsd.toFixed(2)} estimated (no daily cap)`
       : `  spend today        $${s.spendTodayUsd.toFixed(2)} estimated / $${s.caps.dailySpendUsd.toFixed(2)}` +
+        // A dollar ceiling over a subscription-billed fleet can never fire, so
+        // showing it bare asserts headroom that is not the constraint (#984).
+        // The cap stays visible because it IS configured — what changes is that
+        // the row stops implying it governs anything.
+        (s.spendTelemetry?.kind === "subscription" ? " (inert — subscription-billed, see below)" : "") +
         (s.reservedSpendUsd === undefined || s.reservedSpendUsd === 0
           ? ""
           : ` (+$${s.reservedSpendUsd.toFixed(2)} reserved by runs in flight)`),
@@ -857,15 +863,28 @@ function formatProjectBody(
   } else {
     lines.push(leases.length === 0 ? "mutation leases  (none)" : "mutation leases");
     for (const r of leases) {
-      const phase = workerPhases.get(r.issue);
+      const pauseView = workerPhases.get(r.issue);
+      const phase = pauseView?.phase;
       // A run in a live review round reads `review-revision N`, distinct from
       // a failure and from an ordinary continuation, with the round number
       // from the durable revision row (#692). The live pause overlay still
       // wins: an operator pause is the newer fact about the same session.
+      // The overlay names its source and age when the daemon recorded them
+      // (#997): `paused (board, 4m)` answers "who and since when" at the
+      // surface where the question actually gets asked.
       const paused = phase === "pausing" || phase === "paused";
       const round = s.reviewRounds?.[r.id];
+      const pauseDetail =
+        paused && (pauseView?.source !== undefined || pauseView?.pausedAtMs !== undefined)
+          ? ` (${[
+              ...(pauseView.source === undefined ? [] : [pauseView.source]),
+              ...(pauseView.pausedAtMs === undefined
+                ? []
+                : [formatDownDuration(Math.max(0, now - pauseView.pausedAtMs))]),
+            ].join(", ")})`
+          : "";
       const state = paused
-        ? phase
+        ? `${phase}${pauseDetail}`
         : round !== undefined
           ? `review-revision ${round.round}`
           : r.state;

package/src/store.ts

@@ -211,6 +211,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
   graphTools: true,
   failureCharges: true,
   continuationCharges: true,
+  terminalEvidence: true,
 };
 
 /** Everything SQLite will accept from us. */
@@ -263,6 +264,7 @@ interface RunRow {
   lastError: string | null;
   settlementFlags: string | null;
   report: string | null;
+  terminalEvidence: string | null;
   failureClass: string | null;
   recoveryAction: string | null;
   recoveredAt: number | null;
@@ -746,6 +748,11 @@ CREATE TABLE IF NOT EXISTS runs (
   endedAt     INTEGER,
   lastError   TEXT,
   settlementFlags TEXT,
+  -- The terminal evidence the settlement sweep recorded for this attempt
+  -- (#986): exit path, first recoverable error line, audit flags. Nullable:
+  -- rows written before the column, or not yet looked at by the sweep, carry
+  -- NULL, which reads as "not recorded" — never "nothing was known".
+  terminalEvidence TEXT,
   report      TEXT,
   failureClass TEXT,
   recoveryAction TEXT,
@@ -1494,6 +1501,7 @@ function toRecord(row: RunRow): RunRecord {
     if (flags !== undefined) record.settlementFlags = flags;
   }
   if (row.report !== null) record.report = row.report;
+  if (row.terminalEvidence !== null) record.terminalEvidence = row.terminalEvidence;
   if (row.provider429Count !== null) record.provider429Count = row.provider429Count;
   if (row.resolvedModel !== null) record.resolvedModel = row.resolvedModel;
   if (row.resolvedProvider !== null) record.resolvedProvider = row.resolvedProvider;
@@ -1992,6 +2000,13 @@ export function openStore(dbPath: string): Store {
   if (!columns.some((column) => column.name === "report")) {
     db.exec("ALTER TABLE runs ADD COLUMN report TEXT");
   }
+  // The terminal evidence the settlement sweep records for every non-clean
+  // row (#986). Rows written before the column were never looked at by the
+  // recorder, so NULL is the honest reading — no backfill, because the
+  // recorder derives evidence from facts only the settling pass holds.
+  if (!columns.some((column) => column.name === "terminalEvidence")) {
+    db.exec("ALTER TABLE runs ADD COLUMN terminalEvidence TEXT");
+  }
   // Every row written before #132 is unclassified, and NULL is the honest
   // reading of that: the budget counters below deliberately still count an
   // unclassified terminal row exactly as this release's predecessor did, so an
@@ -2516,7 +2531,7 @@ export function openStore(dbPath: string): Store {
   // the same predicate against the pre-claim row, so a preserved charge can
   // never disagree with what the counters would have counted.
   const CHARGEABLE_FAILURE_EXCLUSIONS =
-    "'ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity', 'returned-for-revision'";
+    "'ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity', 'returned-for-revision', 'model-empty-stop'";
   const CHARGEABLE_CONTINUATION_EXCLUSIONS =
     "'admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity'";
 
@@ -2545,7 +2560,14 @@ export function openStore(dbPath: string): Store {
             AND (
               (state IN ('killed', 'orphaned', 'blocked')
                 AND (failureClass IS NULL OR failureClass NOT IN (${CHARGEABLE_CONTINUATION_EXCLUSIONS})))
-              OR (state = 'failed' AND failureClass = 'returned-for-revision')
+              -- Two failed-state classes charge a continuation instead of an
+              -- attempt (#795, #1001): a reviewer's decision, and a provider
+              -- that answered with empty turns until the harness gave up.
+              -- Neither is the work failing, and both resume from the head the
+              -- attempt already pushed - but each still consumes a bounded
+              -- budget, so a provider stuck in that state escalates to a human
+              -- rather than looping forever.
+              OR (state = 'failed' AND failureClass IN ('returned-for-revision', 'model-empty-stop'))
             )) AS n`,
   );
   // How many times one issue reached a given class. Recovery uses it to bound a
@@ -2665,6 +2687,15 @@ export function openStore(dbPath: string): Store {
       GROUP BY failureClass
       ORDER BY n DESC, failureClass ASC`,
   );
+  // The surviving `unknown` rows (#986): classified but never explained, each
+  // carrying its recorded terminal evidence for `status` to print. Bounded —
+  // a backlog must not turn one status render into an unbounded scan.
+  const selectUnknownEvidence = db.query<RunRow, [string, number]>(
+    `SELECT * FROM runs
+      WHERE project = ? AND failureClass = 'unknown'
+      ORDER BY startedAt DESC, rowid DESC
+      LIMIT ?`,
+  );
   const selectRecoveredSince = db.query<RunRow, [string, number]>(
     `SELECT * FROM runs
       WHERE project = ? AND recoveredAt IS NOT NULL AND recoveredAt >= ?
@@ -5082,6 +5113,11 @@ export function openStore(dbPath: string): Store {
       return entry;
     },
 
+    /** The surviving `unknown` rows, newest first, with whatever terminal
+     *  evidence the sweep recorded on them (#986). Bounded per call. */
+    unknownRuns(project: string, limit = 20): RunRecord[] {
+      return selectUnknownEvidence.all(project, limit).map(toRecord);
+    },
     verbLedger(
       project: string,
       opts: { runId?: string; issue?: number; limit?: number } = {},

package/src/types.ts

@@ -1007,6 +1007,16 @@ export type ArmProof = (typeof ARM_PROOFS)[number];
  */
 export const DEFAULT_ARM_PROOF: ArmProof = "challenge";
 
+/**
+ * The grooming trigger (#988): a numeric threshold keeps today's behaviour —
+ * the tick grooms when the routable count is below it — and the literal
+ * `"always"` grooms on demand every tick while any ungroomed, unrefused,
+ * unparked candidate exists, judged from candidate state rather than queue
+ * volume. Never spell "always" as a large number: a big queue must not
+ * silence the duty.
+ */
+export type GroomTrigger = number | "always";
+
 /**
  * Everything the dispatcher needs to service one product: where work comes
  * from, where code goes, and what it may spend doing it. Config is per project
@@ -1020,9 +1030,10 @@ export interface ProjectConfig {
   /** The one label that means "a human has signed this off as agent-ready". */
   queueLabel: string;
   /** Routable-candidate count below which the tick prompt tells the orchestrator
-   *  to groom the queue. Optional; defaults to {@link DEFAULT_GROOM_BELOW} in
+   *  to groom the queue, or {@link GroomTrigger} `"always"` to groom on demand
+   *  every tick (#988). Optional; defaults to {@link DEFAULT_GROOM_BELOW} in
    *  orchestrator-tick.ts. */
-  groomBelow?: number;
+  groomBelow?: GroomTrigger;
   /** Labels the dispatcher writes back so the tracker alone shows live state
    *  to a human who never opens the daemon's logs. `backlog` is the
    *  operator's own park gesture (#507) — not dispatcher-written, but read by
@@ -1683,6 +1694,19 @@ export const FAILURE_CLASSES = [
   /** A reviewer closed pushed-green or pushed-pending work without merging it:
    *  a review decision, not a worker failure. */
   "returned-for-revision",
+  /** The session ended without delivering any settlement verdict: no yield, no
+   *  green claim, no error — the row's last words are mid-delivery narration
+   *  (#986). Distinct from `unknown` on purpose: the evidence proves the run
+   *  reached its worker and died between turns, so the gap it names is "the
+   *  model stopped before handing over" rather than "nothing is known". It
+   *  charges the attempt exactly as `unknown` did; only the name is specific. */
+  "no-verdict",
+  /** The provider answered with empty assistant turns until the harness's own
+   *  retry cap gave up and ended the session (#986): the terminal line reads
+   *  `empty-stop-retry-cap`. Provider flake, not worker logic — both of the
+   *  2026-08-23 failures were this, at 100/180 and 54/180 turns, while a third
+   *  run on the same model finished green in the same window. */
+  "model-empty-stop",
   "unknown",
 ] as const;
 
@@ -2084,6 +2108,15 @@ export interface RunRecord {
   endedAt?: number;
   /** Last failure text, surfaced verbatim in escalations. */
   lastError?: string;
+  /**
+   * The terminal evidence the settlement sweep recorded for this attempt
+   * (#986): the exit path the row took, the first line of whatever error was
+   * recoverable from the row or its transcript, and any settlement-audit
+   * flags — one bounded line, readable in `status` beside a surviving
+   * `unknown`. Absent means the row predates the column, or the sweep has not
+   * looked at it yet; it never means "nothing was known".
+   */
+  terminalEvidence?: string;
   /** Advisory settlement-audit findings for this attempt (#128). Deliberately
    *  never consulted by anything that decides `state`: a flagged run settles
    *  exactly as an unflagged one does, and the flags are evidence for whoever
@@ -3285,6 +3318,10 @@ export interface Store {
    * every actionable class behind a monotonically growing tally.
    */
   recordedOnlyClassCounts(project: string): { cls: FailureClass; n: number }[];
+  /** The surviving `unknown` rows, newest first, each with whatever terminal
+   *  evidence the sweep recorded on it (#986). Bounded per call — a status
+   *  render must never walk a whole backlog. */
+  unknownRuns(project: string, limit?: number): RunRecord[];
   /** Rows whose recovery ran at or after `since`, newest first — the tick's
    *  "auto-recovered since last tick" line, and its count. */
   recoveredSince(project: string, since: number): RunRecord[];

package/src/usage.ts

@@ -688,6 +688,31 @@ export async function readPlanUsage(
  * to fit on one row. Derived from the same status object as the long form so
  * the two cannot disagree about whether the guard is active.
  */
+/**
+ * The allowance window nearest its ceiling, named for a reader (#984).
+ *
+ * A subscription has an allowance where an API key has a price, so this is what
+ * "the constraint that binds" means on a subscription-billed fleet — and the
+ * one nearest exhaustion is the one that will stop the fleet first, whatever
+ * the others say. Read from the provider's own reply, never derived: a fleet
+ * whose provider reports no window gets `undefined`, which is the honest answer
+ * and keeps a genuine telemetry fault a fault.
+ *
+ * Only windows that reported a comparable fraction are eligible, for the reason
+ * this module already documents at length: `used` is not comparable across
+ * providers, so a raw count can never stand in for one.
+ */
+export function bindingAllowanceWindow(reading: UsageReading): string | undefined {
+  if (reading.kind !== "ok") return undefined;
+  const comparable = reading.windows.filter((w) => w.usedFraction !== undefined);
+  if (comparable.length === 0) return undefined;
+  let worst = comparable[0]!;
+  for (const window of comparable.slice(1)) {
+    if ((window.usedFraction ?? 0) > (worst.usedFraction ?? 0)) worst = window;
+  }
+  return worst.label === undefined ? worst.id : `${worst.id} (${worst.label})`;
+}
+
 export function planUsageBadge(status: PlanUsageStatus | undefined): string {
   if (status === undefined) return "plan not read";
   let core: string;