omp-conductor 0.19.2 → 0.19.4

package/src/spend-telemetry.ts

@@ -66,7 +66,34 @@ export type SpendTelemetryVerdict =
   /** A majority of working runs reported nothing. */
   | { kind: "partial"; worked: number; missing: number }
   /** Every working run reported nothing — the same fact, stated louder. */
-  | { kind: "absent"; worked: number };
+  | { kind: "absent"; worked: number }
+  /**
+   * Every working run reported nothing **and this fleet does not bill in
+   * dollars** (#984). Not a fault: a subscription request has no per-request
+   * price, so the harness reports none and `spendUsd` is permanently 0.00 —
+   * accurate rather than missing. The constraint that actually binds is the
+   * provider's allowance window, named here.
+   */
+  | { kind: "subscription"; worked: number; window: string; evidence: "declared" | "observed" };
+
+/**
+ * What is known about how this project's runs are billed (#984).
+ *
+ * Two independent grounds, because on a real fleet the declaration is often
+ * absent while the evidence is unambiguous:
+ *
+ * - `declaredSubscription` — the project requires its providers to bill to a
+ *   subscription credential (`requireOauthProviders`, #852). A statement of
+ *   intent, so it is believed on its own.
+ * - `allowanceWindow` — the provider itself reports an allowance window
+ *   (`omp usage --json`), which is what a subscription has instead of a price.
+ *   Positive evidence only: a fleet whose provider reports no window has
+ *   nothing here, and its zeros stay a fault.
+ */
+export interface SpendBilling {
+  declaredSubscription?: boolean;
+  allowanceWindow?: string;
+}
 
 /**
  * Judge a newest-first sample.
@@ -78,13 +105,29 @@ export type SpendTelemetryVerdict =
 export function judgeSpendTelemetry(
   samples: readonly SpendSample[],
   limit: number,
+  billing: SpendBilling = {},
 ): SpendTelemetryVerdict {
   const working = samples.filter((row) => row.turns > 0).slice(0, limit);
   if (working.length < limit) {
     return { kind: "insufficient", worked: working.length, needed: limit };
   }
   const missing = working.filter((row) => row.spendUsd === 0).length;
-  if (missing === working.length) return { kind: "absent", worked: working.length };
+  if (missing === working.length) {
+    // Total absence is the only verdict a billing class can reclassify, and
+    // that asymmetry is the whole safety property: a metered neighbour proves
+    // dollars ARE being reported, so a zero beside one is real telemetry loss
+    // no matter how the fleet is billed. `partial` therefore always warns.
+    const window = subscriptionWindow(billing);
+    if (window !== undefined) {
+      return {
+        kind: "subscription",
+        worked: working.length,
+        window: window.window,
+        evidence: window.evidence,
+      };
+    }
+    return { kind: "absent", worked: working.length };
+  }
   if (missing / working.length >= SPEND_MISSING_SHARE) {
     return { kind: "partial", worked: working.length, missing };
   }
@@ -98,8 +141,37 @@ export function judgeSpendTelemetry(
 
 /** The operator-facing sentence, or undefined when there is nothing to say.
  *  Shared so `doctor` and `status` cannot word the same fact differently. */
+/**
+ * The allowance window to name, or `undefined` when nothing establishes that
+ * this fleet is subscription-billed.
+ *
+ * A declaration with no readable window still counts — the operator has said
+ * how the fleet bills, and a provider that cannot be read is not evidence
+ * against it — but it says so rather than inventing a window name.
+ */
+function subscriptionWindow(
+  billing: SpendBilling,
+): { window: string; evidence: "declared" | "observed" } | undefined {
+  if (billing.allowanceWindow !== undefined && billing.allowanceWindow.trim() !== "") {
+    return {
+      window: billing.allowanceWindow.trim(),
+      evidence: billing.declaredSubscription === true ? "declared" : "observed",
+    };
+  }
+  if (billing.declaredSubscription === true) {
+    return { window: "unreadable (omp usage --json reported no window)", evidence: "declared" };
+  }
+  return undefined;
+}
+
 export function spendTelemetryDetail(verdict: SpendTelemetryVerdict): string | undefined {
   switch (verdict.kind) {
+    case "subscription":
+      return (
+        `the last ${verdict.worked} working runs recorded $0.00, which is correct: this fleet bills to a ` +
+        `subscription (${verdict.evidence}), so requests carry no per-request price and no USD cap can fire` +
+        ` — the constraint that binds is the ${verdict.window} allowance`
+      );
     case "absent":
       return (
         `the last ${verdict.worked} completed runs that did any work all recorded $0.00 spend` +

package/src/status-render.ts

@@ -26,7 +26,7 @@ import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
 import { formatDownDuration, formatOrchestratorDown } from "./orchestrator-down.ts";
 import { planUsageLine } from "./usage.ts";
 import { HEALTH_TIMEOUT_MS, SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
-import type { WorkerPausePhase } from "./worker.ts";
+import type { WorkerPauseView } from "./fleet.ts";
 import { formatRss, rssBytesFromHealthz } from "./host.ts";
 import {
   formatBaseHealth,
@@ -269,7 +269,8 @@ const GROOMING_IN_FLIGHT_REASON = "in-flight";
  *    pass saw that no durable row covers: `routed − rows`, clamped at zero so
  *    stale rows (candidates since removed from the queue) cannot push it
  *    negative. `routed` is the same denominator the tick's grooming trigger
- *    reads (`summary.routed >= groomBelow`), and the fleet's queue-digest
+ *    reads (`summary.routed >= groomBelow`; under `groomBelow: "always"`,
+ *    #988, the duty never clears on volume), and the fleet's queue-digest
  *    arithmetic already treats durable rows as current-queue facts
  *    (`claimable = routed − knownBlocked`), so this is the operator-facing
  *    half of the same subtraction. Parent/epic exclusions and backlog
@@ -472,7 +473,7 @@ export function formatFleetStatus(
   brief: string | undefined = undefined,
   decisions: string | undefined = undefined,
   failureClasses: string | undefined = undefined,
-  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
+  workerPhases: ReadonlyMap<number, WorkerPauseView> = new Map(),
   intake: string | undefined = undefined,
   lastStop: DaemonStop | undefined = undefined,
   siblings: { project: string; live: number }[] = [],
@@ -653,7 +654,7 @@ function formatSiblingLive(siblings: { project: string; live: number }[]): strin
 
 function formatProjectBody(
   s: StatusSnapshot,
-  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
+  workerPhases: ReadonlyMap<number, WorkerPauseView> = new Map(),
   now = Date.now(),
   siblings: { project: string; live: number }[] = [],
 ): string {
@@ -702,6 +703,11 @@ function formatProjectBody(
     s.caps.dailySpendUsd === null
       ? `  spend today        $${s.spendTodayUsd.toFixed(2)} estimated (no daily cap)`
       : `  spend today        $${s.spendTodayUsd.toFixed(2)} estimated / $${s.caps.dailySpendUsd.toFixed(2)}` +
+        // A dollar ceiling over a subscription-billed fleet can never fire, so
+        // showing it bare asserts headroom that is not the constraint (#984).
+        // The cap stays visible because it IS configured — what changes is that
+        // the row stops implying it governs anything.
+        (s.spendTelemetry?.kind === "subscription" ? " (inert — subscription-billed, see below)" : "") +
         (s.reservedSpendUsd === undefined || s.reservedSpendUsd === 0
           ? ""
           : ` (+$${s.reservedSpendUsd.toFixed(2)} reserved by runs in flight)`),
@@ -857,15 +863,28 @@ function formatProjectBody(
   } else {
     lines.push(leases.length === 0 ? "mutation leases  (none)" : "mutation leases");
     for (const r of leases) {
-      const phase = workerPhases.get(r.issue);
+      const pauseView = workerPhases.get(r.issue);
+      const phase = pauseView?.phase;
       // A run in a live review round reads `review-revision N`, distinct from
       // a failure and from an ordinary continuation, with the round number
       // from the durable revision row (#692). The live pause overlay still
       // wins: an operator pause is the newer fact about the same session.
+      // The overlay names its source and age when the daemon recorded them
+      // (#997): `paused (board, 4m)` answers "who and since when" at the
+      // surface where the question actually gets asked.
       const paused = phase === "pausing" || phase === "paused";
       const round = s.reviewRounds?.[r.id];
+      const pauseDetail =
+        paused && (pauseView?.source !== undefined || pauseView?.pausedAtMs !== undefined)
+          ? ` (${[
+              ...(pauseView.source === undefined ? [] : [pauseView.source]),
+              ...(pauseView.pausedAtMs === undefined
+                ? []
+                : [formatDownDuration(Math.max(0, now - pauseView.pausedAtMs))]),
+            ].join(", ")})`
+          : "";
       const state = paused
-        ? phase
+        ? `${phase}${pauseDetail}`
         : round !== undefined
           ? `review-revision ${round.round}`
           : r.state;

package/src/store.ts

@@ -211,6 +211,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
   graphTools: true,
   failureCharges: true,
   continuationCharges: true,
+  terminalEvidence: true,
 };
 
 /** Everything SQLite will accept from us. */
@@ -263,6 +264,7 @@ interface RunRow {
   lastError: string | null;
   settlementFlags: string | null;
   report: string | null;
+  terminalEvidence: string | null;
   failureClass: string | null;
   recoveryAction: string | null;
   recoveredAt: number | null;
@@ -746,6 +748,11 @@ CREATE TABLE IF NOT EXISTS runs (
   endedAt     INTEGER,
   lastError   TEXT,
   settlementFlags TEXT,
+  -- The terminal evidence the settlement sweep recorded for this attempt
+  -- (#986): exit path, first recoverable error line, audit flags. Nullable:
+  -- rows written before the column, or not yet looked at by the sweep, carry
+  -- NULL, which reads as "not recorded" — never "nothing was known".
+  terminalEvidence TEXT,
   report      TEXT,
   failureClass TEXT,
   recoveryAction TEXT,
@@ -1494,6 +1501,7 @@ function toRecord(row: RunRow): RunRecord {
     if (flags !== undefined) record.settlementFlags = flags;
   }
   if (row.report !== null) record.report = row.report;
+  if (row.terminalEvidence !== null) record.terminalEvidence = row.terminalEvidence;
   if (row.provider429Count !== null) record.provider429Count = row.provider429Count;
   if (row.resolvedModel !== null) record.resolvedModel = row.resolvedModel;
   if (row.resolvedProvider !== null) record.resolvedProvider = row.resolvedProvider;
@@ -1992,6 +2000,13 @@ export function openStore(dbPath: string): Store {
   if (!columns.some((column) => column.name === "report")) {
     db.exec("ALTER TABLE runs ADD COLUMN report TEXT");
   }
+  // The terminal evidence the settlement sweep records for every non-clean
+  // row (#986). Rows written before the column were never looked at by the
+  // recorder, so NULL is the honest reading — no backfill, because the
+  // recorder derives evidence from facts only the settling pass holds.
+  if (!columns.some((column) => column.name === "terminalEvidence")) {
+    db.exec("ALTER TABLE runs ADD COLUMN terminalEvidence TEXT");
+  }
   // Every row written before #132 is unclassified, and NULL is the honest
   // reading of that: the budget counters below deliberately still count an
   // unclassified terminal row exactly as this release's predecessor did, so an
@@ -2516,7 +2531,7 @@ export function openStore(dbPath: string): Store {
   // the same predicate against the pre-claim row, so a preserved charge can
   // never disagree with what the counters would have counted.
   const CHARGEABLE_FAILURE_EXCLUSIONS =
-    "'ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity', 'returned-for-revision'";
+    "'ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity', 'returned-for-revision', 'model-empty-stop'";
   const CHARGEABLE_CONTINUATION_EXCLUSIONS =
     "'admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity'";
 
@@ -2545,7 +2560,14 @@ export function openStore(dbPath: string): Store {
             AND (
               (state IN ('killed', 'orphaned', 'blocked')
                 AND (failureClass IS NULL OR failureClass NOT IN (${CHARGEABLE_CONTINUATION_EXCLUSIONS})))
-              OR (state = 'failed' AND failureClass = 'returned-for-revision')
+              -- Two failed-state classes charge a continuation instead of an
+              -- attempt (#795, #1001): a reviewer's decision, and a provider
+              -- that answered with empty turns until the harness gave up.
+              -- Neither is the work failing, and both resume from the head the
+              -- attempt already pushed - but each still consumes a bounded
+              -- budget, so a provider stuck in that state escalates to a human
+              -- rather than looping forever.
+              OR (state = 'failed' AND failureClass IN ('returned-for-revision', 'model-empty-stop'))
             )) AS n`,
   );
   // How many times one issue reached a given class. Recovery uses it to bound a
@@ -2665,6 +2687,15 @@ export function openStore(dbPath: string): Store {
       GROUP BY failureClass
       ORDER BY n DESC, failureClass ASC`,
   );
+  // The surviving `unknown` rows (#986): classified but never explained, each
+  // carrying its recorded terminal evidence for `status` to print. Bounded —
+  // a backlog must not turn one status render into an unbounded scan.
+  const selectUnknownEvidence = db.query<RunRow, [string, number]>(
+    `SELECT * FROM runs
+      WHERE project = ? AND failureClass = 'unknown'
+      ORDER BY startedAt DESC, rowid DESC
+      LIMIT ?`,
+  );
   const selectRecoveredSince = db.query<RunRow, [string, number]>(
     `SELECT * FROM runs
       WHERE project = ? AND recoveredAt IS NOT NULL AND recoveredAt >= ?
@@ -3727,7 +3758,7 @@ export function openStore(dbPath: string): Store {
          ELSE 0
        END,
        continuationCharges = COALESCE(continuationCharges, 0) + CASE
-         WHEN state = 'failed' AND failureClass = 'returned-for-revision' THEN 1
+         WHEN state = 'failed' AND failureClass IN ('returned-for-revision', 'model-empty-stop') THEN 1
          WHEN state = 'killed' AND (failureClass IS NULL OR failureClass NOT IN (${CHARGEABLE_CONTINUATION_EXCLUSIONS})) THEN 1
          ELSE 0
        END
@@ -5082,6 +5113,11 @@ export function openStore(dbPath: string): Store {
       return entry;
     },
 
+    /** The surviving `unknown` rows, newest first, with whatever terminal
+     *  evidence the sweep recorded on them (#986). Bounded per call. */
+    unknownRuns(project: string, limit = 20): RunRecord[] {
+      return selectUnknownEvidence.all(project, limit).map(toRecord);
+    },
     verbLedger(
       project: string,
       opts: { runId?: string; issue?: number; limit?: number } = {},

package/src/types.ts

@@ -1007,6 +1007,16 @@ export type ArmProof = (typeof ARM_PROOFS)[number];
  */
 export const DEFAULT_ARM_PROOF: ArmProof = "challenge";
 
+/**
+ * The grooming trigger (#988): a numeric threshold keeps today's behaviour —
+ * the tick grooms when the routable count is below it — and the literal
+ * `"always"` grooms on demand every tick while any ungroomed, unrefused,
+ * unparked candidate exists, judged from candidate state rather than queue
+ * volume. Never spell "always" as a large number: a big queue must not
+ * silence the duty.
+ */
+export type GroomTrigger = number | "always";
+
 /**
  * Everything the dispatcher needs to service one product: where work comes
  * from, where code goes, and what it may spend doing it. Config is per project
@@ -1020,9 +1030,10 @@ export interface ProjectConfig {
   /** The one label that means "a human has signed this off as agent-ready". */
   queueLabel: string;
   /** Routable-candidate count below which the tick prompt tells the orchestrator
-   *  to groom the queue. Optional; defaults to {@link DEFAULT_GROOM_BELOW} in
+   *  to groom the queue, or {@link GroomTrigger} `"always"` to groom on demand
+   *  every tick (#988). Optional; defaults to {@link DEFAULT_GROOM_BELOW} in
    *  orchestrator-tick.ts. */
-  groomBelow?: number;
+  groomBelow?: GroomTrigger;
   /** Labels the dispatcher writes back so the tracker alone shows live state
    *  to a human who never opens the daemon's logs. `backlog` is the
    *  operator's own park gesture (#507) — not dispatcher-written, but read by
@@ -1683,6 +1694,19 @@ export const FAILURE_CLASSES = [
   /** A reviewer closed pushed-green or pushed-pending work without merging it:
    *  a review decision, not a worker failure. */
   "returned-for-revision",
+  /** The session ended without delivering any settlement verdict: no yield, no
+   *  green claim, no error — the row's last words are mid-delivery narration
+   *  (#986). Distinct from `unknown` on purpose: the evidence proves the run
+   *  reached its worker and died between turns, so the gap it names is "the
+   *  model stopped before handing over" rather than "nothing is known". It
+   *  charges the attempt exactly as `unknown` did; only the name is specific. */
+  "no-verdict",
+  /** The provider answered with empty assistant turns until the harness's own
+   *  retry cap gave up and ended the session (#986): the terminal line reads
+   *  `empty-stop-retry-cap`. Provider flake, not worker logic — both of the
+   *  2026-08-23 failures were this, at 100/180 and 54/180 turns, while a third
+   *  run on the same model finished green in the same window. */
+  "model-empty-stop",
   "unknown",
 ] as const;
 
@@ -2084,6 +2108,15 @@ export interface RunRecord {
   endedAt?: number;
   /** Last failure text, surfaced verbatim in escalations. */
   lastError?: string;
+  /**
+   * The terminal evidence the settlement sweep recorded for this attempt
+   * (#986): the exit path the row took, the first line of whatever error was
+   * recoverable from the row or its transcript, and any settlement-audit
+   * flags — one bounded line, readable in `status` beside a surviving
+   * `unknown`. Absent means the row predates the column, or the sweep has not
+   * looked at it yet; it never means "nothing was known".
+   */
+  terminalEvidence?: string;
   /** Advisory settlement-audit findings for this attempt (#128). Deliberately
    *  never consulted by anything that decides `state`: a flagged run settles
    *  exactly as an unflagged one does, and the flags are evidence for whoever
@@ -3285,6 +3318,10 @@ export interface Store {
    * every actionable class behind a monotonically growing tally.
    */
   recordedOnlyClassCounts(project: string): { cls: FailureClass; n: number }[];
+  /** The surviving `unknown` rows, newest first, each with whatever terminal
+   *  evidence the sweep recorded on it (#986). Bounded per call — a status
+   *  render must never walk a whole backlog. */
+  unknownRuns(project: string, limit?: number): RunRecord[];
   /** Rows whose recovery ran at or after `since`, newest first — the tick's
    *  "auto-recovered since last tick" line, and its count. */
   recoveredSince(project: string, since: number): RunRecord[];

package/src/usage.ts

@@ -688,6 +688,31 @@ export async function readPlanUsage(
  * to fit on one row. Derived from the same status object as the long form so
  * the two cannot disagree about whether the guard is active.
  */
+/**
+ * The allowance window nearest its ceiling, named for a reader (#984).
+ *
+ * A subscription has an allowance where an API key has a price, so this is what
+ * "the constraint that binds" means on a subscription-billed fleet — and the
+ * one nearest exhaustion is the one that will stop the fleet first, whatever
+ * the others say. Read from the provider's own reply, never derived: a fleet
+ * whose provider reports no window gets `undefined`, which is the honest answer
+ * and keeps a genuine telemetry fault a fault.
+ *
+ * Only windows that reported a comparable fraction are eligible, for the reason
+ * this module already documents at length: `used` is not comparable across
+ * providers, so a raw count can never stand in for one.
+ */
+export function bindingAllowanceWindow(reading: UsageReading): string | undefined {
+  if (reading.kind !== "ok") return undefined;
+  const comparable = reading.windows.filter((w) => w.usedFraction !== undefined);
+  if (comparable.length === 0) return undefined;
+  let worst = comparable[0]!;
+  for (const window of comparable.slice(1)) {
+    if ((window.usedFraction ?? 0) > (worst.usedFraction ?? 0)) worst = window;
+  }
+  return worst.label === undefined ? worst.id : `${worst.id} (${worst.label})`;
+}
+
 export function planUsageBadge(status: PlanUsageStatus | undefined): string {
   if (status === undefined) return "plan not read";
   let core: string;