omp-conductor 0.20.1 → 0.20.3

package/src/tracker/github.ts

@@ -94,6 +94,12 @@ interface GhCheck {
   status?: string;
   conclusion?: string;
   state?: string;
+  /** When the run started (`startedAt` on a GraphQL CheckRun, `started_at`
+   *  over REST); a StatusContext stamps `createdAt` instead. Undefined when
+   *  the payload spelled none — unordered entries resolve pessimistically
+   *  (#1066), never invented. */
+  startedAt?: string;
+  createdAt?: string;
   detailsUrl?: string;
 }
 
@@ -419,6 +425,75 @@ function checkVerdict(check: GhCheck): CheckVerdict {
   return "failed";
 }
 
+/** One check normalized onto the verdict vocabulary, carrying whatever run
+ *  timestamp its payload spelled: undefined means unordered, which
+ *  {@link resolveNewestPerName} treats pessimistically. */
+interface NamedVerdict {
+  readonly name: string;
+  readonly verdict: CheckVerdict;
+  readonly state: string;
+  readonly startedAt: string | undefined;
+}
+
+/** Epoch ms of an ISO stamp, or undefined when absent or unreadable — both
+ *  leave the entry unordered rather than inventing an age for it. */
+function startedAtMs(startedAt: string | undefined): number | undefined {
+  if (startedAt === undefined) return undefined;
+  const ms = Date.parse(startedAt);
+  return Number.isFinite(ms) ? ms : undefined;
+}
+
+/** Worse beats better when a name's deciding entries disagree. */
+const VERDICT_RANK: Readonly<Record<CheckVerdict, number>> = { failed: 0, pending: 1, green: 2 };
+
+/**
+ * Decide every check name by its newest run, never by payload order (#1066).
+ *
+ * One head can carry several check-runs of one name — a re-run creates a new
+ * run record without retiring the old one — and folding them first-match lets
+ * a stale `skipped` mask a newer `failure`: exactly how veltro#717 nearly
+ * merged green over its only upgrade-path check, whose failing run started
+ * five minutes behind the skipped row of the same name. Entries are grouped
+ * by name and each group is decided by its newest tier, GitHub's own checks-UI
+ * rule:
+ *
+ * - timed entries at the maximum `started_at` decide; older runs of the name
+ *   cannot mask them, so a re-run that went green supersedes its red
+ *   predecessor;
+ * - ties at one instant resolve pessimistically — a failure anywhere in the
+ *   deciding tier fails the name;
+ * - an entry without a readable timestamp cannot be proven older than
+ *   anything, so it belongs to every deciding tier (pessimistic again).
+ *
+ * The representative kept per name is the deciding entry itself, so the
+ * folds' refusal lines render the decided verdict's own spelling.
+ */
+function resolveNewestPerName(entries: readonly NamedVerdict[]): NamedVerdict[] {
+  const groups = new Map<string, { entry: NamedVerdict; startedMs: number | undefined }[]>();
+  for (const entry of entries) {
+    const stamped = { entry, startedMs: startedAtMs(entry.startedAt) };
+    const group = groups.get(entry.name);
+    if (group === undefined) groups.set(entry.name, [stamped]);
+    else group.push(stamped);
+  }
+  const resolved: NamedVerdict[] = [];
+  for (const group of groups.values()) {
+    let newest = -Infinity;
+    for (const { startedMs } of group) {
+      if (startedMs !== undefined && startedMs > newest) newest = startedMs;
+    }
+    let decided: NamedVerdict | undefined;
+    for (const { entry, startedMs } of group) {
+      if (startedMs !== undefined && startedMs !== newest) continue;
+      if (decided === undefined || VERDICT_RANK[entry.verdict] < VERDICT_RANK[decided.verdict]) {
+        decided = entry;
+      }
+    }
+    if (decided !== undefined) resolved.push(decided);
+  }
+  return resolved;
+}
+
 /**
  * Verify one `gh pr view --json state,isDraft,headRefOid,statusCheckRollup`
  * payload, optionally against a caller-observed head. A missing rollup is
@@ -430,6 +505,10 @@ function checkVerdict(check: GhCheck): CheckVerdict {
  * a different question: *what is this PR's state*, and a merged or closed PR
  * is reported as a fact rather than a failure — the merge gate paths never
  * pass `read`, so the `expected OPEN` refusal stays exactly where it gates.
+ *
+ * Same-name entries are decided by recency, not payload order (#1066): only
+ * the newest run of a name answers for it, ties and unreadable stamps resolve
+ * pessimistically, and the green count names de-duplicated checks.
  */
 export function prVerificationFrom(
   raw: string,
@@ -467,23 +546,34 @@ export function prVerificationFrom(
   if (checks.length === 0) {
     return { status: "pending", reason: "GitHub has not reported any checks yet", headSha };
   }
-  const failed = checks.filter((check) => checkVerdict(check) === "failed");
+  // Same-name runs decide by recency, not payload order (#1066): the rollup
+  // can carry an older run of a name beside a newer one, and first-match
+  // folding would let the older verdict answer for the name.
+  const resolved = resolveNewestPerName(
+    checks.map((check): NamedVerdict => ({
+      name: checkName(check),
+      verdict: checkVerdict(check),
+      state: check.conclusion ?? check.state ?? "unknown",
+      startedAt: check.startedAt ?? check.createdAt,
+    })),
+  );
+  const failed = resolved.filter((check) => check.verdict === "failed");
   if (failed.length > 0) {
     return {
       status: "failed",
-      reason: `Checks failed: ${failed.map((check) => `${checkName(check)} (${check.conclusion ?? check.state ?? "unknown"})`).join(", ")}`,
+      reason: `Checks failed: ${failed.map((check) => `${check.name} (${check.state})`).join(", ")}`,
       headSha,
     };
   }
-  const pending = checks.filter((check) => checkVerdict(check) === "pending");
+  const pending = resolved.filter((check) => check.verdict === "pending");
   if (pending.length > 0) {
     return {
       status: "pending",
-      reason: `Checks pending: ${pending.map(checkName).join(", ")}`,
+      reason: `Checks pending: ${pending.map((check) => check.name).join(", ")}`,
       headSha,
     };
   }
-  return { status: "green", reason: `${checks.length} checks succeeded or were skipped`, headSha };
+  return { status: "green", reason: `${resolved.length} checks succeeded or were skipped`, headSha };
 }
 
 /**
@@ -538,6 +628,9 @@ interface RestCheckConclusion {
    *  GitHub spelled it — surfaced in refusals exactly like the GraphQL one. */
   state: string;
   verdict: CheckVerdict;
+  /** The run's `started_at` (or a commit status's `updated_at`), undefined
+   *  when the page spelled none — unordered, never invented (#1066). */
+  startedAt: string | undefined;
 }
 
 /** One REST check plane, parsed strictly. */
@@ -579,6 +672,7 @@ function restRunConclusionFrom(row: { readonly [key: string]: unknown }): RestCh
   const name = row["name"];
   const status = row["status"];
   const conclusion = row["conclusion"];
+  const startedAt = row["started_at"];
   if (typeof name !== "string" || name === "") return undefined;
   if (typeof status !== "string") return undefined;
   if (conclusion !== undefined && conclusion !== null && typeof conclusion !== "string") return undefined;
@@ -591,7 +685,12 @@ function restRunConclusionFrom(row: { readonly [key: string]: unknown }): RestCh
       : conclusion === "success" || conclusion === "skipped" || conclusion === "neutral"
         ? "green"
         : "failed";
-  return { name, state: typeof conclusion === "string" ? conclusion : status, verdict };
+  return {
+    name,
+    state: typeof conclusion === "string" ? conclusion : status,
+    verdict,
+    startedAt: typeof startedAt === "string" ? startedAt : undefined,
+  };
 }
 
 /** One REST commit-status member onto the rollup vocabulary, or undefined when
@@ -599,12 +698,14 @@ function restRunConclusionFrom(row: { readonly [key: string]: unknown }): RestCh
 function restStatusConclusionFrom(row: { readonly [key: string]: unknown }): RestCheckConclusion | undefined {
   const context = row["context"];
   const state = row["state"];
+  const updatedAt = typeof row["updated_at"] === "string" ? row["updated_at"] : undefined;
+  const createdAt = typeof row["created_at"] === "string" ? row["created_at"] : undefined;
   if (typeof context !== "string" || context === "") return undefined;
   if (typeof state !== "string") return undefined;
   // StatusContext semantics: `success` is green, `pending` pending, anything
   // else (`failure`/`error`) is failed.
   const verdict: CheckVerdict = state === "success" ? "green" : state === "pending" ? "pending" : "failed";
-  return { name: context, state, verdict };
+  return { name: context, state, verdict, startedAt: updatedAt ?? createdAt };
 }
 
 /**
@@ -706,6 +807,10 @@ function restProjectionVerdict(
  * Throws on a payload that cannot prove the head or either check plane, so the
  * caller maps it to `head-unresolvable` exactly as it does for the GraphQL
  * parser.
+ *
+ * Same-name runs are decided by recency, not page order (#1066): the newest
+ * attempt of a name answers for it, so a stale `skipped` beside a newer
+ * `failure` fails the verdict whatever order the pages arrive in.
  */
 export function restVerificationFrom(
   prRaw: string,
@@ -732,7 +837,11 @@ export function restVerificationFrom(
   if (checks.length === 0) {
     return { status: "pending", reason: "GitHub has not reported any checks yet", headSha };
   }
-  const failed = checks.filter((c) => c.verdict === "failed");
+  // Same-name runs decide by recency, not page order (#1066): the flattened
+  // planes carry every attempt of a name, and first-match folding would let
+  // the oldest attempt answer for it.
+  const resolved = resolveNewestPerName(checks);
+  const failed = resolved.filter((c) => c.verdict === "failed");
   if (failed.length > 0) {
     return {
       status: "failed",
@@ -740,7 +849,7 @@ export function restVerificationFrom(
       headSha,
     };
   }
-  const pending = checks.filter((c) => c.verdict === "pending");
+  const pending = resolved.filter((c) => c.verdict === "pending");
   if (pending.length > 0) {
     return {
       status: "pending",
@@ -748,7 +857,7 @@ export function restVerificationFrom(
       headSha,
     };
   }
-  return { status: "green", reason: `${checks.length} checks succeeded or were skipped`, headSha };
+  return { status: "green", reason: `${resolved.length} checks succeeded or were skipped`, headSha };
 }
 
 /**
@@ -2074,6 +2183,19 @@ export function makeTracker(
           const rest = await verifyPrRest(runGh, cache, url, expectedHead, opts, hooks.onNotModified);
           return rest ?? verification;
         }
+        if (verification.status === "green") {
+          // A rollup context is one NAME, not one RUN (#1066): veltro#717
+          // read eighteen green contexts while the same SHA carried a newer
+          // `failure` run of a name whose rollup row still said `skipped`.
+          // Recency ordering helps only when both rows arrive together; a
+          // rollup that omits the newer run entirely cannot be ordered back
+          // to correctness, so green is re-proven from the flattened REST
+          // planes — the same evidence the empty-rollup path above trusts.
+          // REST that cannot answer leaves this verdict standing, exactly as
+          // a degraded plane leaves the #999 pending standing.
+          const rest = await verifyPrRest(runGh, cache, url, expectedHead, opts, hooks.onNotModified);
+          return rest ?? verification;
+        }
         if (verification.status !== "failed") return verification;
 
         const detailsUrl = failedCheck(raw)?.detailsUrl;

package/src/types.ts

@@ -90,6 +90,12 @@ export interface Caps {
   /** Wall-clock ceiling for one worker (90 min): a session that is merely
    *  stuck spends no turns, so turns alone cannot detect it. */
   workerWallClockMs: number;
+  /** How long a live worker's transcript may stay unwritten before the
+   *  daemon settles it as a progress stall (#1086). `null` derives it from the
+   *  wall-clock ceiling — one third, so a hung run is caught long before its
+   *  clock runs out. A single turn can legitimately run for minutes on a slow
+   *  provider, so this reads *transcript writes*, never the turn counter. */
+  workerStallSilenceMs: number | null;
   /** Failed implementation/CI attempts allowed before escalation. Operational
    *  continuations do not consume this budget. */
   maxAttemptsPerIssue: number;
@@ -1831,6 +1837,13 @@ export const FAILURE_CLASSES = [
    *  2026-08-23 failures were this, at 100/180 and 54/180 turns, while a third
    *  run on the same model finished green in the same window. */
   "model-empty-stop",
+  /** A live session stopped writing its transcript for longer than the
+   *  configured silence window, and the daemon settled it (#1086). Not a size
+   *  verdict: the work did not run out of budget, the session simply never
+   *  came back — the evidence names how long it was silent and at which turn.
+   *  When the attempt had already pushed a green PR, the settle path lands the
+   *  row `pushed-green` instead, so landed work is never charged anything. */
+  "progress-stall",
   "unknown",
 ] as const;
 
@@ -2131,6 +2144,14 @@ export interface RunRecord {
   autoCompactionCount?: number;
   /** omp session transcript, so a human can read what the worker actually did. */
   sessionFile?: string;
+  /**
+   * When the run's transcript was last observed to grow, as epoch ms (#1086).
+   * Recorded by the daemon's progress watch from the file's own mtime — the
+   * write instant, not the observation instant — and read by `status` to show
+   * a silence interval while the row is live. Absent means the pass has not
+   * looked at this run (or it has no transcript yet), never "no progress".
+   */
+  lastProgressAt?: number;
   /**
    * The exact `session-host` pid this run's Herdr representation was reported
    * for, and the pane Herdr knows it as (#842).
@@ -3372,6 +3393,11 @@ export interface Store {
    * predicate pick the window. Read-only.
    */
   recentSpendSamples(project: string, limit: number): { turns: number; spendUsd: number }[];
+  /** The newest runs carrying a usable per-turn latency (#1063): terminal
+   *  rows with turns and elapsed time recorded, newest first. Bounded by
+   *  `limit` rows like `recentSpendSamples` — the caller over-samples and lets
+   *  its own model-attribution filter pick the window. Read-only. */
+  recentLatencySamples(project: string, limit: number): RunRecord[];
   /** Total tracked `gh` calls between two UTC day keys, inclusive (#198). */
   ghCallsBetween(sinceDay: string, untilDay: string): number;
   /** Idempotence guard so a retry loop cannot page a human repeatedly for the
@@ -4034,6 +4060,11 @@ export const DEFAULT_CAPS: Caps = {
   workerMaxTurns: 120,
   workerMaxTurnsCeiling: 240,
   workerWallClockMs: 90 * 60 * 1000,
+  // Derived as a third of the wall-clock ceiling unless an operator names the
+  // window: a hung run is caught with most of its budget unspent, while a
+  // slow-but-writing session (measured fleet latency has reached
+  // 1.5 min/turn) never trips it (#1086).
+  workerStallSilenceMs: null,
   maxAttemptsPerIssue: 2,
   maxContinuationsPerIssue: 2,
 };

package/src/upgrade.ts

@@ -1696,7 +1696,8 @@ export async function upgradeConductor(
     journal({ kind: "phase", phase: "drain", ok: false, version: release.version, detail: err instanceof Error ? err.message : String(err) });
     const failure = err instanceof Error ? err.message : String(err);
     throw new Error(
-      `upgrade failed while draining workers: ${failure}; no installation started; dispatch remains paused`,
+      `upgrade failed while draining workers: ${failure}; no installation started; ` +
+        `dispatch remains paused — resume dispatch with \`omp-conductor resume --all\``,
     );
   }
   journal({ kind: "phase", phase: "drain", ok: true, version: release.version });
@@ -1916,7 +1917,8 @@ export async function upgradeConductor(
           `Restored: cli=${surfaces.cliVersion}, omp=${surfaces.ompVersion ?? "missing"}, ` +
             `herdr=${surfaces.herdrSource ?? "missing"}`,
           ...(preUpgradeBackup === undefined ? [] : [`Pre-upgrade config snapshot: ${preUpgradeBackup}`]),
-          "Dispatch stays paused until the first tick after the restart records the rollback verified.",
+          "Dispatch stays paused until the first tick after the restart records the rollback verified; " +
+            "if it is still paused, resume it with `omp-conductor resume --all`.",
         ],
         `upgrade:${release.version}:rolled-back`,
       );
@@ -1931,16 +1933,21 @@ export async function upgradeConductor(
           `Failure: ${failure}`,
           installAttribution,
           `Rollback failure: ${rollback}`,
-          "The fleet may be at mixed versions; do not resume dispatch.",
+          "The fleet may be at mixed versions; do not resume dispatch until versions are verified.",
+          "Once one version is confirmed everywhere, `omp-conductor resume --all` lifts the pause.",
           "Restore manually from the journal at " + upgradeJournalPath(stateDir()) + ".",
         ],
         `upgrade:${release.version}:rollback-failed`,
       );
       throw new Error(
-        `upgrade failed: ${failure}; rollback incomplete: ${rollback}; dispatch remains paused`,
+        `upgrade failed: ${failure}; rollback incomplete: ${rollback}; dispatch remains paused — ` +
+          `\`omp-conductor resume --all\` lifts the pause once mixed versions are ruled out`,
       );
     }
-    throw new Error(`upgrade failed: ${failure}; previous installation restored; dispatch remains paused`);
+    throw new Error(
+      `upgrade failed: ${failure}; previous installation restored; dispatch remains paused — ` +
+        `resume dispatch with \`omp-conductor resume --all\``,
+    );
   }
 
   // The install landed; name any host-runtime destination this version no

package/src/verbs/server.ts

@@ -426,7 +426,8 @@ export function releaseRequirementRefusal(
       const drain =
         ". Settle the fleet with a bounded drain: " +
         `omp-conductor drain start${facts.project === undefined ? "" : ` --project ${facts.project}`} --until <deadline>` +
-        " — claiming pauses while settlement runs, and admission resumes at the deadline on its own.";
+        " — claiming pauses until the deadline even once the fleet goes quiet (an emptied fleet opens" +
+        " the window rather than ending the drain early), and admission resumes at the deadline on its own.";
       return (
         `policy.release.requires includes ${requirement} and ${facts.unsettledRuns} run(s) have not settled` +
         (parts.length === 0 ? "" : ` — ${parts.join("; ")}`) +
@@ -2292,22 +2293,25 @@ async function prUpdateVerb(
  */
 
 /**
- * The run states a review revision may start from (#795).
+ * The run states a review revision may start from (#795, #1101).
  *
  * A revision round resumes the exact run whose row owns the PR, so the
- * revisable states are exactly the terminal runs that pushed one: a settled
- * `pushed-green` row, or a `failed` / `killed` row — a run that capped or
- * failed *after* pushing a green PR. The PR is the durable artefact, the
- * exact-head green verification in the verb is the gate on "green at the
- * reviewed SHA", and a terminal row proves no worker is in flight, so the
- * findings are returned without the close-PR → unblock → continuation dance.
+ * revisable states are exactly the terminal runs that own the named PR: a
+ * settled `pushed-green` row, a `failed` / `killed` row — a run that capped or
+ * failed *after* pushing a green PR — and, since #1101, a `stopped` row that
+ * had already pushed when the operator stopped it. Ownership is proved by the
+ * selection itself (`runsForProjectPr` matches rows that recorded the PR), so
+ * a stopped run that never pushed can never reach this gate. Without the
+ * stopped leg a stopped-with-green-PR run was both unreviewable (this verb)
+ * and unreplaceable (the open-PR dispatch guard) at once — closing the PR by
+ * hand was the only exit, which is exactly what #1096 hit.
  *
  * Closed on purpose: a live row (`running` / `claimed`) is already doing its
  * own work, a `pushed-pending` PR is not green yet, and a `blocked` /
- * `orphaned` / `stopped` / `merged` row is not work returned for revision.
- * The set itself is the shared `REVISABLE_RUN_STATES` in `decisions.ts` —
- * the one definition the review verb and the `pr-review-ready` watch gate on,
- * so the two can never drift (#844).
+ * `orphaned` / `merged` row is not work returned for revision. The set itself
+ * is the shared `REVISABLE_RUN_STATES` in `decisions.ts` — the one definition
+ * the review verb and the `pr-review-ready` watch gate on, so the two can
+ * never drift (#844).
  */
 
 async function prReviewVerb(