omp-conductor 0.7.1 → 0.9.0

package/src/decisions.ts

@@ -17,7 +17,7 @@ import type { DecisionRecord, Store, Tracker } from "./types.ts";
 /**
  * A precondition whose truth this package can check on its own.
  *
- * Exactly three kinds, deliberately. Each one is a question the tracker or npm
+ * Exactly six kinds, deliberately. Each one is a question the tracker or npm
  * already answers, so the row moves from "parked" to "act on this" without a
  * human re-reading it. Anything richer — a label appearing, a workflow going
  * green — is a follow-on issue rather than a grammar nobody validated.
@@ -25,11 +25,22 @@ import type { DecisionRecord, Store, Tracker } from "./types.ts";
 export type DecisionCondition =
   | { kind: "pr-merged"; url: string }
   | { kind: "issue-closed"; issue: number }
-  | { kind: "npm-version"; spec: string };
+  | { kind: "npm-version"; spec: string }
+  | { kind: "pr-checks-green"; url: string }
+  | { kind: "pr-mergeable"; url: string }
+  | { kind: "rate-limit-reset" };
 
 /** `pkg@version`, including a scoped package (`@scope/pkg@1.2.3`). */
 const NPM_SPEC = /^(@[^/@\s]+\/)?[^@\s]+@[^\s]+$/;
 
+/**
+ * Check states that count as a green verdict for `pr-checks-green`, copied from
+ * the daemon's failure classifier (`failure-class.ts` `SUCCESS_CHECK_STATES`):
+ * both spellings are terminally successful, and anything else — a failure, a
+ * still-running/pending check, a cancelled or skipped runner — is not.
+ */
+const GREEN_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
+
 /**
  * Parse a raw condition, or `undefined` when it is not one of the three forms.
  *
@@ -46,8 +57,8 @@ export function parseCondition(raw: string): DecisionCondition | undefined {
   const rest = text.slice(at + 1).trim();
   if (rest.length === 0) return undefined;
 
-  if (kind === "pr-merged") {
-    return rest.startsWith("https://") ? { kind: "pr-merged", url: rest } : undefined;
+  if (kind === "pr-merged" || kind === "pr-checks-green" || kind === "pr-mergeable") {
+    return rest.startsWith("https://") ? { kind, url: rest } : undefined;
   }
   if (kind === "issue-closed") {
     if (!/^\d+$/.test(rest)) return undefined;
@@ -57,14 +68,22 @@ export function parseCondition(raw: string): DecisionCondition | undefined {
   if (kind === "npm-version") {
     return NPM_SPEC.test(rest) ? { kind: "npm-version", spec: rest } : undefined;
   }
+  if (kind === "rate-limit-reset") {
+    // Spelled `rate-limit-reset:github`: the grammar requires a non-empty rest,
+    // and `github` names the one GraphQL provider the fleet is on.
+    return rest === "github" ? { kind: "rate-limit-reset" } : undefined;
+  }
   return undefined;
 }
 
-/** The three accepted forms, for a refusal that can be acted on in one turn. */
+/** The six accepted forms, for a refusal that can be acted on in one turn. */
 export const CONDITION_FORMS = [
   "pr-merged:https://github.com/owner/repo/pull/123",
   "issue-closed:123",
   "npm-version:omp-conductor@0.4.3",
+  "pr-checks-green:https://github.com/owner/repo/pull/123",
+  "pr-mergeable:https://github.com/owner/repo/pull/123",
+  "rate-limit-reset:github",
 ] as const;
 
 /** Probe for `npm-version`, injectable so tests never reach the network. */
@@ -90,6 +109,33 @@ export const probeNpmVersion: NpmProbe = async (spec) => {
   }
 };
 
+/** Probe for `rate-limit-reset`, injectable so tests never reach the network. */
+export type RateLimitProbe = () => Promise<boolean>;
+
+/**
+ * `gh api rate_limit` — any remaining GraphQL quota means the limit has reset
+ * (or never bound). Bounded at 10 s like {@link probeNpmVersion}, for the same
+ * reason: this runs inside a tick, and a hung `gh` must cost one unevaluated
+ * decision, not the tick.
+ */
+export const probeRateLimitReset: RateLimitProbe = async () => {
+  const proc = Bun.spawn(
+    ["gh", "api", "rate_limit", "--jq", ".resources.graphql.remaining"],
+    { stdout: "pipe", stderr: "ignore" },
+  );
+  const timer = setTimeout(() => {
+    proc.kill();
+  }, 10_000);
+  try {
+    const [text, code] = await Promise.all([new Response(proc.stdout).text(), proc.exited]);
+    return code === 0 && Number.parseInt(text.trim(), 10) > 0;
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+};
+
 /**
  * Check every open decision that carries a condition and has not met it yet.
  *
@@ -105,7 +151,7 @@ export async function evaluateDecisionConditions(
   store: Store,
   project: string,
   tracker: Tracker,
-  probeNpm: NpmProbe,
+  probes: { npm: NpmProbe; rateLimit: RateLimitProbe },
   now: () => number,
 ): Promise<DecisionRecord[]> {
   const met: DecisionRecord[] = [];
@@ -119,8 +165,22 @@ export async function evaluateDecisionConditions(
         satisfied = (await tracker.prState(condition.url)) === "merged";
       } else if (condition.kind === "issue-closed") {
         satisfied = (await tracker.issueState(condition.issue)) === "closed";
+      } else if (condition.kind === "npm-version") {
+        satisfied = await probes.npm(condition.spec);
+      } else if (condition.kind === "pr-checks-green") {
+        // The same conclusion values the daemon's failure classifier treats as
+        // a green verdict (`success` / `neutral`, lowercased): a non-empty list
+        // in which every check is terminally successful and none is failing or
+        // pending (#189).
+        const checks = await tracker.checkConclusions(condition.url);
+        satisfied =
+          checks.length > 0 && checks.every((c) => GREEN_CHECK_STATES[c.state.trim().toLowerCase()] === true);
+      } else if (condition.kind === "pr-mergeable") {
+        // `clean` is the tracker's "this PR can merge" literal; `unknown` and a
+        // conflict are both unsatisfied (#189).
+        satisfied = (await tracker.mergeable(condition.url)) === "clean";
       } else {
-        satisfied = await probeNpm(condition.spec);
+        satisfied = await probes.rateLimit();
       }
     } catch {
       continue;

package/src/diff-flags.ts

@@ -435,6 +435,14 @@ function evidence(text: string): string {
 export interface SettlementAudit {
   /** The worker's final report, verbatim. */
   report: string;
+  /** Earlier attempts' reports for the same issue, oldest first. Their
+   *  `changed:` disclosures are pooled as coverage so a file disclosed in a
+   *  prior attempt is never flagged as undisclosed in the final one, while the
+   *  reverse (unmatched-claim) direction reads only the current report — a file
+   *  claimed in attempt 1 and reverted by attempt 3 must not come back as a new
+   *  finding (#199). Absent means this is the only attempt, or older rows never
+   *  stored a report. */
+  priorReports?: readonly string[];
   /** The dispatching issue's title and body — the attribution source. */
   issueText: string;
   diff: PrDiff;
@@ -457,8 +465,13 @@ export function analyseSettlement(audit: SettlementAudit): SettlementFlag[] {
 function reconcileChanged(audit: SettlementAudit, flags: SettlementFlag[]): void {
   const touched = audit.diff.files.filter((f) => DERIVED_FILE[basename(f.path)] !== true);
   const claims = claimedPaths(audit.report);
+  const priorClaims = claimedPaths((audit.priorReports ?? []).join("\n"));
+  // Coverage pools the current report with every prior attempt's disclosures:
+  // a file the final report no longer names was disclosed while the work was
+  // still in flight, so it must not be flagged undisclosed (#199).
+  const coverage = [...new Set([...claims, ...priorClaims])];
 
-  if (claims.length === 0) {
+  if (claims.length === 0 && priorClaims.length === 0) {
     // The degenerate case of the same check: with no usable `changed:` line
     // every file is undisclosed, and saying so once beats saying it per file.
     // Reporting it at all is what stops the check being defeated by writing
@@ -476,8 +489,8 @@ function reconcileChanged(audit: SettlementAudit, flags: SettlementFlag[]): void
   }
 
   for (const file of touched) {
-    if (claims.some((claim) => covers(claim, file.path))) continue;
-    if (file.previousPath !== undefined && claims.some((claim) => covers(claim, file.previousPath ?? ""))) {
+    if (coverage.some((claim) => covers(claim, file.path))) continue;
+    if (file.previousPath !== undefined && coverage.some((claim) => covers(claim, file.previousPath ?? ""))) {
       continue;
     }
     flags.push({
@@ -491,6 +504,10 @@ function reconcileChanged(audit: SettlementAudit, flags: SettlementFlag[]): void
   // file it edited and then reverted, or one it renamed away from. Worth
   // surfacing because a `changed:` line that describes a different PR is the
   // signature of a report written from memory rather than from `git diff`.
+  //
+  // It iterates only the CURRENT report's claims (#199). A file claimed in
+  // attempt 1 and reverted by attempt 3 was true then and false now; pooling
+  // priors into this loop would turn every honest revert into a fresh finding.
   for (const claim of claims) {
     if (audit.diff.files.some((f) => covers(claim, f.path) || covers(claim, f.previousPath ?? ""))) {
       continue;
@@ -661,18 +678,30 @@ const HEADING = "settlement audit";
  * that could not be read in full — then the absence of flags is not evidence of
  * anything, and saying so is the honest report.
  */
+/** Appended to the heading when a multi-attempt audit pooled prior disclosures,
+ *  so a reader is told the coverage is wider than this one report (#199). */
+function pooledHeading(attempts: number | undefined): string {
+  return attempts !== undefined && attempts > 1
+    ? ` — disclosures pooled across ${attempts} attempt(s)`
+    : "";
+}
+
 export function formatSettlementFlags(
   flags: readonly SettlementFlag[],
-  diff: { truncated: boolean } = { truncated: false },
+  diff: { truncated: boolean; attempts?: number } = { truncated: false },
 ): string[] {
   if (flags.length === 0) {
     return diff.truncated
-      ? [`${HEADING}: no flags, but the PR diff was too large to read in full — this is not a clean bill`]
+      ? [
+          `${HEADING}: no flags, but the PR diff was too large to read in full` +
+            `${pooledHeading(diff.attempts)} — this is not a clean bill`,
+        ]
       : [];
   }
   const lines = [
     `${HEADING}: ${flags.length} advisory flag(s) — the run's state is unchanged by them` +
-      (diff.truncated ? ", and the PR diff was too large to read in full" : ""),
+      (diff.truncated ? ", and the PR diff was too large to read in full" : "") +
+      pooledHeading(diff.attempts),
   ];
   for (const flag of flags.slice(0, RENDERED_FLAGS)) {
     lines.push(

package/src/fleet.ts

@@ -42,6 +42,8 @@ import {
   formatReleaseGrants,
   formatSalvagedRuns,
   isPaused,
+  pauseProvenance,
+  pausedAt,
   setPaused,
   statusSnapshot,
   type StatusSnapshot,
@@ -55,6 +57,7 @@ import {
   SYSTEMD_UNIT,
 } from "./lifecycle.ts";
 import { formatRss, rssBytesFromHealthz } from "./host.ts";
+import { fetchRateLimit } from "./tracker/github.ts";
 import {
   readTickConfig,
   readTickRuntimeStatus,
@@ -252,9 +255,9 @@ export interface HaltWithPaneResult extends HaltResult {
   pane: PaneStopResult;
 }
 
-export function hold(projectName?: string): HoldResult {
+export function hold(projectName?: string, source: string = "hold"): HoldResult {
   const wasPaused = isPaused();
-  setPaused(true);
+  setPaused(true, { source });
   return { wasPaused, disarmed: disarmTicks(projectName) };
 }
 
@@ -263,7 +266,7 @@ export function releaseHold(): void {
 }
 
 export async function halt(projectName?: string): Promise<HaltResult> {
-  const held = hold(projectName);
+  const held = hold(projectName, "halt");
   const stop = await stopDaemon();
   return { hold: held, stop };
 }
@@ -1019,8 +1022,27 @@ export function formatFleetStatus(
 
   const graphBlock = formatCodeGraphHealth(codeGraph, now);
 
+  // Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
+  // a halt) answers "who stopped the fleet" without opening a file (#185). An
+  // unparseable sentinel — paused but with no datable line 1 — is itself news:
+  // it means a run admitted before an *unknown* pause cannot prove innocence
+  // (#174), so every mutation is refused.
+  const dispatchLine =
+    layers.dispatch === "paused"
+      ? (() => {
+          const prov = pauseProvenance();
+          if (prov !== undefined) {
+            const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
+            return `dispatch  paused (source: ${prov.source}${reason})`;
+          }
+          return isPaused() && pausedAt() === undefined
+            ? "dispatch  paused (unparseable sentinel — all mutations refused)"
+            : "dispatch  paused";
+        })()
+      : `dispatch  ${layers.dispatch}`;
+
   return [
-    `dispatch  ${layers.dispatch}`,
+    dispatchLine,
     tickLine,
     ...(nextTickLine === undefined ? [] : [nextTickLine]),
     paneLine,
@@ -1053,6 +1075,39 @@ function formatProjectBody(s: StatusSnapshot): string {
     // independent controls and an operator has to see which one stopped the
     // fleet (#110).
     `  plan usage         ${planUsageLine(s.planUsage)}`,
+    // The tracker's API budget, when the renderer could read it. Absent on a
+    // broken `gh`: one missing row, never a broken report (#188). Next to it,
+    // the refusals the tracker actually observed, so a budget that looks
+    // healthy next to every call being refused is still visible (#198).
+    ...(s.github === undefined
+      ? []
+      : [
+          `  github             graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
+            (s.ghRefusals === undefined || s.ghRefusals.count === 0
+              ? ""
+              : ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
+        ]),
+    // The daemon's own observed github traffic today (#198): a separate row
+    // from the polled budget above, so one missing `status` read never hides
+    // the other. `onCall` counts every spawn including ones that end 304, so
+    // `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
+    // difference, and the polled budget row stays the authority (#203).
+    `  github calls       daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
+      ((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
+        ? ""
+        : ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
+    // Label-projection ops the tracker has not applied yet (#201): GitHub is
+    // behind what the store decided, and the operator can see the lag instead
+    // of discovering it as a stale label or a missing one.
+    ...(s.labelOps === undefined
+      ? []
+      : [
+          `  labels             projection ${s.labelOps.pending} pending (oldest ${
+            s.labelOps.oldestAgeMs >= 60_000
+              ? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
+              : `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
+          })`,
+        ]),
     `  new worker turns   ${s.caps.workerMaxTurns}`,
     `  worker wall clock  ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
     `  failed attempts    ${s.caps.maxAttemptsPerIssue}`,
@@ -1097,18 +1152,22 @@ export async function renderStatus(projectName?: string): Promise<string> {
   const layers = fleetLayers(projectName);
   const project = findProject(loadConfig(), projectName);
   const rec = livingDaemon();
-  const [health, telegram, planUsage] = await Promise.all([
+  const [health, telegram, planUsage, github] = await Promise.all([
     rec === undefined ? undefined : healthCheck(rec.port),
     probeTelegramHealth(projectName),
     // Read here rather than in `statusSnapshot`, which is synchronous and used
     // by callers that must not shell out. An unmetered project never spawns
     // the provider at all.
     readPlanUsage(s.caps.planUsage, sharedUsageSource()),
+    // Same reasoning as `planUsage`: the snapshot is synchronous, this read is
+    // a shell-out, and undefined on any failure — one missing row, never a
+    // broken report (#188).
+    fetchRateLimit(),
   ]);
   const cached = codeGraphFromHealthz(health?.body, project.name);
   const codeGraph = cached ?? (await probeCodeGraph(project));
   return formatFleetStatus(
-    { ...s, planUsage },
+    { ...s, planUsage, github },
     layers,
     health,
     telegram,

package/src/label-projection.ts

@@ -0,0 +1,93 @@
+/**
+ * The label projection outbox (#201).
+ *
+ * The dispatcher used to write GitHub labels directly, which made the label —
+ * not the store — the crash guard against double dispatch, and let one refused
+ * label write strand an issue (`#184`, `#198`). Dispatch state is now
+ * store-authoritative; every decided label change is an outbox row that this
+ * module projects onto the tracker with retry.
+ *
+ * Both the daemon tick and the `unblock` CLI path drain through the same
+ * `projectLabels` so a 403 no longer takes either down: the op defers with
+ * backoff and the daemon retries it. Per-issue ops apply in id order, which is
+ * what makes a swap atomic — `add` lands before the paired `remove` exactly as
+ * enqueued.
+ */
+
+import { GhRateLimitError, isRateLimitRefusal } from "./tracker/github.ts";
+import type { ProjectConfig, Store, Tracker } from "./types.ts";
+
+/** Retry backoff for one failed label op: 30s doubling, capped at 15m (#201). */
+export function labelOpBackoffMs(attempts: number): number {
+  return Math.min(30_000 * 2 ** attempts, 15 * 60_000);
+}
+
+function errText(e: unknown): string {
+  return e instanceof Error ? e.message : String(e);
+}
+
+/**
+ * Drain as many due label ops as the tracker will take, in id order.
+ *
+ * A failed op defers with backoff and holds the rest of its issue for the next
+ * pass: a later op for the same issue must not land while an earlier one could
+ * not (that is the atomicity `#184` needed — add-queue-label must not beat
+ * remove-blocked when they were enqueued in that order). Other issues continue.
+ *
+ * Returns what happened so `unblock` can say "queued, the daemon retries"
+ * instead of pretending a deferred op was applied.
+ */
+export async function projectLabels(
+  store: Store,
+  tracker: Tracker,
+  project: ProjectConfig,
+): Promise<{ applied: number; deferred: number }> {
+  let applied = 0;
+  let deferred = 0;
+  // `pendingLabelOps` returns at most the oldest owed row per issue (it
+  // excludes any row that still has an earlier pending sibling), so a swap's
+  // second op can never land before its first — per-issue atomicity enforced
+  // across calls and processes, not just one pass. Re-querying lets a healthy
+  // chain drain fully in this one call (unblock clears every state label in
+  // one go), while a failing op defers its issue and parks the rest until it
+  // settles.
+  for (;;) {
+    const due = store.pendingLabelOps(project.name, Date.now());
+    if (due.length === 0) break;
+    let progressed = false;
+    for (const op of due) {
+      try {
+        if (op.op === "add") await tracker.addLabel(op.issue, op.label);
+        else await tracker.removeLabel(op.issue, op.label);
+        store.settleLabelOp(op.id);
+        applied += 1;
+        progressed = true;
+      } catch (err) {
+        // A rate-limit refusal defers WITHOUT counting an attempt: a shared
+        // outage (all of GitHub refusing) is not this op's fault, so it must
+        // not burn the op's backoff escalation (#208). The circuit breaker's
+        // fast-fail (`GhRateLimitError`) is the most common shape — its stderr
+        // says "circuit breaker open", which `isRateLimitRefusal`'s body regex
+        // does not match — so classify it by type first and honour its
+        // `retryAtMs`. Every other failure counts and doubles the next wait.
+        if (err instanceof GhRateLimitError) {
+          store.deferLabelOp(op.id, errText(err), err.retryAtMs, false);
+        } else {
+          const rateLimited = isRateLimitRefusal(err);
+          store.deferLabelOp(
+            op.id,
+            errText(err),
+            Date.now() + labelOpBackoffMs(op.attempts + 1),
+            !rateLimited,
+          );
+        }
+        deferred += 1;
+      }
+    }
+    // Every remaining op in this pass was deferred (parked behind its backoff),
+    // so re-querying would only re-select the same deferred rows; nothing more
+    // is applicable until a backoff elapses.
+    if (!progressed) break;
+  }
+  return { applied, deferred };
+}

package/src/lifecycle.ts

@@ -292,6 +292,13 @@ export type StopResult =
   | { kind: "stopped"; pid: number; via: "systemctl" | "signal" }
   | { kind: "not-running" };
 
+/** What {@link restartDaemon} returns: the pre-restart record, the new one, and the path taken. */
+export interface RestartResult {
+  previous: DaemonRecord | undefined;
+  record: DaemonRecord;
+  via: "systemctl" | "cli";
+}
+
 /**
  * Stops the daemon.
  *
@@ -365,7 +372,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
  */
 export async function restartDaemon(
   o: { port?: number; project?: string; timeoutMs?: number } = {},
-): Promise<{ previous: DaemonRecord | undefined; record: DaemonRecord; via: "systemctl" | "cli" }> {
+): Promise<RestartResult> {
   const previous = livingDaemon();
   const ownership = probeUnit();
   if (ownership.kind === "unknown") {